"""Ingestion: this website's lesson pages → clean, size-capped chunks with metadata (lesson 07).

Each lesson is HTML with an <h1> and <h2 id="..."> sections. We keep <main>, drop scripts and quizzes
(their wrong answer options are false statements), turn diagrams into their aria-label text, split at
<h2>, then split long sections at paragraph boundaries so no chunk exceeds MAX_WORDS.
"""

from __future__ import annotations

import hashlib
import os
import re
from dataclasses import dataclass
from pathlib import Path

from bs4 import BeautifulSoup

MAX_WORDS = 220
LESSON_FILE = re.compile(r"^\d\d-.*\.html$")


@dataclass(frozen=True)
class Chunk:
    id: str             # stable: url plus a part number when a section is split
    course: str
    course_name: str
    lesson: str
    title: str
    section: str
    url: str            # course/lesson.html#anchor: a real link into the site
    text: str

    @property
    def content_hash(self) -> str:
        return hashlib.sha256(self.text.encode()).hexdigest()[:16]

    def header(self) -> str:
        """Contextual header, prepended when indexing (lesson 07)."""
        return f"{self.course_name} › {self.title} › {self.section}"


def site_root() -> Path:
    if os.environ.get("LWP_SITE"):
        return Path(os.environ["LWP_SITE"])
    for folder in Path(__file__).resolve().parents:
        if (folder / "shared" / "course.css").exists():
            return folder
    raise FileNotFoundError("cannot find the site root; set LWP_SITE to the folder containing shared/")


def discover_courses(root: Path) -> dict[str, str]:
    """Every folder with a curriculum.js is a course: {folder: display name}."""
    courses = {}
    for cur in sorted(root.glob("*/curriculum.js")):
        match = re.search(r'name:\s*"([^"]+)"', cur.read_text(encoding="utf-8"))
        courses[cur.parent.name] = match.group(1) if match else cur.parent.name
    return courses


BREAK = "\x00"          # a non-whitespace marker for "line break here", turned into "\n" at the end


def _clean(main) -> None:
    # quizzes: the wrong options are false statements. .out: printed example output, which on this
    # site includes other questions and URLs and polluted retrieval until it was dropped.
    for tag in main.select("script, style, #lessonFoot, .quiz, .out"):
        tag.decompose()
    for svg in main.find_all("svg"):
        label = svg.get("aria-label")
        svg.replace_with(f"{BREAK}[diagram: {label}]{BREAK}" if label else "")
    for pre in main.find_all("pre"):                   # keep code line by line
        pre.string = pre.get_text().strip("\n").replace("\n", BREAK)
    for tag in main.select("h3, h4, summary, li, tr, .lbl"):
        tag.append(BREAK)                              # headings and list items end their line


def _text(el) -> str:
    """Element text with HTML source wrapping collapsed, but real line breaks kept."""
    flat = " ".join(el.get_text(" ").split())
    lines = (line.strip() for line in flat.replace(BREAK, "\n").splitlines())
    return "\n".join(line for line in lines if line)


def _split(parts: list[str], max_words: int = MAX_WORDS) -> list[str]:
    """Pack paragraphs into chunks of at most max_words; overlong paragraphs split at sentences."""
    units = []
    for p in parts:
        units += [p] if len(p.split()) <= max_words else re.split(r"(?<=[.!?])\s+", p)
    chunks, current, size = [], [], 0
    for u in units:
        n = len(u.split())
        if current and size + n > max_words:
            chunks.append("\n".join(current))
            current, size = [current[-1]], len(current[-1].split())      # one paragraph of overlap
            if size + n > max_words:
                current, size = [], 0
        current.append(u)
        size += n
    if current:
        chunks.append("\n".join(current))
    return chunks


def parse_lesson(path: Path, course: str, course_name: str) -> list[Chunk]:
    soup = BeautifulSoup(path.read_text(encoding="utf-8"), "lxml")
    main = soup.find("main")
    if main is None or main.find("h1") is None:
        return []
    _clean(main)
    title = main.find("h1").get_text(" ", strip=True)
    sections, current = [], ("Introduction", "", [])
    for el in main.find_all(recursive=False):
        if el.name == "h2":
            sections.append(current)
            current = (el.get_text(" ", strip=True), el.get("id", ""), [])
        elif el.name != "h1" and "crumb" not in (el.get("class") or []):
            current[2].extend(line for line in _text(el).splitlines() if line)
    sections.append(current)

    chunks = []
    for name, anchor, parts in sections:
        if sum(len(p.split()) for p in parts) < 8:
            continue
        url = f"{course}/{path.name}" + (f"#{anchor}" if anchor else "")
        pieces = _split(parts)
        for i, text in enumerate(pieces, start=1):
            cid = url if len(pieces) == 1 else f"{url}~{i}"
            chunks.append(Chunk(cid, course, course_name, path.name, title, name, url, text))
    return chunks


def load_chunks(root: Path | None = None, courses: list[str] | None = None) -> list[Chunk]:
    root = root or site_root()
    names = discover_courses(root)
    chunks = []
    for course in courses or list(names):
        for path in sorted((root / course).glob("*.html")):
            if LESSON_FILE.match(path.name):
                chunks += parse_lesson(path, course, names.get(course, course))
    return chunks