"""site_docs.py — this website's own lessons as a small document collection for the RAG examples.

Every lesson page is HTML with an <h1> title and <h2 id="..."> sections. load_sections() turns each
<h2> section into one record, with enough metadata to cite it and link straight to it:

    {"course": "sql", "lesson": "06-pagination.html", "title": "Pagination",
     "section": "3. Keyset pagination", "url": "sql/06-pagination.html#keyset", "text": "..."}

Where is the site? $LWP_SITE if set, otherwise the first folder at or above the current directory
(or this file) that contains shared/course.css. So it works when you run an example from inside
ai-engineering/ or from the repository root.
"""

from __future__ import annotations

import os
import re
from pathlib import Path

from bs4 import BeautifulSoup

# Courses whose lessons are finished. The examples use these so their numbers stay reproducible.
STABLE_COURSES = ("course", "sql", "mysql", "python", "dsa", "databricks", "openai-sdk")
COURSE_NAMES = {"course": "PySpark", "sql": "SQL", "mysql": "MySQL", "python": "Advanced Python",
                "dsa": "DSA in Python", "databricks": "Databricks", "openai-sdk": "OpenAI SDK",
                "ai-engineering": "AI Engineering", "core-python": "Core Python", "fastapi": "FastAPI"}
LESSON_FILE = re.compile(r"^\d\d-.*\.html$")


def site_root() -> Path:
    if os.environ.get("LWP_SITE"):
        return Path(os.environ["LWP_SITE"])
    for start in (Path.cwd(), Path(__file__).resolve().parent):
        for folder in (start, *start.parents):
            if (folder / "shared" / "course.css").exists():
                return folder
    raise FileNotFoundError("run this from inside the learn-with-project site, or set LWP_SITE")


def _clean(main) -> None:
    """Drop what should never be retrieved; keep diagrams as their text description."""
    for tag in main.select("script, style, #lessonFoot, .quiz"):
        tag.decompose()                      # quiz: the wrong options are false statements
    for svg in main.find_all("svg"):
        label = svg.get("aria-label")
        svg.replace_with(f"[diagram: {label}]" if label else "")


def parse_lesson(path: Path, course: str) -> list[dict]:
    soup = BeautifulSoup(path.read_text(encoding="utf-8"), "lxml")
    main = soup.find("main")
    if main is None:
        return []
    _clean(main)
    title = main.find("h1").get_text(" ", strip=True)
    sections, current = [], {"section": "Introduction", "anchor": "", "parts": []}
    for el in main.find_all(recursive=False):
        if el.name == "h2":
            sections.append(current)
            current = {"section": el.get_text(" ", strip=True), "anchor": el.get("id", ""), "parts": []}
        elif el.name != "h1" and "crumb" not in (el.get("class") or []):
            text = el.get_text(" ", strip=True)
            if text:
                current["parts"].append(text)
    sections.append(current)
    out = []
    for s in sections:
        text = "\n".join(s["parts"]).strip()
        if len(text) < 40:                   # headings with nothing under them
            continue
        url = f"{course}/{path.name}" + (f"#{s['anchor']}" if s["anchor"] else "")
        out.append({"course": course, "lesson": path.name, "title": title, "section": s["section"],
                    "url": url, "text": text})
    return out


def load_sections(courses=STABLE_COURSES) -> list[dict]:
    root = site_root()
    docs = []
    for course in courses:
        folder = root / course
        for path in sorted(folder.glob("*.html")):
            if LESSON_FILE.match(path.name):
                docs += parse_lesson(path, course)
    return docs


if __name__ == "__main__":
    docs = load_sections()
    print(len(docs), "sections from", len({d["url"].split("#")[0] for d in docs}), "lessons")
    print(docs[0]["url"], "|", docs[0]["title"], "|", docs[0]["section"])