from __future__ import annotations

import argparse
import asyncio
import json
import re
from pathlib import Path
from urllib.parse import urljoin, urlparse

from core_db import SOURCE_ROOT, connect
from secret_store import load_secret
from source_store import (
    init_source_schema,
    save_snapshot,
    set_login_ok,
    set_source_error,
    set_source_sync_result,
    source_status,
)


BASE_URL_DEFAULT = "http://nps3.cntd.ru/docs"

LOGIN_WORDS = ("Вход", "Войти", "Авторизация", "Личный кабинет")
SEARCH_WORDS = ("поиск", "найти", "search", "запрос")
BAD_LOGIN_WORDS = (
    "неверный пароль",
    "неверный логин",
    "ошибка авторизации",
    "неправильный пароль",
)


def _require_playwright():
    try:
        from playwright.async_api import async_playwright
    except ImportError as exc:
        raise RuntimeError(
            "Playwright не установлен. Установите зависимости QUALITY."
        ) from exc
    return async_playwright


async def _first_visible(locator, max_items: int = 50):
    try:
        count = await locator.count()
    except Exception:
        return None

    for i in range(min(count, max_items)):
        item = locator.nth(i)
        try:
            if await item.is_visible():
                return item
        except Exception:
            pass
    return None


async def _frame_body_text(frame) -> str:
    try:
        return await frame.locator("body").inner_text(timeout=3000)
    except Exception:
        return ""


async def _auth_state(page) -> dict:
    """
    TechExpert 6 intranet exposes authorization state in the shell:
      window.noAuthorized
      window.userSettings

    We intentionally verify it instead of assuming that the absence
    of a login form means successful authentication.
    """
    try:
        return await page.evaluate(
            """() => ({
                noAuthorized:
                    typeof window.noAuthorized === 'boolean'
                        ? window.noAuthorized
                        : null,
                hasUser:
                    !!(
                        window.userSettings &&
                        typeof window.userSettings === 'object' &&
                        (
                            window.userSettings.LogID ||
                            window.userSettings.LoginName ||
                            window.userSettings.FullName
                        )
                    ),
                product:
                    typeof window.ProductName === 'string'
                        ? window.ProductName
                        : ''
            })"""
        )
    except Exception:
        return {
            "noAuthorized": None,
            "hasUser": False,
            "product": "",
        }


async def _find_login_form(page):
    # Main document.
    password = await _first_visible(page.locator('input[type="password"]'))
    if password:
        return page.main_frame, password

    # Nested frames.
    for frame in page.frames:
        if frame == page.main_frame:
            continue
        try:
            password = await _first_visible(
                frame.locator('input[type="password"]')
            )
            if password:
                return frame, password
        except Exception:
            pass

    # Buttons/links which can open a login form.
    for word in LOGIN_WORDS:
        for frame in list(page.frames):
            try:
                candidate = frame.get_by_text(word, exact=False)
                item = await _first_visible(candidate)
                if not item:
                    continue
                await item.click()
                await page.wait_for_timeout(800)

                for target in list(page.frames):
                    password = await _first_visible(
                        target.locator('input[type="password"]')
                    )
                    if password:
                        return target, password
            except Exception:
                continue

    return None, None


async def _login(page, username: str, password: str, base_url: str) -> dict:
    """
    Exact login flow for this TechExpert 6 intranet installation.

    Real page behavior discovered from /docs/:
      form: #authForm
      login: #user / name=login
      password: #pass / name=password
      submit: #submitButton
      AJAX POST: /users/login.asp
      payload: user=<login>, pass=<password>, path=/docs

    The initial /docs/ request also performs a cookie availability check.
    """
    maintenance_titles = (
        "регламентные работы",
        "технические работы",
    )
    cookie_error_titles = (
        "браузер не принимает cookies",
        "не удалось установить сессию",
    )
    auth_title = "авторизация по базе пользователей"

    # A few retries are useful because this installation can temporarily
    # return a maintenance page before the real /docs/ application.
    for attempt in range(4):
        response = await page.goto(
            base_url,
            wait_until="domcontentloaded",
            timeout=60000,
        )
        await page.wait_for_timeout(1200)

        title = (await page.title()).strip()
        title_lower = title.lower()

        state = await _auth_state(page)
        if state.get("noAuthorized") is False and state.get("hasUser"):
            return state

        if any(x in title_lower for x in maintenance_titles):
            if attempt < 3:
                await page.wait_for_timeout(3500)
                continue
            raise RuntimeError(
                "Техэксперт временно недоступен: сервер возвращает страницу "
                f"«{title or 'Регламентные работы'}». "
                "Логин и пароль сохранены, повторите проверку позже."
            )

        if any(x in title_lower for x in cookie_error_titles):
            raise RuntimeError(
                "Техэксперт не смог создать cookie-сессию в браузере. "
                f"Страница: «{title}»."
            )

        # Exact authorization page used by nps3.cntd.ru.
        auth_form = page.locator("#authForm")
        auth_form_visible = False
        try:
            auth_form_visible = (
                await auth_form.count() > 0
                and await auth_form.first.is_visible()
            )
        except Exception:
            auth_form_visible = False

        if auth_title in title_lower or auth_form_visible:
            user_input = page.locator("#user")
            pass_input = page.locator("#pass")
            submit = page.locator("#submitButton")

            if await user_input.count() == 0:
                user_input = page.locator('input[name="login"]')
            if await pass_input.count() == 0:
                pass_input = page.locator('input[name="password"]')
            if await submit.count() == 0:
                submit = page.locator('input[type="submit"]')

            if (
                await user_input.count() == 0
                or await pass_input.count() == 0
                or await submit.count() == 0
            ):
                raise RuntimeError(
                    "Страница авторизации Техэксперта открылась, "
                    "но обязательные поля #user / #pass / #submitButton "
                    "не найдены."
                )

            await user_input.first.fill(username)
            await pass_input.first.fill(password)

            # formSubmit() intercepts submit and performs:
            # POST /users/login.asp
            # user=<...>&pass=<...>&path=/docs
            try:
                async with page.expect_response(
                    lambda r: (
                        "/users/login.asp" in r.url
                        and r.request.method.upper() == "POST"
                    ),
                    timeout=30000,
                ) as response_info:
                    await submit.first.click()
                login_response = await response_info.value
            except Exception as exc:
                raise RuntimeError(
                    "Форма входа найдена, но Техэксперт не выполнил "
                    "ожидаемый POST /users/login.asp."
                ) from exc

            if login_response.status >= 400:
                raw = ""
                try:
                    raw = await login_response.text()
                except Exception:
                    pass

                message = ""
                try:
                    payload = json.loads(raw or "{}")
                    error = str(payload.get("error") or "").strip()
                    msg = str(payload.get("msg") or "").strip()
                    user = str(payload.get("user") or "").strip()
                    limit = payload.get("limit")

                    if limit:
                        message = (
                            f"Вход в систему для {user or 'пользователя'} "
                            "ограничен."
                        )
                        if error:
                            message += f" {error}"
                        if msg:
                            message += f" {msg}"
                    elif error or msg:
                        message = " ".join(x for x in (error, msg) if x)
                except Exception:
                    pass

                if not message:
                    message = (
                        "Техэксперт отклонил логин или пароль "
                        f"(HTTP {login_response.status})."
                    )

                raise RuntimeError(message[:1800])

            # On success TechExpert executes window.location.reload().
            # Wait for the page to reload and for the authenticated shell
            # variables to appear.
            try:
                await page.wait_for_load_state(
                    "domcontentloaded",
                    timeout=30000,
                )
            except Exception:
                pass

            for _ in range(60):
                await page.wait_for_timeout(500)

                state = await _auth_state(page)
                if (
                    state.get("noAuthorized") is False
                    and state.get("hasUser")
                ):
                    return state

                current_title = (await page.title()).strip().lower()

                if auth_title in current_title:
                    # The page can stay on the auth title briefly while JS
                    # processes the response. Keep waiting.
                    continue

                if any(x in current_title for x in maintenance_titles):
                    raise RuntimeError(
                        "Авторизация отправлена, но Техэксперт после входа "
                        "перешёл на страницу регламентных работ."
                    )

                combined = " ".join(
                    (await _frame_body_text(f)).lower()
                    for f in list(page.frames)
                )
                if any(word in combined for word in BAD_LOGIN_WORDS):
                    raise RuntimeError(
                        "Техэксперт отклонил логин или пароль."
                    )

            raise RuntimeError(
                "POST /users/login.asp выполнен успешно, "
                "но Техэксперт не подтвердил пользовательскую сессию "
                "после перезагрузки /docs/."
            )

        # If the server returned another page, do not misclassify it
        # as a bad password. Give a precise diagnostic.
        status = response.status if response else "?"
        raise RuntimeError(
            "Техэксперт вернул неожиданную страницу до авторизации: "
            f"HTTP {status}, title «{title or 'без заголовка'}»."
        )

    raise RuntimeError("Не удалось открыть страницу авторизации Техэксперта.")


async def _wait_for_techexpert_ui(page, timeout_ms: int = 45000) -> None:
    """
    The uploaded TechExpert 6 page is only a shell. Its actual work area
    is loaded asynchronously into an iframe which initially has about:blank.
    Wait until at least one child frame navigates or a usable search control
    appears.
    """
    deadline = asyncio.get_running_loop().time() + timeout_ms / 1000

    while asyncio.get_running_loop().time() < deadline:
        # A child frame with a real URL is the strongest signal that
        # TechExpert finished opening its start page.
        for frame in list(page.frames):
            if frame == page.main_frame:
                continue
            try:
                if frame.url and frame.url != "about:blank":
                    return
            except Exception:
                pass

        found = await _find_search_input(page, wait=False)
        if found:
            return

        await page.wait_for_timeout(500)

    # Do not fail here. The next stage produces a more useful diagnostic
    # including all frame URLs and inputs.
    return


async def _search_candidates_in_frame(frame):
    selectors = [
        'input[type="search"]',
        'input[placeholder*="поиск" i]',
        'input[placeholder*="найти" i]',
        'input[placeholder*="запрос" i]',
        'input[name*="search" i]',
        'input[id*="search" i]',
        'input[name*="query" i]',
        'input[id*="query" i]',
        'input[class*="search" i]',
        'textarea[placeholder*="поиск" i]',
        'textarea[name*="search" i]',
        '[contenteditable="true"][class*="search" i]',
    ]

    for selector in selectors:
        try:
            item = await _first_visible(frame.locator(selector))
            if item:
                return item
        except Exception:
            continue

    # Generic visible text fields: use attributes and nearby accessible name
    # to avoid grabbing unrelated controls.
    try:
        inputs = frame.locator(
            'input[type="text"], input:not([type]), textarea, '
            '[contenteditable="true"]'
        )
        count = await inputs.count()
    except Exception:
        return None

    for i in range(min(count, 80)):
        item = inputs.nth(i)
        try:
            if not await item.is_visible():
                continue

            attrs = []
            for attr in (
                "placeholder",
                "name",
                "id",
                "class",
                "title",
                "aria-label",
            ):
                attrs.append(await item.get_attribute(attr) or "")

            haystack = " ".join(attrs).lower()
            if any(word in haystack for word in SEARCH_WORDS):
                return item
        except Exception:
            continue

    return None


async def _find_search_input(page, *, wait: bool = True):
    attempts = 50 if wait else 1

    for _ in range(attempts):
        # Search in every frame because TechExpert 6 keeps its work page
        # in an iframe and the main document only contains the shell.
        for frame in list(page.frames):
            item = await _search_candidates_in_frame(frame)
            if item:
                return frame, item

        if wait:
            await page.wait_for_timeout(500)

    return None


def _code_tokens(code: str) -> list[str]:
    clean = (
        code.upper()
        .replace("ЕАЭС", "")
        .replace("ТС", "")
        .replace("РФ", "")
    )
    return [x for x in re.findall(r"\d{2,4}", clean) if x]


def _normalize_nd_value(value: str | None) -> str | None:
    if not value:
        return None

    value = str(value).strip()

    if value.isdigit():
        return value

    found = _extract_nd(value)
    if found:
        return found

    match = re.search(r"\b(\d{6,})\b", value)
    return match.group(1) if match else None


def _result_score(code: str, text: str) -> int:
    blob = re.sub(r"\s+", " ", text).upper()
    wanted = code.upper()
    score = 0

    if wanted in blob:
        score += 24

    for token in _code_tokens(code):
        if token in blob:
            score += 3

    if "ТЕХНИЧЕСК" in blob and "РЕГЛАМЕНТ" in blob:
        score += 14

    if "О ПРИНЯТИИ" in blob:
        score -= 5

    if "ПЕРЕЧЕН" in blob or "ПРОГРАММ" in blob:
        score -= 5

    if "ИЗМЕНЕНИ" in blob and "ТЕХНИЧЕСК" not in blob:
        score -= 3

    return score


async def _search_result_candidates_in_frame(frame, code: str) -> list[dict]:
    """
    TechExpert search results are not always ordinary <a> elements.
    In the observed UI the result title can live in <span class="title">,
    while the internal nd is stored on a parent/sibling/descendant element.

    Extract candidates directly from the DOM and walk nearby nodes to find
    TechExpert's internal document id.
    """
    script = r"""
    (wantedCode) => {
      const out = [];
      const seen = new Set();

      const normalize = (s) =>
        String(s || '').replace(/\s+/g, ' ').trim();

      const ndFromValue = (v) => {
        const s = String(v || '');
        if (/^\d+$/.test(s.trim())) return s.trim();

        let m = s.match(/(?:^|[?&])nd=(\d+)/i);
        if (m) return m[1];

        m = s.match(/\b(\d{6,})\b/);
        return m ? m[1] : null;
      };

      const ownNd = (el) => {
        if (!el || !el.getAttribute) return null;

        const attrs = [
          'nd',
          'data-nd',
          'data-href',
          'href',
          'data-url',
          'data-document',
          'data-doc'
        ];

        for (const a of attrs) {
          const nd = ndFromValue(el.getAttribute(a));
          if (nd) return nd;
        }
        return null;
      };

      const nearbyNd = (node) => {
        let cur = node;

        for (let depth = 0; cur && depth < 8; depth++, cur = cur.parentElement) {
          let nd = ownNd(cur);
          if (nd) return nd;

          try {
            const nested = cur.querySelector(
              '[nd],[data-nd],[data-href*="nd="],[href*="nd="]'
            );
            nd = ownNd(nested);
            if (nd) return nd;
          } catch (_) {}
        }

        return null;
      };

      const nodes = document.querySelectorAll([
        'span.title',
        '.title',
        'a',
        '[nd]',
        '[data-nd]',
        '[data-href*="nd="]',
        '[href*="nd="]'
      ].join(','));

      for (const node of nodes) {
        const ownText = normalize(node.innerText || node.textContent);
        if (!ownText) continue;

        const upper = ownText.toUpperCase();
        const codeUpper = String(wantedCode || '').toUpperCase();

        const maybeRelevant =
          upper.includes(codeUpper) ||
          upper.includes('ТЕХНИЧЕСК') ||
          upper.includes('РЕГЛАМЕНТ');

        if (!maybeRelevant) continue;

        const nd = nearbyNd(node);
        if (!nd) continue;

        let contextText = ownText;
        let cur = node.parentElement;
        for (let depth = 0; cur && depth < 4; depth++, cur = cur.parentElement) {
          const t = normalize(cur.innerText || cur.textContent);
          if (t && t.length >= contextText.length && t.length < 1800) {
            contextText = t;
          }
        }

        const key = nd + '|' + ownText;
        if (seen.has(key)) continue;
        seen.add(key);

        out.push({
          nd,
          text: ownText,
          context: contextText
        });
      }

      return out.slice(0, 500);
    }
    """

    try:
        rows = await frame.evaluate(script, code)
    except Exception:
        return []

    result = []

    for row in rows or []:
        nd = _normalize_nd_value(row.get("nd"))
        text = str(row.get("text") or "").strip()
        context = str(row.get("context") or "").strip()

        if not nd or not text:
            continue

        result.append(
            {
                "frame": frame,
                "nd": nd,
                "text": text,
                "context": context,
                "score": max(
                    _result_score(code, text),
                    _result_score(code, context),
                ),
            }
        )

    return result


async def _best_result_candidate(page, code: str):
    """
    Return the best TechExpert result using its internal nd.

    This is intentionally not limited to visible <a> tags because the
    TechExpert 6 result list stores titles and document ids in mixed DOM
    structures.
    """
    candidates = []

    for frame in list(page.frames):
        candidates.extend(
            await _search_result_candidates_in_frame(frame, code)
        )

    if not candidates:
        return None

    # Strong preference: exact code + "technical regulation".
    candidates.sort(
        key=lambda x: (
            x["score"],
            "ТЕХНИЧЕСК" in x["context"].upper(),
            "РЕГЛАМЕНТ" in x["context"].upper(),
            -len(x["context"]),
        ),
        reverse=True,
    )

    best = candidates[0]

    # Avoid opening loosely related result cards.
    if best["score"] < 20:
        return None

    return best


async def _frame_signature(frame) -> tuple[str, str]:
    try:
        url = frame.url or ""
    except Exception:
        url = ""
    text = await _frame_body_text(frame)
    return url, text[:8000]


async def _search_document(
    page,
    base_url: str,
    code: str,
    query: str,
):
    await page.goto(
        base_url,
        wait_until="domcontentloaded",
        timeout=60000,
    )

    await _wait_for_techexpert_ui(page)

    found = await _find_search_input(page, wait=True)
    if not found:
        raise RuntimeError(
            "Интерфейс Техэксперта загружен, но строка поиска "
            "не найдена ни в основном окне, ни во вложенных фреймах."
        )

    search_frame, search_input = found
    await search_input.fill(query)
    await search_input.press("Enter")

    # TechExpert renders search results asynchronously.
    candidate = None

    for _ in range(60):
        await page.wait_for_timeout(500)
        candidate = await _best_result_candidate(page, code)
        if candidate:
            break

    if not candidate:
        raise RuntimeError(
            f"Поиск выполнен, но документ «{code}» "
            "не найден в результатах."
        )

    nd = candidate["nd"]
    result_text = candidate["text"] or candidate["context"]

    target_url = (
        base_url.rstrip("/")
        + "/?nd="
        + nd
    )

    await page.goto(
        target_url,
        wait_until="domcontentloaded",
        timeout=60000,
    )
    await page.wait_for_timeout(1200)

    document_frame = None

    for _ in range(50):
        document_frame = await _pick_document_frame(page, code)

        if document_frame:
            text = await _frame_body_text(document_frame)

            try:
                has_doc_dom = (
                    await document_frame.locator(".document.activeDoc").count() > 0
                    or await document_frame.locator("#tabBody_0").count() > 0
                )
            except Exception:
                has_doc_dom = False

            if has_doc_dom and len(text.strip()) >= 200:
                break

        await page.wait_for_timeout(400)

    if not document_frame:
        raise RuntimeError(
            f"Результат найден (nd={nd}), но документ не открылся."
        )

    return document_frame, target_url, result_text


def _is_technical_regulation_code(code: str) -> bool:
    return code.strip().upper().startswith("ТР ")


def _extract_nd(value: str | None) -> str | None:
    if not value:
        return None
    match = re.search(r"(?:^|[?&])nd=(\d+)", value)
    return match.group(1) if match else None


async def _find_linked_regulation(frame, code: str):
    """
    Search pages often open the EEC/Customs Union decision which *adopts*
    a technical regulation, not the regulation itself.

    Example discovered for TR TS 004/2011:
      wrapper nd=902298070
      linked regulation nd=902299536

    Prefer an internal TechExpert document link whose visible text contains
    the exact TR code and identifies a technical regulation.
    """
    try:
        links = frame.locator('a[data-href*="nd="], a[rel="document"]')
        count = await links.count()
    except Exception:
        return None

    wanted = code.upper()
    best = None
    best_score = -1

    for i in range(min(count, 1200)):
        link = links.nth(i)
        try:
            text = re.sub(
                r"\s+",
                " ",
                (await link.inner_text()).strip(),
            )
            data_href = await link.get_attribute("data-href") or ""
            href = await link.get_attribute("href") or ""
            nd = _extract_nd(data_href) or _extract_nd(href)
            if not nd:
                continue

            upper = text.upper()
            score = 0

            if wanted in upper:
                score += 12
            for token in _code_tokens(code):
                if token in upper:
                    score += 2

            if "ТЕХНИЧЕСК" in upper and "РЕГЛАМЕНТ" in upper:
                score += 8

            if "О ПРИНЯТИИ" in upper or "РЕШЕНИЕ" in upper:
                score -= 6

            if score > best_score and score >= 12:
                best_score = score
                best = {
                    "nd": nd,
                    "text": text,
                    "data_href": data_href,
                }
        except Exception:
            continue

    return best


async def _pick_document_frame(page, code: str):
    """
    Pick the frame that really contains an opened TechExpert document.

    Important: the literal code (for example "ТР ТС 004/2011") is not
    guaranteed to be printed inside the regulation text itself. Therefore
    DOM structure is a stronger signal than an exact text match.
    """
    wanted = code.upper()
    best = None
    best_score = -1

    for frame in list(page.frames):
        text = await _frame_body_text(frame)
        if len(text.strip()) < 200:
            continue

        blob = text.upper()
        score = 0

        # Strong TechExpert document-page signals.
        try:
            if await frame.locator(".document.activeDoc").count() > 0:
                score += 12
        except Exception:
            pass

        try:
            if await frame.locator("#tabBody_0").count() > 0:
                score += 4
        except Exception:
            pass

        try:
            if await frame.locator(".text-for-mark").count() > 0:
                score += 4
        except Exception:
            pass

        if wanted in blob:
            score += 8

        for token in _code_tokens(code):
            if token in blob:
                score += 1

        if "ТЕХНИЧЕСК" in blob and "РЕГЛАМЕНТ" in blob:
            score += 3

        try:
            title = (await frame.title()).upper()
        except Exception:
            title = ""

        if wanted in title:
            score += 5

        if "О ПРИНЯТИИ ТЕХНИЧЕСКОГО РЕГЛАМЕНТА" in title:
            score -= 8

        if score > best_score:
            best_score = score
            best = frame

    return best


async def _follow_actual_regulation(
    page,
    frame,
    source_url: str,
    code: str,
    base_url: str,
):
    """
    If TechExpert opened the adoption decision instead of the TR text,
    follow the internal linked regulation.

    Returns:
      (
        frame,
        canonical_url,
        followed,
        parent_url,
        expected_nd,
        candidate_title,
      )
    """
    if not _is_technical_regulation_code(code):
        return frame, source_url, False, None, None, None

    try:
        title = (await frame.title()).strip()
    except Exception:
        title = ""

    text = await _frame_body_text(frame)
    lower = (title + "\n" + text[:8000]).lower()

    wrapper_signals = (
        "о принятии технического регламента",
        "о принятии техрегламента",
    )

    candidate = await _find_linked_regulation(frame, code)

    should_follow = any(x in lower for x in wrapper_signals)

    if not should_follow and candidate:
        current_nd = _extract_nd(source_url)
        if current_nd and candidate["nd"] != current_nd:
            body_has_regulation_structure = bool(
                re.search(
                    r"\bстатья\s+1\b|\bобласть\s+применения\b",
                    text,
                    re.I,
                )
            )
            should_follow = not body_has_regulation_structure

    if not should_follow:
        return frame, source_url, False, None, _extract_nd(source_url), None

    if not candidate:
        raise RuntimeError(
            f"Открыто решение о принятии {code}, "
            "но ссылка на сам технический регламент не найдена."
        )

    parent_url = source_url
    expected_nd = candidate["nd"]
    candidate_title = candidate["text"]

    target_url = base_url.rstrip("/") + "/?nd=" + expected_nd

    await page.goto(
        target_url,
        wait_until="domcontentloaded",
        timeout=60000,
    )

    # TechExpert can take several seconds to construct the real document DOM.
    confirmed_frame = None

    for _ in range(60):
        await page.wait_for_timeout(400)

        target_frame = await _pick_document_frame(page, code)
        if not target_frame:
            continue

        try:
            target_title = (await target_frame.title()).strip().lower()
        except Exception:
            target_title = ""

        target_text = await _frame_body_text(target_frame)

        has_document_dom = False
        try:
            has_document_dom = (
                await target_frame.locator(".document.activeDoc").count() > 0
                or await target_frame.locator("#tabBody_0").count() > 0
            )
        except Exception:
            pass

        is_wrapper = (
            "о принятии технического регламента" in target_title
            or "о принятии технического регламента" in target_text[:3000].lower()
        )

        if has_document_dom and not is_wrapper and len(target_text.strip()) >= 300:
            confirmed_frame = target_frame
            break

    if not confirmed_frame:
        raise RuntimeError(
            f"Найдена ссылка на {code} (nd={expected_nd}), "
            "но TechExpert не открыл DOM самого технического регламента."
        )

    return (
        confirmed_frame,
        target_url,
        True,
        parent_url,
        expected_nd,
        candidate_title,
    )


async def _expand_and_load_document(frame) -> None:
    """
    TechExpert can split long legal texts into several sequential DOM blocks
    and can render additional content after scrolling.

    Before extraction:
      - expand collapsed spoiler blocks;
      - scroll document containers through their full height;
      - wait until text length stabilizes.
    """
    # Expand collapsed amendment / information sections when possible.
    try:
        spoilers = frame.locator(
            ".spoilerBlock.spoilerClose .spoilerText, "
            ".spoilerBlock.spoilerClose .spoiler"
        )
        count = await spoilers.count()
        for i in range(min(count, 100)):
            try:
                item = spoilers.nth(i)
                if await item.is_visible():
                    await item.click(timeout=1200)
            except Exception:
                pass
    except Exception:
        pass

    stable_rounds = 0
    previous = None

    for _ in range(60):
        try:
            metrics = await frame.evaluate(
                """() => {
                    const roots = [
                        ...document.querySelectorAll(
                            '.document.activeDoc .k6ScrollBlock, '
                            + '.document.activeDoc .scrollBlock, '
                            + '.document.activeDoc .text-for-mark'
                        )
                    ];

                    for (const el of roots) {
                        try {
                            const max = Math.max(
                                0,
                                el.scrollHeight - el.clientHeight
                            );
                            if (max > 0) {
                                el.scrollTop = Math.min(
                                    max,
                                    el.scrollTop
                                    + Math.max(300, el.clientHeight * 0.85)
                                );
                                el.dispatchEvent(
                                    new Event('scroll', {bubbles: true})
                                );
                            }
                        } catch (_) {}
                    }

                    try {
                        window.scrollBy(
                            0,
                            Math.max(500, window.innerHeight * 0.85)
                        );
                    } catch (_) {}

                    const blocks = [
                        ...document.querySelectorAll(
                            '.document.activeDoc .text-for-mark'
                        )
                    ];

                    const lengths = blocks.map(
                        el => (el.textContent || '').length
                    );

                    return {
                        blocks: blocks.length,
                        totalText: lengths.reduce((a, b) => a + b, 0),
                        bodyHeight:
                            document.documentElement
                                ? document.documentElement.scrollHeight
                                : 0,
                    };
                }"""
            )
        except Exception:
            break

        signature = (
            metrics.get("blocks", 0),
            metrics.get("totalText", 0),
            metrics.get("bodyHeight", 0),
        )

        if signature == previous:
            stable_rounds += 1
        else:
            stable_rounds = 0
            previous = signature

        # Give TechExpert's JS time to append the next block.
        await asyncio.sleep(0.25)

        if stable_rounds >= 6:
            break

    # Return to top; this does not affect extraction but keeps screenshots
    # and subsequent UI interaction predictable.
    try:
        await frame.evaluate(
            """() => {
                window.scrollTo(0, 0);
                for (const el of document.querySelectorAll(
                    '.document.activeDoc .k6ScrollBlock, '
                    + '.document.activeDoc .scrollBlock'
                )) {
                    try { el.scrollTop = 0; } catch (_) {}
                }
            }"""
        )
    except Exception:
        pass


async def _extract_clean_document_text(frame) -> str:
    """
    Extract the complete legal text.

    Important TechExpert detail:
    long documents can contain MULTIPLE sequential `.text-for-mark` blocks.
    Previous versions chose only the longest block, which truncated documents
    such as TR TS 020/2011.

    0.4.8 concatenates all document text blocks in DOM order.
    """
    await _expand_and_load_document(frame)

    try:
        result = await frame.evaluate(
            """() => {
                const normalizeBlock = (el) => {
                    if (!el) return '';

                    // textContent is intentionally preferred because it also
                    // contains text in collapsed amendment/spoiler sections.
                    let text = el.textContent || el.innerText || '';
                    return String(text)
                        .replace(/\\u00a0/g, ' ')
                        .replace(/[ \\t]+\\n/g, '\\n')
                        .replace(/\\n{4,}/g, '\\n\\n\\n')
                        .trim();
                };

                const doc =
                    document.querySelector('.document.activeDoc')
                    || document.querySelector('#tabBody_0 .document')
                    || document.querySelector('#workspace .document');

                if (!doc) {
                    return {
                        text: '',
                        fragments: 0,
                        mode: 'none'
                    };
                }

                // TechExpert may split one document into several blocks.
                const fragments = [
                    ...doc.querySelectorAll('.text-for-mark')
                ];

                if (fragments.length) {
                    const parts = fragments
                        .map(normalizeBlock)
                        .filter(Boolean);

                    return {
                        text: parts.join('\\n\\n'),
                        fragments: parts.length,
                        mode: 'text-for-mark'
                    };
                }

                const fallback =
                    doc.querySelector('.text')
                    || doc.querySelector('.scrollBlock')
                    || doc;

                return {
                    text: normalizeBlock(fallback),
                    fragments: 1,
                    mode: 'fallback'
                };
            }"""
        )
    except Exception:
        result = None

    if result:
        text = str(result.get("text") or "").strip()
        if len(text) >= 300:
            text = text.replace("\xa0", " ")
            text = re.sub(r"[ \t]+\n", "\n", text)
            text = re.sub(r"\n{4,}", "\n\n\n", text)
            return text.strip()

    # Last-resort fallback only.
    return (await _frame_body_text(frame)).strip()


async def _validate_saved_document(
    frame,
    code: str,
    clean_text: str,
    *,
    source_url: str = "",
    expected_nd: str | None = None,
    candidate_title: str | None = None,
) -> None:
    """
    Validate identity without requiring the literal TR code to be printed
    in the body. Some TechExpert regulation pages omit the short code from
    the visible regulation text.

    Identity is based on:
      1) the exact internal TechExpert nd followed from the adoption decision;
      2) the document DOM;
      3) absence of the "О принятии..." wrapper;
      4) regulation-like structure/text.
    """
    if len(clean_text.strip()) < 300:
        raise RuntimeError("Нормативный текст слишком короткий.")

    try:
        title = (await frame.title()).strip()
    except Exception:
        title = ""

    combined = (title + "\n" + clean_text[:10000]).lower()

    if "о принятии технического регламента" in title.lower():
        raise RuntimeError(
            f"Вместо текста {code} снова открыто только решение "
            "о принятии технического регламента."
        )

    if expected_nd:
        actual_nd = _extract_nd(source_url)
        if actual_nd != expected_nd:
            raise RuntimeError(
                f"Открыт другой документ TechExpert: ожидался nd={expected_nd}, "
                f"получен nd={actual_nd or 'не определён'}."
            )

    if _is_technical_regulation_code(code):
        if not expected_nd:
            raise RuntimeError(
                f"Для {code} не определён внутренний nd документа. "
                "Временный iframe URL не может использоваться как "
                "идентификатор нормативного документа."
            )

        has_regulation_signal = (
            "технический регламент" in combined
            or bool(
                re.search(
                    r"\bстатья\s+1\b|\bобласть\s+применения\b",
                    clean_text,
                    re.I,
                )
            )
        )

        if not has_regulation_signal:
            hint = f" ({candidate_title})" if candidate_title else ""
            raise RuntimeError(
                f"Документ nd={expected_nd} открыт, "
                f"но его структура не похожа на текст {code}{hint}."
            )


async def _download_candidate(page, frame) -> tuple[str | None, bytes | None]:
    selectors = [
        'a[href$=".pdf" i]',
        'a[href*="get-pdf" i]',
        'a[href*="download" i]',
        'a:has-text("Скачать")',
        'a:has-text("PDF")',
        'a[title*="сохран" i]',
    ]

    for selector in selectors:
        item = await _first_visible(frame.locator(selector))
        if not item:
            continue

        try:
            href = await item.get_attribute("href")
            if not href or href.startswith("javascript:"):
                continue

            url = urljoin(frame.url or page.url, href)
            response = await page.context.request.get(
                url,
                timeout=60000,
            )
            if not response.ok:
                continue

            body = await response.body()
            content_type = (
                response.headers.get("content-type") or ""
            ).lower()
            disposition = (
                response.headers.get("content-disposition") or ""
            )

            name = None
            match = re.search(
                r'filename\*?=(?:UTF-8\'\')?"?([^";]+)',
                disposition,
                re.I,
            )
            if match:
                name = match.group(1).strip()

            if not name:
                parsed = Path(urlparse(url).path).name
                name = parsed or "source_file"

            if "pdf" in content_type and not name.lower().endswith(".pdf"):
                name += ".pdf"

            if len(body) > 5000:
                return name, body
        except Exception:
            continue

    return None, None


async def _frame_diagnostics(page) -> dict:
    info = {
        "page_url": page.url,
        "title": "",
        "auth": await _auth_state(page),
        "frames": [],
    }

    try:
        info["title"] = await page.title()
    except Exception:
        pass

    for idx, frame in enumerate(list(page.frames)):
        frame_info = {
            "index": idx,
            "url": getattr(frame, "url", ""),
            "inputs": [],
        }

        try:
            inputs = frame.locator("input, textarea, [contenteditable='true']")
            count = await inputs.count()
            for i in range(min(count, 100)):
                item = inputs.nth(i)
                try:
                    frame_info["inputs"].append(
                        {
                            "tag": await item.evaluate(
                                "(e) => e.tagName.toLowerCase()"
                            ),
                            "type": await item.get_attribute("type"),
                            "name": await item.get_attribute("name"),
                            "id": await item.get_attribute("id"),
                            "class": await item.get_attribute("class"),
                            "placeholder": await item.get_attribute("placeholder"),
                            "title": await item.get_attribute("title"),
                            "aria_label": await item.get_attribute("aria-label"),
                            "visible": await item.is_visible(),
                        }
                    )
                except Exception:
                    pass
        except Exception:
            pass

        info["frames"].append(frame_info)

    return info


async def _save_debug(page, name: str) -> None:
    debug_dir = SOURCE_ROOT / "techexpert" / "_debug"
    debug_dir.mkdir(parents=True, exist_ok=True)
    safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", name)

    try:
        await page.screenshot(
            path=str(debug_dir / f"{safe}.png"),
            full_page=True,
        )
    except Exception:
        pass

    try:
        (debug_dir / f"{safe}.html").write_text(
            await page.content(),
            encoding="utf-8",
        )
    except Exception:
        pass

    # TechExpert 6 uses nested frames. Save every frame separately.
    for idx, frame in enumerate(list(page.frames)):
        try:
            html = await frame.content()
            (debug_dir / f"{safe}_frame_{idx}.html").write_text(
                html,
                encoding="utf-8",
            )
        except Exception:
            pass

    try:
        diagnostics = await _frame_diagnostics(page)
        (debug_dir / f"{safe}_diagnostic.json").write_text(
            json.dumps(
                diagnostics,
                ensure_ascii=False,
                indent=2,
            ),
            encoding="utf-8",
        )
    except Exception:
        pass


async def sync_all(
    *,
    headless: bool = True,
    limit: int | None = None,
    document_slug: str | None = None,
) -> dict:
    async_playwright = _require_playwright()
    credentials = load_secret("techexpert")
    init_source_schema()

    base_url = credentials.get("base_url") or BASE_URL_DEFAULT
    username = credentials["username"]
    password = credentials["password"]

    results = []
    global_error = None

    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=headless,
            args=[
                "--disable-dev-shm-usage",
                "--no-sandbox",
            ],
        )

        context = await browser.new_context(
            locale="ru-RU",
            viewport={"width": 1440, "height": 1100},
        )
        page = await context.new_page()

        try:
            state = await _login(
                page,
                username,
                password,
                base_url,
            )
            set_login_ok()
        except Exception as exc:
            await _save_debug(page, "login_error")
            await browser.close()
            set_source_sync_result(error=str(exc))
            raise

        with connect() as con:
            rows = con.execute(
                """
                SELECT document_slug, document_code,
                       search_query, discovered_url
                FROM tracked_documents
                WHERE source_key='techexpert' AND enabled=1
                ORDER BY document_code
                """
            ).fetchall()

        if document_slug:
            rows = [
                row
                for row in rows
                if row["document_slug"] == document_slug
            ]

        if limit:
            rows = rows[:limit]

        if document_slug and not rows:
            await browser.close()
            raise RuntimeError(
                f"Неизвестный отслеживаемый документ: {document_slug}"
            )

        for row in rows:
            slug = row["document_slug"]
            code = row["document_code"]

            try:
                # In TechExpert 6 intranet the reliable path is UI search.
                # A stored frame URL may be session-specific, so we search
                # again if direct reuse fails.
                document_frame = None
                source_url = ""
                title_hint = ""

                stored_url = row["discovered_url"]

                # TechExpert exposes many transient iframe URLs such as:
                #   /docs/?frame=left#frameLeft
                # They are NOT stable document identifiers and must never
                # be reused as canonical document URLs.
                #
                # Reuse only a URL carrying the internal TechExpert nd.
                stored_nd = _extract_nd(stored_url)
                reusable_stored_url = bool(
                    stored_url
                    and stored_url.startswith(("http://", "https://"))
                    and stored_nd
                )

                if reusable_stored_url:
                    try:
                        await page.goto(
                            stored_url,
                            wait_until="domcontentloaded",
                            timeout=60000,
                        )
                        await page.wait_for_timeout(1500)

                        document_frame = await _pick_document_frame(
                            page,
                            code,
                        )

                        if document_frame is not None:
                            # Keep the canonical URL with nd rather than the
                            # transient frame URL returned by frame.url.
                            source_url = stored_url
                    except Exception:
                        document_frame = None

                if document_frame is None:
                    (
                        document_frame,
                        source_url,
                        title_hint,
                    ) = await _search_document(
                        page,
                        base_url,
                        code,
                        row["search_query"],
                    )

                    # source_url returned by the UI can still be a transient
                    # frame URL. _follow_actual_regulation() will replace it
                    # with a canonical /docs/?nd=... URL for a technical
                    # regulation after following the internal document link.

                # Search can return the EEC/Customs Union decision
                # "О принятии..." rather than the regulation itself.
                (
                    document_frame,
                    source_url,
                    followed_regulation,
                    parent_url,
                    expected_nd,
                    candidate_title,
                ) = await _follow_actual_regulation(
                    page,
                    document_frame,
                    source_url or page.url,
                    code,
                    base_url,
                )

                clean_text = await _extract_clean_document_text(
                    document_frame
                )
                html = await document_frame.content()

                try:
                    frame_title = await document_frame.title()
                except Exception:
                    frame_title = ""

                title = frame_title or title_hint or code

                if len(clean_text.strip()) < 200:
                    raise RuntimeError(
                        "Документ открыт, но извлечённый нормативный текст "
                        "слишком короткий."
                    )

                await _validate_saved_document(
                    document_frame,
                    code,
                    clean_text,
                    source_url=source_url or page.url,
                    expected_nd=expected_nd,
                    candidate_title=candidate_title,
                )

                download_name, download_bytes = await _download_candidate(
                    page,
                    document_frame,
                )

                saved = save_snapshot(
                    document_slug=slug,
                    document_code=code,
                    source_url=source_url or page.url,
                    title=title,
                    text=clean_text,
                    html=html,
                    downloaded_name=download_name,
                    downloaded_bytes=download_bytes,
                )

                results.append(
                    {
                        "document": code,
                        "status": (
                            "changed"
                            if saved["changed"]
                            else "unchanged"
                        ),
                        "url": source_url or page.url,
                        "followed_regulation": followed_regulation,
                        "parent_url": parent_url,
                    }
                )

            except Exception as exc:
                set_source_error(slug, str(exc))
                await _save_debug(
                    page,
                    f"{slug}_error",
                )
                results.append(
                    {
                        "document": code,
                        "status": "error",
                        "error": str(exc),
                    }
                )

        await browser.close()

    errors = [
        item
        for item in results
        if item["status"] == "error"
    ]

    if errors:
        global_error = (
            f"Ошибки по {len(errors)} документам "
            f"из {len(results)}"
        )

    set_source_sync_result(error=global_error)

    return {
        "ok": not errors,
        "results": results,
    }


async def test_login(*, headless: bool = True) -> None:
    async_playwright = _require_playwright()
    credentials = load_secret("techexpert")

    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=headless,
            args=[
                "--disable-dev-shm-usage",
                "--no-sandbox",
            ],
        )
        context = await browser.new_context(
            locale="ru-RU",
            viewport={"width": 1440, "height": 1000},
        )
        page = await context.new_page()

        try:
            state = await _login(
                page,
                credentials["username"],
                credentials["password"],
                credentials.get("base_url") or BASE_URL_DEFAULT,
            )

            set_login_ok()

            product = state.get("product") or "Техэксперт"
            print("Авторизация: OK")
            print("Система:", product)
            print("Текущая страница:", page.url)

        except Exception:
            await _save_debug(
                page,
                "login_test_error",
            )
            raise
        finally:
            await browser.close()


def print_status() -> None:
    print(
        json.dumps(
            source_status(),
            ensure_ascii=False,
            indent=2,
        )
    )


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Синхронизация нормативных источников QUALITY"
    )
    sub = parser.add_subparsers(
        dest="command",
        required=True,
    )

    sub.add_parser("status")

    login_parser = sub.add_parser("test-login")
    login_parser.add_argument("--headed", action="store_true")

    sync_parser = sub.add_parser("sync")
    sync_parser.add_argument("--headed", action="store_true")
    sync_parser.add_argument("--limit", type=int)
    sync_parser.add_argument("--document")

    args = parser.parse_args()

    if args.command == "status":
        print_status()
        return

    if args.command == "test-login":
        asyncio.run(
            test_login(
                headless=not args.headed,
            )
        )
        return

    if args.command == "sync":
        result = asyncio.run(
            sync_all(
                headless=not args.headed,
                limit=args.limit,
                document_slug=args.document,
            )
        )
        print(
            json.dumps(
                result,
                ensure_ascii=False,
                indent=2,
            )
        )
        if not result["ok"]:
            raise SystemExit(2)


if __name__ == "__main__":
    main()