import json
import os
from collections.abc import Generator, Iterable
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast

from tree_sitter import Node as TreeSitterNode
from tree_sitter import Parser, Query

from sphinx_codelinks.analyse import multiline_parser, utils
from sphinx_codelinks.analyse.entity import (
    CodeScope,
    code_scope,
    is_doc_comment,
    scope_language,
)
from sphinx_codelinks.analyse.models import (
    MarkedContentType,
    MultilineNeed,
    NeedIdRefs,
    OneLineNeed,
    SourceComment,
    SourceFile,
    SourceMap,
    WarningSubTypeEnum,
)
from sphinx_codelinks.analyse.oneline_parser import (
    OnelineParserInvalidWarning,
    docstring_tag,
    doxygen_tag,
    jsdoc_tag,
    oneline_parser,
    opens_powershell_literal,
)
from sphinx_codelinks.analyse.references import _relative_posix
from sphinx_codelinks.config import (
    UNIX_NEWLINE,
    CommentCategory,
    OneLineCommentStyle,
    SourceAnalyseConfig,
)
from sphinx_codelinks.logger import get_logger
from sphinx_codelinks.source_discover.config import CommentType

if TYPE_CHECKING:
    from sphinx_codelinks.analyse.preproc.libclang_parser import LibclangComment

logger = get_logger(__name__)

# The last key of the marked content's sort. Two entries can start at one position
# only with overlapping markers (a need-id marker that ends with the one-line start
# sequence), which the configuration check does not refuse. Such a tie is listed
# references, then one-line needs, then multi-line needs: an explicit rank, because
# the type names sort the other way round. Two references never tie: the marker
# matches of a line do not overlap (``find_marker_matches``).
_KIND_RANK: dict[MarkedContentType, int] = {
    MarkedContentType.need_id_refs: 0,
    MarkedContentType.need: 1,
    MarkedContentType.multiline_need: 2,
}


def _is_alphanumeric(char: str) -> bool:
    """Whether ``char``'s Unicode general category is a letter (L*) or a number (N*).

    The one notion of alphanumeric of the reference markers' rules, shared with another
    implementation of them. ``str.isalnum`` is exactly that test (no other character
    passes it): characters Unicode calls Alphabetic without being letters -- a circled
    letter (So), a combining mark (Mn, Mc) -- are not alphanumeric here. The
    classification follows each tool's own Unicode tables (here the running Python's,
    ``unicodedata.unidata_version``), so a character newer than the older of two tables
    can be classed differently.
    """
    return char.isalnum()


def _is_word_char(char: str) -> bool:
    """A word character, for a reference marker's boundary: alphanumeric (see
    :func:`_is_alphanumeric`) or ``_``."""
    return _is_alphanumeric(char) or char == "_"


def _find_marker(line: str, marker: str, start: int) -> int:
    """The index of the first match of ``marker`` in ``line`` at or after ``start``.

    A marker whose last character is a word character matches only where the
    character after it is not one (or the line ends): ``\\verifies`` does not match
    ``\\verifiesall``. No condition applies before the match. ``-1`` when none.
    """
    word_final = _is_word_char(marker[-1])
    idx = line.find(marker, start)
    while idx != -1:
        after = idx + len(marker)
        if not (word_final and after < len(line) and _is_word_char(line[after])):
            return idx
        idx = line.find(marker, idx + 1)
    return -1


def find_marker_matches(
    line: str, markers: Iterable[str], *, first_only: bool = False
) -> list[tuple[str, int, int]]:
    """The reference-marker matches of one line, left to right: ``(marker, start, end)``.

    The line is scanned from left to right. At each index, among the markers that
    match there (see :func:`_find_marker`), the LONGEST wins, and the scan resumes at
    the end of that match, so matches never overlap. The same marker can match more
    than once. An empty marker never matches.

    :param line: One line of a comment.
    :param markers: The configured reference markers.
    :param first_only: Stop after the first match.
    """
    candidates = [marker for marker in markers if marker]
    matches: list[tuple[str, int, int]] = []
    pos = 0
    while candidates:
        best: tuple[int, str] | None = None
        for marker in candidates:
            idx = _find_marker(line, marker, pos)
            if idx == -1:
                continue
            if (
                best is None
                or idx < best[0]
                or (idx == best[0] and len(marker) > len(best[1]))
            ):
                best = (idx, marker)
        if best is None:
            break
        idx, marker = best
        matches.append((marker, idx, idx + len(marker)))
        if first_only:
            break
        pos = idx + len(marker)
    return matches


def _char_column(src: bytes, byte_offset: int) -> int:
    """The column, in characters, of ``byte_offset`` in the UTF-8 ``src``."""
    line_start = src.rfind(b"\n", 0, byte_offset) + 1
    return len(src[line_start:byte_offset].decode("utf-8", errors="replace"))


def _count(n: int, noun: str) -> str:
    """Format ``n noun`` with a naive (append-s) plural for progress summaries."""
    return f"{n} {noun}" if n == 1 else f"{n} {noun}s"


def _row_count(src_comment: SourceComment) -> int:
    """The rows a comment covers: a Rust ``///`` node's text ends with its newline, which
    starts no row of its own."""
    return src_comment.text.rstrip(b"\n").count(b"\n") + 1


#: The message of a ``marker_above_conditional`` warning.
ABOVE_CONDITIONAL_MSG = (
    "directly above a preprocessor conditional, so attached to no entity: "
    "put the marker inside the arm, above the definition"
)

#: How a tag warning names the tag it found.
_TAG_NAMES = {
    WarningSubTypeEnum.docstring_tag: "docstring tag",
    WarningSubTypeEnum.jsdoc_tag: "JSDoc tag",
    WarningSubTypeEnum.doxygen_tag: "Doxygen tag",
}


def _is_jsdoc(text: bytes) -> bool:
    """Whether the comment ``text`` is a JSDoc comment: a block comment opened by
    exactly ``/**``, as JSDoc reads one (``/***`` and the empty ``/**/`` are not)."""
    return (
        text.startswith(b"/**")
        and not text.startswith(b"/***")
        and not text.startswith(b"/**/")
    )


def _docstring_contents(
    node: TreeSitterNode, raw: bytes, column: int
) -> list[tuple[str, int, int]]:
    """The content of each string of a docstring statement, between its quotes.

    Each with its 0-based row and its column, in characters: the quotes' column plus
    the prefix letters' and the delimiter's length. ``raw`` is the statement's text as
    the file writes it, ``column`` its column.
    """
    contents = []
    for string in node.named_children:
        if string.type != "string":
            continue
        kinds = {child.type: child for child in string.children}
        start, end = kinds.get("string_start"), kinds.get("string_end")
        if start is None or end is None:
            continue
        begin = start.end_byte - node.start_byte
        before = raw[:begin]
        row_start = before.rfind(b"\n") + 1
        content_column = len(before[row_start:].decode("utf-8", errors="replace"))
        if row_start == 0:
            content_column += column
        contents.append(
            (
                raw[begin : end.start_byte - node.start_byte].decode(
                    "utf-8", errors="replace"
                ),
                start.end_point.row,
                content_column,
            )
        )
    return contents


def _scanned_texts(
    src_comment: SourceComment, claimed_rows: set[int] | None
) -> list[tuple[str, int, int]]:
    """What the one-line and reference extractors scan of a comment.

    Each text comes with the 0-based row and the column, in characters, at which it
    starts. A docstring is scanned as its content, without the quotes; a block
    comment without its closing ``*/``, a PowerShell ``<# … #>`` comment without its
    ``#>``, and a legacy ``<!-- … -->`` comment without its ``-->``, which are not
    marker text. The rows a multi-line need claims are blanked.
    """
    node = src_comment.node
    if getattr(node, "type", None) == CommentCategory.docstring:
        texts = _docstring_contents(node, src_comment.text, src_comment.column)
    else:
        text = src_comment.text.decode("utf-8")
        if (text.startswith("/*") and text.endswith("*/")) or (
            text.startswith("<#") and text.endswith("#>")
        ):
            text = text[:-2]
        elif text.startswith("<!--") and text.endswith("-->"):
            text = text[:-3]
        texts = [(text, node.start_point.row, src_comment.column)]
    if claimed_rows:
        # a block's lines are its own: no one-line need and no reference in them
        texts = [
            (multiline_parser.blank_rows(text, row, claimed_rows), row, column)
            for text, row, column in texts
        ]
    return texts


@dataclass
class AnalyseWarning:
    file_path: str
    lineno: int
    msg: str
    type: str
    sub_type: str


@dataclass
class BuiltinHeaderWarnings:
    """What the libclang engine has already warned about clang's builtin headers.

    One object shared by several analyses makes each warning once across all of them:
    the CLI shares one across its projects, a Sphinx build one across its directives
    and its scan of config-only projects.
    """

    files: set[Path] = field(default_factory=set)
    """The parsed files already warned about a builtin header they could not find."""
    unavailable: bool = False
    """Whether the shipped builtin headers were warned to be unavailable."""


@dataclass(frozen=True)
class _Waiting:
    """A compiled source with no compilation-database entry, waiting for its includer."""

    position: int
    """Its index in ``src_files``."""
    path: Path
    database: Path | None
    """The database that applies to it."""
    candidates: tuple[Path, ...]
    """The main files of the database's entries whose ``#include`` lines name it,
    ordered by path."""


class SourceAnalyse:
    def __init__(
        self,
        analyse_config: SourceAnalyseConfig,
        *,
        name: str = "",
        builtin_header_warnings: BuiltinHeaderWarnings | None = None,
    ) -> None:
        """
        :param builtin_header_warnings: What was already warned about clang's builtin
            headers, shared with other analyses so that each warning is made once
            across them all; by default this analysis' own.
        """
        self.name = name
        self.analyse_config = analyse_config
        self.src_files: list[SourceFile] = []
        self.src_comments: list[SourceComment] = []
        self.need_id_refs: list[NeedIdRefs] = []
        self.oneline_needs: list[OneLineNeed] = []
        self.multiline_needs: list[MultilineNeed] = []
        self.all_marked_content: list[NeedIdRefs | OneLineNeed | MultilineNeed] = []
        # Use explicitly configured git_root if provided, otherwise auto-detect
        if self.analyse_config.git_root is not None:
            self.git_root: Path | None = self.analyse_config.git_root.resolve()
        else:
            self.git_root = utils.locate_git_root(self.analyse_config.src_dir)
        self.git_remote_url: str | None = (
            utils.get_remote_url(self.git_root) if self.git_root else None
        )
        self.git_commit_rev: str | None = (
            utils.get_current_rev(self.git_root) if self.git_root else None
        )
        self.project_path: Path = self.git_root or self.analyse_config.src_dir
        self.warnings: list[AnalyseWarning] = []
        """The analysis' warnings, one-line and multi-line alike, in extraction order."""
        # Per-run memo of parsed compile_commands.json, keyed by DB path, so the
        # database is read once per run instead of once per source file.
        self._flags_map_cache: dict[Path, dict[Path, list[str]] | None] = {}
        # Per-run memo of each database's inclusion index (included file -> the
        # entries' main files that include it), built only when a source needs it.
        self._inclusion_index_cache: dict[Path, dict[Path, list[Path]]] = {}
        self.builtin_header_warnings = (
            builtin_header_warnings
            if builtin_header_warnings is not None
            else BuiltinHeaderWarnings()
        )
        self.dependencies: set[Path] = set()
        """Files besides ``src_files`` the extraction read: the main file of every
        entry tried as a source's includer, the files its unit included that are not
        system headers, and the database. An edit to one of them can change what is
        extracted."""

    @property
    def oneline_warnings(self) -> list[AnalyseWarning]:
        """The old name of :attr:`warnings`, kept for one release; read-only."""
        return self.warnings

    def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]:
        """Load source files and extract their content."""
        for src_path in self.analyse_config.src_files:
            if not utils.is_text_file(src_path):
                continue
            with src_path.open("r", encoding="utf-8", newline="") as f:
                # Normalize all line endings to Unix LF
                text = f.read()
            text = text.replace("\r\n", "\n").replace("\r", "\n")
            yield src_path, text.encode("utf-8")

    def create_src_objects(self) -> None:
        comment_type = self.analyse_config.comment_type
        # One (parser, query) pair per distinct grammar actually needed, built
        # lazily so a parser is never rebuilt per file. Every comment type
        # except TypeScript uses a single grammar for the whole run;
        # TypeScript alone varies its grammar per file (utils.ts_grammar_key)
        # because a legacy TypeScript-only cast parses as JSX under the wrong
        # grammar — see the CommentType.ts branch of utils.init_tree_sitter.
        parser_cache: dict[str, tuple[Parser, Query]] = {}

        for src_path, src_string in self.get_src_strings():
            # `comment_type` is normally a CommentType member, but a few call
            # sites carry it as a plain (possibly invalid) str instead — see
            # SourceAnalyseConfig.comment_type — so key on `str(comment_type)`
            # rather than `.value`, which only the enum has.
            cache_key = (
                utils.ts_grammar_key(src_path)
                if comment_type == CommentType.ts
                else str(comment_type)
            )
            if cache_key not in parser_cache:
                parser_cache[cache_key] = utils.init_tree_sitter(comment_type, src_path)
            parser, query = parser_cache[cache_key]

            tree, cpp_names = utils.parse_source(
                src_string, parser, comment_type, src_path
            )
            comments: list[TreeSitterNode] | None = utils.captured_comments(tree, query)
            if not comments:
                continue
            src_comments: list[SourceComment] = [
                SourceComment(node, _char_column(src_string, node.start_byte))
                for node in comments
            ]

            src_file = SourceFile(src_path.absolute())
            src_file.cpp_names = cpp_names
            src_file.add_comments(src_comments)
            if self.analyse_config.get_multiline_needs:
                src_file.lines = src_string.decode("utf-8").split(UNIX_NEWLINE)
            self.src_files.append(src_file)
            self.src_comments.extend(src_comments)

    def _flags_map_for(self, db_path: Path) -> dict[Path, list[str]] | None:
        """Return the parsed compile_commands.json for ``db_path``, memoized.

        The database is read and parsed once per run instead of once per source
        file (O(files x entries) -> O(entries)). A present-but-malformed or
        unreadable database is warned once and cached as ``None`` so callers fall
        back to the configured defines without re-reading or re-warning per file.
        """
        from sphinx_codelinks.analyse.preproc import compile_db

        if db_path in self._flags_map_cache:
            return self._flags_map_cache[db_path]
        try:
            flags = compile_db.load_flags_map(db_path)
        except (OSError, ValueError, TypeError) as exc:
            logger.warning(
                f"codelinks: failed to read {db_path} ({exc}); "
                f"falling back to the configured defines"
            )
            self._flags_map_cache[db_path] = None
            return None
        self._flags_map_cache[db_path] = flags
        return flags

    def _compile_db_for(self, src_path: Path) -> Path | None:
        """The compilation database that applies to ``src_path``, if one is named
        or found (an explicit path that is not a readable file included)."""
        from sphinx_codelinks.analyse.preproc import compile_db

        preproc = self.analyse_config.preprocessor
        if preproc is None:
            return None
        if preproc.compile_commands is not None:
            return preproc.compile_commands
        return compile_db.find_compile_db(src_path, self.project_path)

    def _resolve_preproc_args(self, src_path: Path) -> list[str] | None:
        from sphinx_codelinks.analyse.preproc import compile_db

        # `run()` calls this (via create_src_objects_libclang) only when
        # `preprocessor is not None`, but keep the guard: it narrows the type for
        # the checker and is a cheap defense (an `assert` would trip bandit S101).
        preproc = self.analyse_config.preprocessor
        if preproc is None:
            return []
        db_path = self._compile_db_for(src_path)
        if db_path is not None and db_path.is_file():
            flags = self._flags_map_for(db_path)
            if flags is None:
                # Present-but-malformed/unreadable DB (warned once in the helper):
                # fall back to the configured defines so extraction still runs.
                return compile_db.defines_to_args(
                    preproc.defines, preproc.includes, preproc.std
                )
            args = flags.get(src_path.absolute().resolve())
            if args is not None:
                return args
            # Absent from the DB. compile_commands.json lists only compiled
            # translation units, never headers — so a header here is parsed
            # standalone with the global defines (one run = one variant). A
            # compiled source absent from the DB has no arguments of its own: it is
            # read through an entry that includes it, or skipped (spec §3.3).
            if compile_db.is_translation_unit_source(src_path):
                return None
            return compile_db.defines_to_args(
                preproc.defines, preproc.includes, preproc.std
            )
        # No readable DB. `find_compile_db` only ever returns a real file, so a
        # non-None `db_path` that reaches here is an explicit `compile_commands`
        # path that is not a readable file (typo/missing): warn and fall back
        # rather than skip silently. A genuinely absent DB (db_path is None) just
        # falls back to the manual defines applied globally.
        if db_path is not None:
            logger.warning(
                f"codelinks: compile_commands path {db_path} is not a readable "
                f"file; falling back to the configured defines"
            )
        return compile_db.defines_to_args(
            preproc.defines, preproc.includes, preproc.std
        )

    def _with_builtin_headers(self, args: list[str]) -> list[str]:
        """``args`` with the resource directory that gives a parse clang's builtin
        headers (:func:`.preproc.builtin_headers.resource_dir_args`); warns once (see
        :class:`BuiltinHeaderWarnings`) when the headers that ship with the package
        are wanted but missing."""
        from sphinx_codelinks.analyse.preproc import builtin_headers

        preproc = self.analyse_config.preprocessor
        configured = preproc.resource_dir if preproc is not None else None
        args, unavailable = builtin_headers.resource_dir_args(args, configured)
        if unavailable and not self.builtin_header_warnings.unavailable:
            self.builtin_header_warnings.unavailable = True
            logger.warning(
                f"codelinks: clang's builtin headers are not under "
                f"{builtin_headers.BUNDLED_RESOURCE_DIR / 'include'}, so C/C++ files "
                f"are parsed without them, and an #if on what <stddef.h>, <stdint.h>, "
                f"<limits.h> or <stdatomic.h> define may take the wrong branch; set "
                f"resource_dir in [analyse.preprocessor] to a clang resource directory "
                f"(clang -print-resource-dir)",
                subtype="builtin_header",
            )
        return args

    def _warn_missing_builtin_header(
        self, path: Path, tu: Any, args: list[str]
    ) -> None:
        """Warn, once per parsed file (see :class:`BuiltinHeaderWarnings`), when its
        parse found no builtin header. When its arguments ``args`` leave clang's
        builtin headers out (``-nostdinc``), the warning names that flag, which no
        ``resource_dir`` overrides; otherwise it points at ``resource_dir``."""
        from sphinx_codelinks.analyse.preproc import builtin_headers, compile_db

        warned = self.builtin_header_warnings.files
        if path in warned:
            return
        missing = builtin_headers.missing_builtin_header(
            tu, compile_db.working_directory(args)
        )
        if missing is None:
            return
        warned.add(path)
        name, file, line = missing
        where = f" (included at {file}:{line})" if file is not None else ""
        excluding = builtin_headers.excluding_flag(args)
        remedy = (
            f"the compile command's {excluding} leaves clang's builtin headers out"
            if excluding is not None
            else "set resource_dir in [analyse.preprocessor] to a clang resource "
            "directory that has it (clang -print-resource-dir)"
        )
        logger.warning(
            f"codelinks: {path}: clang's builtin header '{name}' was not "
            f"found{where}, so an #if on what it defines takes its #else branch; "
            f"{remedy}",
            subtype="builtin_header",
        )

    def _includer_candidates(self, db_path: Path, src_path: Path) -> list[Path]:
        """The main files of ``db_path``'s entries whose ``#include`` lines name
        ``src_path``, ordered by path (see :mod:`.preproc.includer`)."""
        from sphinx_codelinks.analyse.preproc import includer

        if db_path not in self._inclusion_index_cache:
            flags = self._flags_map_for(db_path) or {}
            self._inclusion_index_cache[db_path] = includer.inclusion_index(flags)
        return self._inclusion_index_cache[db_path].get(src_path.resolve(), [])

    def _read_through_includers(
        self, waiting: list[_Waiting]
    ) -> dict[int, list["LibclangComment"]]:
        """Read each waiting source through its includer; warn about the rest.

        A source's includer is the first of its candidates, by path, whose parse
        confirms that its main file includes the source (the lexical scan cannot see
        ``#if`` arms). The candidates are parsed in path order across every waiting
        source, each once, and each unit is released before the next is parsed: when
        a candidate is parsed it is the next candidate of every source that lists
        it, so it is never needed again. One libclang cannot load confirms nothing.

        :return: The comments read, by the source's position in ``src_files``.
        """
        from sphinx_codelinks.analyse.preproc import compile_db, libclang_parser, loader

        translation_unit_load_error = (
            loader.load_clang_cindex().TranslationUnitLoadError
        )
        read: dict[int, list[LibclangComment]] = {}
        # each unread source's next candidate, by position
        next_candidate = {w.position: 0 for w in waiting}
        unread = {w.position: w for w in waiting}
        while True:
            heads = {
                (w.database, w.candidates[next_candidate[position]])
                for position, w in unread.items()
                if w.database is not None
                and next_candidate[position] < len(w.candidates)
            }
            if not heads:
                break
            database, main = min(
                heads, key=lambda head: (os.fsencode(head[1]), os.fsencode(head[0]))
            )
            args = self._with_builtin_headers(
                (self._flags_map_for(database) or {})[main]
            )
            self.dependencies.update((database, main))
            try:
                tu = libclang_parser.parse_translation_unit(main, args)
            except translation_unit_load_error:
                logger.debug(
                    f"codelinks: {main} could not be loaded as a translation unit, "
                    f"so it includes no source"
                )
                tu = None
                includes: set[Path] = set()
            else:
                self._warn_missing_builtin_header(main, tu, args)
                working_directory = compile_db.working_directory(args)
                includes = libclang_parser.main_file_inclusions(tu, working_directory)
                self.dependencies.update(
                    libclang_parser.included_files(tu, working_directory)
                )
            for position, w in list(unread.items()):
                if w.database != database or next_candidate[position] >= len(
                    w.candidates
                ):
                    continue
                if w.candidates[next_candidate[position]] != main:
                    continue
                if tu is not None and w.path.resolve() in includes:
                    logger.debug(
                        f"codelinks: reading {w.path} through {main}, which includes it"
                    )
                    read[position] = libclang_parser.extract_included_active_comments(
                        tu, w.path
                    )
                    del unread[position]
                else:
                    next_candidate[position] += 1
            del tu  # released before the next candidate is parsed
        for w in unread.values():
            logger.warning(
                f"codelinks: skipping {w.path} — it has no entry in "
                f"{w.database} and no entry of it includes it",
                subtype="not_in_compile_database",
            )
        return read

    # @Extract traceability objects with the preprocessor-aware libclang engine, IMPL_PREPROC_1, impl, [FE_PREPROC]
    def create_src_objects_libclang(self) -> None:
        from sphinx_codelinks.analyse.preproc import (
            libclang_parser,
            loader,
        )

        # Resolve the exception via the loader (never a direct ``import
        # clang.cindex``) so a missing ``libclang`` extra still surfaces the
        # loader's friendly install hint rather than a bare ImportError.
        translation_unit_load_error = (
            loader.load_clang_cindex().TranslationUnitLoadError
        )
        src_files = self.analyse_config.src_files
        read: dict[int, list[LibclangComment]] = {}
        # the compiled sources the database has no entry for: read through the entry
        # that includes them, with that entry's flags, once every other file is read
        waiting: list[_Waiting] = []
        for position, src_path in enumerate(src_files):
            if not utils.is_text_file(src_path):
                continue
            args = self._resolve_preproc_args(src_path)
            if args is None:
                database = self._compile_db_for(src_path)
                candidates = (
                    ()
                    if database is None
                    else tuple(self._includer_candidates(database, src_path))
                )
                waiting.append(_Waiting(position, src_path, database, candidates))
                continue
            args = self._with_builtin_headers(args)
            try:
                read[position] = libclang_parser.extract_active_comments(
                    src_path,
                    args,
                    on_parse=lambda tu, path=src_path, args=args: (
                        self._warn_missing_builtin_header(path, tu, args)
                    ),
                )
            except translation_unit_load_error:
                # Last-resort guard. Standalone parses pin ``-x`` to match the
                # ``-std`` (see defines_to_args), so the common case — a ``.h``/
                # ``.c`` header handed a C++ ``-std`` — now parses as C++ and its
                # markers extract. This only fires if libclang still cannot load
                # the file as a translation unit at all; skip it rather than
                # aborting the whole run. Surfaced as a ``warning`` so the silent
                # data-loss (a file's markers dropped) is visible — accepting
                # that this fails ``sphinx-build -W``.
                logger.warning(
                    f"codelinks: skipping {src_path} — libclang could not load "
                    f"it as a translation unit"
                )
        read.update(self._read_through_includers(waiting))

        for position in sorted(read):
            src_path = src_files[position]
            comments = read[position]
            if not comments:
                continue
            # ``c`` is a LibclangComment duck-typing the tree-sitter Node
            # interface SourceComment reads (``.text`` / ``.start_point.row``);
            # the Node-only path (find_associated_scope) is guarded by
            # ``is_libclang`` so it never runs on these. The cast is that duck typing,
            # which is why ty, seeing two unrelated classes, has to be told.
            src_comments = [
                SourceComment(
                    cast("TreeSitterNode", c),  # ty: ignore[disjoint-cast]
                    c.column,
                )
                for c in comments
            ]
            src_file = SourceFile(src_path.absolute())
            src_file.add_comments(src_comments)
            if self.analyse_config.get_multiline_needs:
                # multi-line needs read what precedes a comment on its row from
                # the row itself
                text = src_path.read_text(encoding="utf-8", errors="replace")
                text = text.replace("\r\n", "\n").replace("\r", "\n")
                src_file.lines = text.split(UNIX_NEWLINE)
            self.src_files.append(src_file)
            self.src_comments.extend(src_comments)

    def extract_marker(
        self,
        text: str,
    ) -> Generator[tuple[str, list[str], int, int, int], None, None]:
        """Yield each reference-marker match of ``text`` with its ids.

        Yields ``(marker, need_ids, row_offset, start_column, end_column)``, the
        columns spanning the match's own id text. The matches of a line are found by
        :func:`find_marker_matches`; a match's id text runs from its end to the start
        of the next match on the line, or to the end of the line. A match with no id
        yields nothing.
        """
        markers = self.analyse_config.need_id_refs_config.markers
        row_offset = 0
        for line in text.split(UNIX_NEWLINE):
            matches = find_marker_matches(line, markers)
            for index, (marker, _start, end) in enumerate(matches):
                is_last = index == len(matches) - 1
                after_marker = (
                    line[end:] if is_last else line[end : matches[index + 1][1]]
                )
                markered_text = after_marker.strip()
                if is_last and markered_text.endswith("*/"):
                    # The end of a one-line block comment, not an id.
                    markered_text = markered_text[:-2].rstrip()
                need_ids = markered_text.replace(",", " ").split()
                if not need_ids:
                    continue
                start_column = end + len(after_marker) - len(after_marker.lstrip())
                end_column = start_column + len(markered_text)
                yield marker, need_ids, row_offset, start_column, end_column
            row_offset += 1

    # @Extract need ID references from code comments, IMPL_LNK_1, impl, [FE_LNK]
    def extract_anchors(
        self,
        text: str,
        filepath: Path,
        tagged_scope: TreeSitterNode | None,
        src_comment: SourceComment,
        first_row: int | None = None,
        first_column: int | None = None,
        *,
        scope: CodeScope | None = None,
    ) -> list[NeedIdRefs]:
        """Extract need-ids-refs from a comment.

        ``text`` starts at ``first_row`` and ``first_column`` (by default the
        comment's own row and column). ``scope`` is the named ``tagged_scope``.
        """
        if first_row is None:
            first_row = src_comment.node.start_point.row
        if first_column is None:
            first_column = src_comment.column
        anchors: list[NeedIdRefs] = []
        for (
            marker,
            need_ids,
            row_offset,
            start_column,
            end_column,
        ) in self.extract_marker(text):
            lineno = first_row + row_offset + 1
            line_column = first_column if row_offset == 0 else 0
            remote_url = self.git_remote_url
            if self.git_remote_url and self.git_commit_rev:
                remote_url = utils.form_https_url(
                    self.git_remote_url,
                    self.git_commit_rev,
                    self.project_path,
                    filepath,
                    lineno,
                )
            source_map: SourceMap = {
                "start": {
                    "row": lineno - 1,
                    "column": line_column + start_column,
                },
                "end": {
                    "row": lineno - 1,
                    "column": line_column + end_column,
                },
            }
            anchors.append(
                NeedIdRefs(
                    filepath,
                    remote_url,
                    source_map,
                    src_comment,
                    tagged_scope,
                    need_ids,
                    marker,
                    scope=scope,
                )
            )
        return anchors

    def _is_need_id_refs_line(self, line: str) -> bool:
        """Whether a comment line is an ``@need-ids:`` reference rather than a need.

        A line whose text, after comment decoration and whitespace, starts with a
        configured need-id-refs marker is a reference, and never a one-line need.
        Without it the default one-line start sequence ``@`` also
        matched ``// @need-ids: A, B`` and made a need titled ``need-ids: A`` with the id
        ``B``. A line with anything alphanumeric before the marker is not a reference
        line here (``// see @need-ids: X`` -- the one-line parser ignores it anyway, and
        the references on it are still extracted), and a line holding a one-line need
        before the marker (``[[…]] @need-ids: X``) keeps its need: only a line that
        STARTS with the marker is withheld from the one-line parser. The marker is the
        line's first match as :func:`find_marker_matches` finds it, so a word that only
        begins with a marker (``\\verifiesall`` for ``\\verifies``) is none.
        """
        matches = find_marker_matches(
            line, self.analyse_config.need_id_refs_config.markers, first_only=True
        )
        return bool(matches) and not any(
            _is_alphanumeric(char) for char in line[: matches[0][1]]
        )

    def extract_oneline_need(
        self,
        text: str,
        src_comment: SourceComment,
        oneline_comment_style: OneLineCommentStyle,
        first_row: int | None = None,
    ) -> Generator[tuple[dict[str, str | list[str] | int], int]]:
        if first_row is None:
            first_row = src_comment.node.start_point.row
        # every line counts as terminated, the last one too
        lines = [f"{line}{UNIX_NEWLINE}" for line in text.split(UNIX_NEWLINE)]
        row_offset = 0

        # Only a Python docstring can hold docstring tags; a ``#`` comment never does.
        in_docstring = (
            getattr(src_comment.node, "type", None) == CommentCategory.docstring
        )
        # Only a TypeScript or JavaScript JSDoc comment can hold JSDoc tags.
        in_jsdoc = self.analyse_config.comment_type == CommentType.ts and _is_jsdoc(
            src_comment.text
        )
        in_powershell = self.analyse_config.comment_type == CommentType.powershell
        # Only a VHDL ``--!`` comment, Doxygen's VHDL comment, can hold Doxygen commands
        # (a ``--!!`` comment is a plain one)
        in_doxygen = self.analyse_config.comment_type == CommentType.vhdl and (
            is_doc_comment(src_comment.node, CommentType.vhdl, text=src_comment.text)
        )
        for line in lines:
            if self._is_need_id_refs_line(line):
                row_offset += 1
                continue
            start_sequence = oneline_comment_style.start_sequence
            if in_powershell and opens_powershell_literal(line, start_sequence):
                # a hashtable or an array, ``@{ … }`` or ``@( … )``: code, not a marker
                row_offset += 1
                continue
            tag, kind, holders = None, WarningSubTypeEnum.docstring_tag, "docstrings"
            if in_docstring:
                tag = docstring_tag(line, start_sequence)
            elif in_jsdoc and start_sequence == "@":
                # ``@`` is JSDoc's own sigil: under any other start sequence a JSDoc
                # line is read as any other line is
                tag = jsdoc_tag(line, start_sequence)
                kind, holders = WarningSubTypeEnum.jsdoc_tag, "JSDoc comments"
                if tag is not None and not oneline_parser(line, oneline_comment_style):
                    # a tag line the one-line parser would ignore stays silent
                    row_offset += 1
                    continue
            elif in_doxygen and start_sequence == "@":
                # as for JSDoc: ``@`` is Doxygen's sigil, and a silent line stays silent
                tag = doxygen_tag(line, start_sequence)
                kind, holders = WarningSubTypeEnum.doxygen_tag, "Doxygen comments"
                if tag is not None and not oneline_parser(line, oneline_comment_style):
                    row_offset += 1
                    continue
            if tag is not None:
                if src_comment.source_file:
                    self.warnings.append(
                        AnalyseWarning(
                            str(src_comment.source_file.filepath),
                            first_row + row_offset + 1,
                            f"'{start_sequence}{tag}' is a {_TAG_NAMES[kind]}, not a "
                            "one-line need; use a start sequence that "
                            f"{holders} do not contain",
                            MarkedContentType.need,
                            kind.value,
                        )
                    )
                row_offset += 1
                continue
            resolved = oneline_parser(line, oneline_comment_style)
            if not resolved:
                row_offset += 1
                continue
            if isinstance(resolved, OnelineParserInvalidWarning):
                if not src_comment.source_file:
                    row_offset += 1
                    continue
                lineno = first_row + row_offset + 1
                warning = AnalyseWarning(
                    str(src_comment.source_file.filepath),
                    lineno,
                    resolved.msg,
                    MarkedContentType.need,
                    resolved.sub_type.value,
                )
                self.warnings.append(warning)
                row_offset += 1
                continue
            yield resolved, row_offset
            row_offset += 1

    # @Extract one-line traceability needs from comments, IMPL_ONE_1, impl, [FE_DEF, FE_CMT]
    def extract_oneline_needs(
        self,
        text: str,
        filepath: Path,
        tagged_scope: TreeSitterNode | None,
        src_comment: SourceComment,
        oneline_comment_style: OneLineCommentStyle,
        first_row: int | None = None,
        first_column: int | None = None,
        *,
        scope: CodeScope | None = None,
    ) -> list[OneLineNeed]:
        """Extract the one-line needs of a comment.

        ``text`` starts at ``first_row`` and ``first_column`` (by default the
        comment's own row and column). ``scope`` is the named ``tagged_scope``.
        """
        if first_row is None:
            first_row = src_comment.node.start_point.row
        if first_column is None:
            first_column = src_comment.column
        row_offset = 0
        oneline_needs = []
        for resolved, row_offset in self.extract_oneline_need(
            text, src_comment, oneline_comment_style, first_row
        ):
            lineno = first_row + row_offset + 1
            line_column = first_column if row_offset == 0 else 0
            start_column = line_column + cast("int", resolved["start_column"])
            end_column = line_column + cast("int", resolved["end_column"])
            remote_url = self.git_remote_url
            if self.git_remote_url and self.git_commit_rev:
                remote_url = utils.form_https_url(
                    self.git_remote_url,
                    self.git_commit_rev,
                    self.project_path,
                    filepath,
                    lineno,
                )
            source_map: SourceMap = {
                "start": {
                    "row": lineno - 1,
                    "column": start_column,
                },
                "end": {
                    "row": lineno - 1,
                    "column": end_column,
                },
            }
            del resolved["start_column"]
            del resolved["end_column"]
            oneline_needs.append(
                OneLineNeed(
                    filepath,
                    remote_url,
                    source_map,
                    src_comment,
                    tagged_scope,
                    resolved,  # int arguments were deleted  # ty: ignore[invalid-argument-type]
                    scope=scope,
                )
            )
        return oneline_needs

    # @Extract multi-line needs from comment runs, IMPL_MLN_1, impl, [FE_MULTILINE_NEEDS]
    def extract_multiline_needs(self) -> dict[int, set[int]]:
        """Extract the multi-line needs of every file, and the rows their blocks claim.

        Runs before the other extractors: the rows a block claims (and those of a
        refused header, consumed to its close) are hidden from them.

        :return: For each comment holding claimed rows (keyed by ``id()`` of its
            :class:`SourceComment`), those 0-based rows.
        """
        claimed: dict[int, set[int]] = {}
        config = self.analyse_config.multiline_needs_config
        for src_file in self.src_files:
            for run in multiline_parser.form_runs(
                src_file.src_comments, src_file.lines
            ):
                result = multiline_parser.parse_run(
                    run.lines,
                    config,
                    leaderless_block=run.leaderless_block,
                    mixed_leaders=run.mixed_leaders,
                )
                for issue in result.issues:
                    self.warnings.append(
                        AnalyseWarning(
                            str(src_file.filepath),
                            issue.row + 1,
                            issue.msg,
                            MarkedContentType.multiline_need,
                            issue.kind.value,
                        )
                    )
                for block in result.blocks:
                    self.multiline_needs.append(
                        self._multiline_need(block, run, src_file.filepath)
                    )
                if not result.claimed_rows:
                    continue
                for src_comment in run.comments:
                    first = src_comment.node.start_point.row
                    rows = result.claimed_rows.intersection(
                        range(first, first + _row_count(src_comment))
                    )
                    if rows:
                        claimed.setdefault(id(src_comment), set()).update(rows)
        return claimed

    def _multiline_need(
        self,
        block: multiline_parser.ParsedBlock,
        run: multiline_parser.CommentRun,
        filepath: Path,
    ) -> MultilineNeed:
        """The record of one parsed block (see :class:`MultilineNeed`)."""
        src_comment = next(
            (
                comment
                for comment in run.comments
                if comment.node.start_point.row
                <= block.open_row
                < comment.node.start_point.row + _row_count(comment)
            ),
            run.comments[0],
        )
        tagged_scope, scope, unattached = self._associate(src_comment, filepath)
        lineno = block.open_row + 1
        remote_url = self.git_remote_url
        if self.git_remote_url and self.git_commit_rev:
            remote_url = utils.form_https_url(
                self.git_remote_url,
                self.git_commit_rev,
                self.project_path,
                filepath,
                lineno,
            )
        source_map: SourceMap = {
            "start": {"row": block.open_row, "column": block.open_col},
            "end": {"row": block.close_row, "column": block.end_col},
        }
        content_start = (
            {"line": block.content_start[0] + 1, "col": block.content_start[1]}
            if block.content_start is not None
            else None
        )
        source: dict[str, Any] = {
            "project": self.name,
            "path": _relative_posix(filepath, self.project_path),
            "root": "git" if self.git_root is not None else "src_dir",
            "commit": self.git_commit_rev,
            "start": {"line": lineno, "col": block.open_col},
            "end": {"line": block.close_row + 1, "col": block.end_col},
            "content_start": content_start,
            "option_lines": dict(block.option_rows),
            "scope": scope,
        }
        need = MultilineNeed(
            filepath,
            remote_url,
            source_map,
            src_comment,
            tagged_scope,
            block.need,
            block.markup,
            source,
            scope=scope,
        )
        self._warn_unattached(unattached, [need])
        return need

    def _associate(
        self, src_comment: SourceComment, filepath: Path
    ) -> tuple[TreeSitterNode | None, CodeScope | None, utils.Unattached | None]:
        """The entity a comment of ``filepath`` is attached to: its node and its
        ``scope`` object, and why it is attached to none where the analysis warns
        about that (:class:`~sphinx_codelinks.analyse.utils.Unattached`).

        A libclang comment has no tree-sitter node, so it is attached to nothing.
        """
        if getattr(src_comment.node, "is_libclang", False):
            return None, None, None
        comment_type = self.analyse_config.comment_type
        association = utils.associate(
            src_comment.node, comment_type, text=src_comment.text
        )
        if association is None:
            return None, None, None
        if isinstance(association, utils.Unattached):
            return None, None, association
        source_file = src_comment.source_file
        written = (
            source_file.node_text(association.node)
            if source_file is not None
            else association.node.text
        )
        if not written:
            return None, None, None
        return (
            association.node,
            code_scope(
                association,
                comment_type,
                scope_language(comment_type, filepath),
                source_file.cpp_names if source_file is not None else None,
            ),
            None,
        )

    def _warn_unattached(
        self,
        unattached: utils.Unattached | None,
        records: Iterable[NeedIdRefs | OneLineNeed | MultilineNeed],
    ) -> None:
        """Warn about each marker in ``records`` that rule 7 attached to no entity, at
        its line (a multi-line need's open line)."""
        if unattached is not utils.Unattached.above_conditional:
            return
        for record in records:
            self.warnings.append(
                AnalyseWarning(
                    str(record.filepath),
                    record.source_map["start"]["row"] + 1,
                    ABOVE_CONDITIONAL_MSG,
                    record.type,
                    WarningSubTypeEnum.marker_above_conditional.value,
                )
            )

    def extract_marked_content(self) -> None:
        claimed: dict[int, set[int]] = {}
        if self.analyse_config.get_multiline_needs:
            claimed = self.extract_multiline_needs()
        for src_comment in self.src_comments:
            if not src_comment.text:
                continue
            filepath = (
                src_comment.source_file.filepath if src_comment.source_file else None
            )
            if not filepath:
                continue
            tagged_scope, scope, unattached = self._associate(src_comment, filepath)
            for text, first_row, first_column in _scanned_texts(
                src_comment, claimed.get(id(src_comment))
            ):
                if self.analyse_config.get_need_id_refs:
                    anchors = self.extract_anchors(
                        text,
                        filepath,
                        tagged_scope,
                        src_comment,
                        first_row,
                        first_column,
                        scope=scope,
                    )
                    self._warn_unattached(unattached, anchors)
                    self.need_id_refs.extend(anchors)
                if self.analyse_config.get_oneline_needs:
                    oneline_needs = self.extract_oneline_needs(
                        text,
                        filepath,
                        tagged_scope,
                        src_comment,
                        self.analyse_config.oneline_comment_style,
                        first_row,
                        first_column,
                        scope=scope,
                    )
                    self._warn_unattached(unattached, oneline_needs)
                    self.oneline_needs.extend(oneline_needs)

    def merge_marked_content(self) -> None:
        """List the marked content in source order: by file, row and column (#2150).

        Tree-sitter hands a file's comments over in an order that differs between
        runs, so the column is what keeps two entries of one row in a stable order.
        ``oneline_needs`` is sorted too, by row, then file, then column: ``src-trace``
        creates and renders the needs in its order, so of two markers with one id in
        one file the leftmost is the one created, and on one row the needs of two files
        keep the file order they had before the column was a key -- for a caller that
        passes the files in discovery order, as every production caller does.
        """
        # The file component is the key discovery sorts the files by
        # (``SourceDiscover``), not the ``Path``: a ``Path`` compares by parts, so
        # ``a/b.cpp`` would sort before ``a-c.cpp``, which discovery lists first.
        self.oneline_needs.sort(
            key=lambda x: (
                x.source_map["start"]["row"],
                os.path.normcase(os.path.normpath(x.filepath)),
                x.source_map["start"]["column"],
            )
        )
        self.all_marked_content.extend(self.need_id_refs)
        self.all_marked_content.extend(self.oneline_needs)
        self.all_marked_content.extend(self.multiline_needs)
        self.all_marked_content.sort(
            key=lambda x: (
                x.filepath,
                x.source_map["start"]["row"],
                x.source_map["start"]["column"],
                _KIND_RANK[x.type],
            )
        )

    def dump_marked_content(self, outdir: Path) -> None:
        output_path = outdir / "marked_content.json"
        output_path.parent.mkdir(parents=True, exist_ok=True)
        to_dump = [
            marked_content.to_dict() for marked_content in self.all_marked_content
        ]
        with output_path.open("w") as f:
            json.dump(to_dump, f)

    def run(self, *, log_summary: bool = True) -> None:
        """Parse the source files and extract their markers.

        :param log_summary: Whether to print the per-project summary line; a caller
            that prints its own (the configuration pass of the Sphinx extension) passes
            ``False``.
        """
        if (
            self.analyse_config.preprocessor is not None
            and self.analyse_config.comment_type == CommentType.cpp
        ):
            self.create_src_objects_libclang()
        else:
            self.create_src_objects()
        self.extract_marked_content()
        self.merge_marked_content()
        if log_summary:
            self.log_summary()

    def log_summary(self, extra: str = "") -> None:
        """Emit a per-project marker (default-visible) plus a -v breakdown.

        :param extra: Appended to the default-visible line (the ``src-trace``
            directive's count of the one-line needs it did not create).
        """
        label = f"codelinks [{self.name}]" if self.name else "codelinks"
        logger.info(
            f"{label}: {_count(len(self.src_files), 'file')}, "
            f"{_count(len(self.all_marked_content), 'marker')}{extra}"
        )
        logger.debug(
            f"{label}: {_count(len(self.src_comments), 'comment')}, "
            f"{_count(len(self.oneline_needs), 'oneline need')}, "
            f"{_count(len(self.need_id_refs), 'id-ref')}, "
            f"{_count(len(self.multiline_needs), 'multi-line need')}"
        )