1"""Parse a C/C++ TU via libclang and yield ACTIVE comment tokens only."""
2
3from __future__ import annotations
4
5from collections.abc import Callable
6from dataclasses import dataclass
7from pathlib import Path
8from typing import Any
9
10from sphinx_codelinks.analyse.preproc import loader
11from sphinx_codelinks.analyse.preproc.loader import SkippedRange
12
13
14@dataclass
15class _Point:
16 row: int
17
18
19class LibclangComment:
20 """Minimal stand-in for a tree-sitter comment node.
21
22 Exposes exactly the two attributes the existing extract_* chain reads:
23 ``.text`` (bytes) and ``.start_point.row`` (0-indexed). ``is_libclang``
24 lets extract_marked_content skip tree-sitter scope association.
25 """
26
27 is_libclang = True
28
29 def __init__(self, text: bytes, row: int, column: int = 0) -> None:
30 self.text: bytes = text
31 self.start_point = _Point(row)
32 # The column, in characters, at which the comment starts on its line.
33 self.column: int = column
34
35
36def _main_file_skipped(skipped: list[SkippedRange]) -> list[tuple[int, int]]:
37 """The line spans the main file's own pass through the preprocessor skipped.
38
39 The translation unit's skipped ranges are every pass's, and a file its own
40 include chain enters again (a guarded header whose includes include it back,
41 a file that includes itself) passes through a second time, which skips the
42 guarded body the first pass compiled: the union of both covers the whole
43 file. So only the main file's own pass decides, selected by where a range
44 starts (``clang_Location_isFromMainFile``), never by its path, which a later
45 inclusion of the main file shares. The ranges of every other file go too:
46 only the main file's comments are read.
47
48 A file read through a translation unit that includes it is the other case:
49 there its first inclusion's pass decides (:func:`extract_included_active_comments`).
50 """
51 return [(sr.start_line, sr.end_line) for sr in skipped if sr.from_main_file]
52
53
54def _is_in_skipped(line: int, skipped: list[tuple[int, int]]) -> bool:
55 return any(start <= line <= end for start, end in skipped)
56
57
58def parse_translation_unit(file_path: Path, args: list[str]) -> Any:
59 """Parse ``file_path`` as a translation unit with ``args``.
60
61 :raises clang.cindex.TranslationUnitLoadError: libclang could not load it.
62 """
63 cx = loader.load_clang_cindex()
64 index = cx.Index.create()
65 return index.parse(str(file_path), args=args, options=loader.PARSE_OPTIONS)
66
67
[docs] 68# @Drop comments inside inactive preprocessor branches, IMPL_PREPROC_2, impl, [FE_PREPROC]
69def extract_active_comments(
70 file_path: Path,
71 args: list[str],
72 on_parse: Callable[[Any], None] | None = None,
73) -> list[LibclangComment]:
74 """Return one LibclangComment per ACTIVE comment token in ``file_path``.
75
76 Comments inside the ranges the file's own pass through the preprocessor
77 skipped (its inactive branches) are dropped; what a later pass through the
78 same file skips is not (see :func:`_main_file_skipped`).
79
80 :param on_parse: Called with the translation unit once it is parsed (to read
81 its diagnostics).
82 """
83 tu = parse_translation_unit(file_path, args)
84 if on_parse is not None:
85 on_parse(tu)
86 skipped = _main_file_skipped(loader.get_all_skipped_ranges(tu))
87 return _active_comments(tu, file_path, skipped)
88
89
90def main_file_inclusions(tu: Any, working_directory: Path) -> set[Path]:
91 """The files the translation unit's main file itself ``#include``s, canonical.
92
93 libclang names a file found through a relative include directory relative to
94 the unit's working directory, so a relative name is taken against
95 ``working_directory`` (the ``-working-directory`` it was parsed with).
96
97 Read from the inclusion directives of the parse, so an ``#include`` in a branch
98 the preprocessor skipped (``#if 0``) or in a comment is not one; an include of a
99 file the parse did not find is not one either. Directives of included files are
100 not the main file's (``clang_Location_isFromMainFile``).
101 """
102 cx = loader.load_clang_cindex()
103 loader._bind()
104 is_from_main = cx.conf.lib.clang_Location_isFromMainFile
105 out: set[Path] = set()
106 for cursor in tu.cursor.get_children():
107 if cursor.kind != cx.CursorKind.INCLUSION_DIRECTIVE:
108 continue
109 if not is_from_main(cursor.location):
110 continue
111 included = loader.included_file(cursor)
112 if included is None:
113 continue # an include libclang could not find names no file
114 out.add((working_directory / included).resolve())
115 return out
116
117
118def included_files(tu: Any, working_directory: Path) -> set[Path]:
119 """Every file the unit included, at any depth, that is not a system header,
120 canonical: what the unit's preprocessing read besides its main file.
121
122 A relative name (a file found through a relative include directory) is taken
123 against ``working_directory``, as in :func:`main_file_inclusions`.
124 """
125 cx = loader.load_clang_cindex()
126 out: set[Path] = set()
127 for inclusion in tu.get_includes():
128 file = inclusion.include
129 if file is None:
130 continue
131 if cx.SourceLocation.from_offset(tu, file, 0).is_in_system_header:
132 continue
133 out.add((working_directory / file.name).resolve())
134 return out
135
136
[docs]137# @Read a source through a compile-database entry that includes it, IMPL_PREPROC_7, impl, [FE_PREPROC]
138def extract_included_active_comments(tu: Any, file_path: Path) -> list[LibclangComment]:
139 """Return the ACTIVE comment tokens of ``file_path``, a file ``tu`` includes.
140
141 Active means outside the ranges the preprocessor skipped in the file's FIRST
142 inclusion into ``tu``, the configuration its includer compiled it in first;
143 what a later inclusion skips is not (see
144 :func:`~sphinx_codelinks.analyse.preproc.loader.get_first_inclusion_skipped_ranges`).
145 Only the file's own comments are read, never its includer's.
146
147 :param tu: A translation unit whose main file includes ``file_path``
148 (:func:`main_file_inclusions`).
149 :param file_path: The included file.
150 """
151 file = tu.get_file(str(file_path))
152 skipped = [
153 (sr.start_line, sr.end_line)
154 for sr in loader.get_first_inclusion_skipped_ranges(tu, file)
155 ]
156 return _active_comments(tu, file_path, skipped)
157
158
159def _active_comments(
160 tu: Any, file_path: Path, skipped: list[tuple[int, int]]
161) -> list[LibclangComment]:
162 """One LibclangComment per comment token of ``file_path`` outside ``skipped``."""
163 cx = loader.load_clang_cindex()
164 # Read the raw source bytes once. We derive the token extent from them
165 # (lossily, so a non-UTF-8 byte can't raise and abort the run) AND slice each
166 # comment's text out of them by byte offset below — never via ``tok.spelling``,
167 # which decodes the comment as strict UTF-8 and raises UnicodeDecodeError on a
168 # non-UTF-8 byte inside a comment.
169 raw = file_path.read_bytes()
170 line_count = len(raw.decode("utf-8", errors="replace").splitlines())
171 # a location in a file the unit includes is in its first inclusion
172 # (``clang_getLocation``), the one whose skipped ranges ``skipped`` holds
173 file = tu.get_file(str(file_path))
174 extent = cx.SourceRange.from_locations(
175 cx.SourceLocation.from_position(tu, file, 1, 1),
176 cx.SourceLocation.from_position(tu, file, line_count + 1, 1),
177 )
178
179 out: list[LibclangComment] = []
180 for tok in tu.get_tokens(extent=extent):
181 if tok.kind != cx.TokenKind.COMMENT:
182 continue
183 loc = tok.location
184 if loc.file is None:
185 continue
186 if _is_in_skipped(loc.line, skipped):
187 continue # an inactive branch of the pass that decides -> excluded
188 # Slice the comment text from the raw bytes by offset and decode lossily
189 # (not via tok.spelling, which raises on a non-UTF-8 byte in the comment).
190 # Then normalize CRLF/CR -> LF, matching get_src_strings on the tree-sitter
191 # path: a multi-line block comment (e.g. a reST block) from a CRLF-saved
192 # file otherwise carries embedded \r into the extracted marker text.
193 text = raw[tok.extent.start.offset : tok.extent.end.offset].decode(
194 "utf-8", errors="replace"
195 )
196 spelling = text.replace("\r\n", "\n").replace("\r", "\n")
197 start = tok.extent.start.offset
198 line_start = max(raw.rfind(b"\n", 0, start), raw.rfind(b"\r", 0, start)) + 1
199 column = len(raw[line_start:start].decode("utf-8", errors="replace"))
200 out.append(LibclangComment(spelling.encode("utf-8"), loc.line - 1, column))
201 return out