[docs] 1# @Test suite for source file discovery with gitignore support, TEST_DISC_1, test, [IMPL_DISC_1]
2import json
3import subprocess
4from pathlib import Path
5
6import pytest
7
8from sphinx_codelinks.source_discover.config import (
9 COMMENT_FILETYPE,
10 SourceDiscoverConfig,
11 SourceDiscoverConfigType,
12)
13from sphinx_codelinks.source_discover.source_discover import SourceDiscover
14
15FIXTURES_PATH = Path(__file__).parent / "data" / "discover_fixtures.json"
16
17
18def test_source_directory_is_worker_local(
19 source_directory: Path, tmp_path_factory: pytest.TempPathFactory
20) -> None:
21 assert source_directory.is_relative_to(tmp_path_factory.getbasetemp())
22 assert (source_directory / "charge" / "demo_1.cpp").is_file()
23 assert (source_directory / ".gitignore").read_text(
24 encoding="utf-8"
25 ) == "demo_1.cpp\n"
26
27
28@pytest.mark.parametrize(
29 ("config", "msgs"),
30 [
31 (
32 {
33 "src_dir": 123,
34 "exclude": ["exclude1", "exclude2"],
35 "include": ["include1", "include2"],
36 "gitignore": True,
37 "comment_type": "cpp",
38 },
39 ["Schema validation error in field 'src_dir': 123 is not of type 'string'"],
40 ),
41 (
42 {
43 "src_dir": "/path/to/root",
44 "exclude": ["exclude1", "exclude2"],
45 "include": ["include1", "include2"],
46 "gitignore": "TrueAsString",
47 "comment_type": "cpp",
48 },
49 [
50 "Schema validation error in field 'gitignore': 'TrueAsString' is not of type 'boolean'"
51 ],
52 ),
53 (
54 {
55 "src_dir": "/path/to/root",
56 "exclude": ["exclude1", "exclude2"],
57 "include": ["include1", "include2"],
58 "gitignore": True,
59 "comment_type": "java",
60 },
61 [
62 "Schema validation error in field 'comment_type': 'java' is not one of ['bash', 'cpp', 'cs', 'go', 'jsonc', 'powershell', 'python', 'rust', 'ts', 'vhdl', 'yaml']"
63 ],
64 ),
65 (
66 {
67 "src_dir": "/path/to/root",
68 "exclude": ["exclude1", "exclude2"],
69 "include": ["include1", "include2"],
70 "gitignore": True,
71 "comment_type": ["cpp", "hpp"],
72 },
73 [
74 "Schema validation error in field 'comment_type': ['cpp', 'hpp'] is not of type 'string'"
75 ],
76 ),
77 (
78 {
79 "src_dir": "/path/to/root",
80 "follow_links": "not_a_bool",
81 },
82 [
83 "Schema validation error in field 'follow_links': 'not_a_bool' is not of type 'boolean'"
84 ],
85 ),
86 ],
87)
88def test_schema_negative(config, msgs):
89 source_discover_config = SourceDiscoverConfig(**config)
90 errors = source_discover_config.check_schema()
91 assert sorted(errors) == sorted(msgs)
92
93
94@pytest.mark.parametrize(
95 "config",
96 [
97 {},
98 {
99 "src_dir": "/path/to/root",
100 "exclude": ["exclude1", "exclude2"],
101 "include": ["include1", "include2"],
102 "gitignore": True,
103 "comment_type": "cpp",
104 },
105 {
106 "src_dir": "/path/to/root",
107 "exclude": ["exclude1", "exclude2"],
108 "include": ["include1", "include2"],
109 "gitignore": True,
110 "comment_type": "python",
111 },
112 {
113 "src_dir": "/path/to/root",
114 "exclude": ["exclude1", "exclude2"],
115 "include": ["include1", "include2"],
116 "gitignore": True,
117 "comment_type": "ts",
118 },
119 {
120 "src_dir": "/path/to/root",
121 "follow_links": True,
122 },
123 ],
124)
125def test_schema_positive(config):
126 source_discover_config = SourceDiscoverConfig(**config)
127 errors = source_discover_config.check_schema()
128 assert len(errors) == 0
129
130
131@pytest.mark.parametrize(
132 ("config", "num_files", "suffix"),
133 [
134 (
135 {
136 "gitignore": False,
137 },
138 4,
139 "",
140 ),
141 (
142 {
143 "gitignore": True,
144 },
145 3,
146 "",
147 ),
148 (
149 {
150 "gitignore": True,
151 "exclude": ["charge/*.cpp"],
152 "include": ["**/*.cpp"],
153 },
154 # With ignore-python, include patterns whitelist files (overriding
155 # gitignore) and exclude patterns are applied after, so both
156 # charge/*.cpp files are excluded resulting in 2 instead of 4.
157 2,
158 "",
159 ),
160 (
161 {
162 "gitignore": True,
163 "exclude": ["charge/*.cpp"],
164 },
165 2,
166 "",
167 ),
168 (
169 {"gitignore": False, "comment_type": "cpp"},
170 4,
171 "cpp",
172 ),
173 ],
174)
175def test_source_discover(
176 config: SourceDiscoverConfigType,
177 num_files: int,
178 suffix: str,
179 source_directory: Path,
180) -> None:
181 config["src_dir"] = source_directory
182 src_discover_config = SourceDiscoverConfig(**config)
183 source_discover = SourceDiscover(src_discover_config)
184 assert len(source_discover.source_paths) == num_files
185 if suffix:
186 assert all(path.suffix == ".cpp" for path in source_discover.source_paths)
187
188
189@pytest.fixture(scope="function")
190def create_source_files(tmp_path: Path) -> Path:
191 for file_types in COMMENT_FILETYPE.values():
192 for ext in file_types:
193 (tmp_path / f"file.{ext}").touch()
194 return tmp_path
195
196
197@pytest.mark.parametrize(
198 ("comment_type", "nums_files"),
199 [
200 ("cpp", len(COMMENT_FILETYPE["cpp"])),
201 ("python", len(COMMENT_FILETYPE["python"])),
202 ("ts", len(COMMENT_FILETYPE["ts"])),
203 ("bash", len(COMMENT_FILETYPE["bash"])),
204 ("powershell", 3),
205 ("vhdl", len(COMMENT_FILETYPE["vhdl"])),
206 ],
207)
208def test_comment_filetype(
209 comment_type: str, nums_files: int, create_source_files: Path
210) -> None:
211 src_dir = create_source_files
212
213 config = SourceDiscoverConfig(
214 src_dir=src_dir, comment_type=comment_type, gitignore=False
215 )
216 source_discover = SourceDiscover(config)
217 assert len(source_discover.source_paths) == nums_files
218
219
220def test_jsonc_discover_gate() -> None:
221 """`.jsonc` is always discovered; `.json` only when it opens with a comment."""
222 jsonc_dir = Path(__file__).parent / "data" / "jsonc"
223 config = SourceDiscoverConfig(
224 src_dir=jsonc_dir, comment_type="jsonc", gitignore=False
225 )
226 discovered = {p.name for p in SourceDiscover(config).source_paths}
227 assert "demo.jsonc" in discovered
228 assert "with_modeline.json" in discovered
229 assert "plain.json" not in discovered
230
231
232def _make_generated_output_tree(tmp_path: Path) -> Path:
233 """Lay out a source file alongside checked-in generated output.
234
235 Mirrors a ``tsc``/bundler output tree: ``src/app.ts`` is the real source,
236 while ``dist/``, ``build/``, ``out/``, ``coverage/`` and ``node_modules/``
237 stand in for generated or vendored output that carries the same marker;
238 ``lib/app.js`` and the declaration file ``types/x.d.ts`` are not in the
239 default and stay discovered.
240 """
241 (tmp_path / "src").mkdir()
242 (tmp_path / "src" / "app.ts").write_text(
243 "// @Feature A, IMPL_1, impl\n", encoding="utf-8"
244 )
245 (tmp_path / "lib").mkdir()
246 (tmp_path / "lib" / "app.js").write_text(
247 "// @Feature A, IMPL_1, impl\n", encoding="utf-8"
248 )
249 (tmp_path / "dist").mkdir()
250 (tmp_path / "dist" / "app.js").write_text(
251 "// @Feature A, IMPL_1, impl\n", encoding="utf-8"
252 )
253 for output in ("build", "out", "coverage"):
254 (tmp_path / output).mkdir()
255 (tmp_path / output / "app.js").write_text(
256 "// @Feature A, IMPL_1, impl\n", encoding="utf-8"
257 )
258 (tmp_path / "types").mkdir()
259 (tmp_path / "types" / "x.d.ts").write_text(
260 "/** @Feature A, IMPL_1, impl */\nexport declare function a(): void;\n",
261 encoding="utf-8",
262 )
263 (tmp_path / "node_modules" / "pkg").mkdir(parents=True)
264 (tmp_path / "node_modules" / "pkg" / "index.js").write_text(
265 "// vendored\n", encoding="utf-8"
266 )
267 return tmp_path
268
269
270def test_default_exclude_skips_generated_output(tmp_path: Path) -> None:
271 """The ``ts``-derived default ``exclude`` keeps generated/vendored JS out
272 of discovery, while ``lib/`` — deliberately not in ``TS_DEFAULT_EXCLUDE``
273 — is still discovered (see the constant's docstring for why)."""
274 src_dir = _make_generated_output_tree(tmp_path)
275 config = SourceDiscoverConfig(src_dir=src_dir, comment_type="ts", gitignore=False)
276
277 discover = SourceDiscover(config)
278 discovered = sorted(str(p.relative_to(src_dir)) for p in discover.source_paths)
279 assert discovered == [
280 str(Path("lib") / "app.js"),
281 str(Path("src") / "app.ts"),
282 str(Path("types") / "x.d.ts"),
283 ]
284
285
286def test_cpp_project_default_exclude_is_empty_and_finds_lib_marker(
287 tmp_path: Path,
288) -> None:
289 """The ``ts`` default ``exclude`` is ``ts``'s alone: a ``cpp`` project's is ``[]``, and
290 its hand-written ``lib/`` source is discovered."""
291 lib_dir = tmp_path / "lib"
292 lib_dir.mkdir()
293 (lib_dir / "widget.cpp").write_text(
294 "// @Feature A, IMPL_1, impl\n", encoding="utf-8"
295 )
296
297 config = SourceDiscoverConfig(src_dir=tmp_path, comment_type="cpp", gitignore=False)
298 assert config.exclude == []
299
300 discover = SourceDiscover(config)
301 discovered = sorted(str(p.relative_to(tmp_path)) for p in discover.source_paths)
302 assert discovered == [str(Path("lib") / "widget.cpp")]
303
304
305def test_explicit_exclude_replaces_default(tmp_path: Path) -> None:
306 """An explicit ``exclude`` (even ``[]``) fully replaces the default list."""
307 src_dir = _make_generated_output_tree(tmp_path)
308 config = SourceDiscoverConfig(
309 src_dir=src_dir, comment_type="ts", gitignore=False, exclude=[]
310 )
311 assert config.exclude == []
312
313 discover = SourceDiscover(config)
314 discovered = sorted(str(p.relative_to(src_dir)) for p in discover.source_paths)
315 assert discovered == [
316 str(Path("build") / "app.js"),
317 str(Path("coverage") / "app.js"),
318 str(Path("dist") / "app.js"),
319 str(Path("lib") / "app.js"),
320 str(Path("node_modules") / "pkg" / "index.js"),
321 str(Path("out") / "app.js"),
322 str(Path("src") / "app.ts"),
323 str(Path("types") / "x.d.ts"),
324 ]
325
326
327def test_follow_links(tmp_path: Path) -> None:
328 """Test that follow_links controls whether symbolic links are followed."""
329 # Create a project directory with a real directory the walk excludes, holding a
330 # source file: inside the project, since a file outside it is never discovered
331 # (#2062, ``test_a_link_to_outside_the_root_is_not_listed``)
332 project_dir = tmp_path / "project"
333 real_dir = project_dir / "real"
334 real_dir.mkdir(parents=True)
335 (real_dir / "source.cpp").write_text("// test")
336 (project_dir / "direct.cpp").write_text("// direct")
337 # and a symlink to the real directory
338 link = project_dir / "linked"
339 link.symlink_to(real_dir)
340
341 # Without follow_links, symlinked files should not be discovered
342 config_no_follow = SourceDiscoverConfig(
343 src_dir=project_dir, gitignore=False, exclude=["real/**"], follow_links=False
344 )
345 discover_no_follow = SourceDiscover(config_no_follow)
346 discovered_names = {p.name for p in discover_no_follow.source_paths}
347 assert "direct.cpp" in discovered_names
348 assert "source.cpp" not in discovered_names
349
350 # With follow_links, symlinked files should be discovered
351 config_follow = SourceDiscoverConfig(
352 src_dir=project_dir, gitignore=False, exclude=["real/**"], follow_links=True
353 )
354 discover_follow = SourceDiscover(config_follow)
355 discovered_names = {p.name for p in discover_follow.source_paths}
356 assert "direct.cpp" in discovered_names
357 assert "source.cpp" in discovered_names
358
359
360def _load_discover_fixtures() -> list[dict]:
361 with FIXTURES_PATH.open(encoding="utf-8") as f:
362 return json.load(f)
363
364
365@pytest.mark.parametrize(
366 "case",
367 _load_discover_fixtures(),
368 ids=lambda c: c["name"],
369)
370def test_discover_fixture(case: dict, tmp_path: Path) -> None:
371 """Run portable discovery test cases from the shared JSON fixture."""
372 # Create files
373 for rel_path, content in case["files"].items():
374 file_path = tmp_path / rel_path
375 file_path.parent.mkdir(parents=True, exist_ok=True)
376 file_path.write_text(content, encoding="utf-8")
377
378 # Optionally initialise a git repo (required for .gitignore support)
379 if case.get("git_init", False):
380 subprocess.run(
381 ["git", "init"],
382 cwd=str(tmp_path),
383 check=True,
384 capture_output=True,
385 )
386
387 cfg = case["config"]
388 src_dir = tmp_path / cfg["src_dir"]
389
390 config = SourceDiscoverConfig(
391 src_dir=src_dir,
392 include=cfg.get("include", []),
393 exclude=cfg.get("exclude", []),
394 gitignore=cfg.get("gitignore", True),
395 comment_type=cfg.get("comment_type", "cpp"),
396 )
397
398 discover = SourceDiscover(config)
399
400 # Convert discovered paths to paths relative to tmp_path for comparison
401 discovered_relative = sorted(
402 str(p.relative_to(tmp_path)) for p in discover.source_paths
403 )
404
405 # Normalise expected paths to use the OS path separator
406 expected = sorted(str(Path(p)) for p in case["expected"])
407
408 assert discovered_relative == expected, (
409 f"Case '{case['name']}': expected {expected}, got {discovered_relative}"
410 )
411
412
413def _links_tree(tmp_path: Path) -> Path:
414 """``root/{a.cpp, sub/b.cpp}`` with four links: a file and a directory link inside
415 the root (``b_link.cpp`` -> ``sub/b.cpp``, ``dirlink_in`` -> ``sub``), and a file
416 and a directory link to outside it (``ext_link.cpp`` -> ``outside/ext.cpp``,
417 ``dirlink_out`` -> ``outside/odir``). Returns the root."""
418 root = tmp_path / "root"
419 (root / "sub").mkdir(parents=True)
420 (tmp_path / "outside" / "odir").mkdir(parents=True)
421 (root / "a.cpp").write_text("// a\n", encoding="utf-8")
422 (root / "sub" / "b.cpp").write_text("// b\n", encoding="utf-8")
423 (tmp_path / "outside" / "ext.cpp").write_text("// ext\n", encoding="utf-8")
424 (tmp_path / "outside" / "odir" / "o.cpp").write_text("// o\n", encoding="utf-8")
425 (root / "b_link.cpp").symlink_to(root / "sub" / "b.cpp")
426 (root / "dirlink_in").symlink_to(root / "sub", target_is_directory=True)
427 (root / "ext_link.cpp").symlink_to(tmp_path / "outside" / "ext.cpp")
428 (root / "dirlink_out").symlink_to(
429 tmp_path / "outside" / "odir", target_is_directory=True
430 )
431 return root
432
433
434def _listing(root: Path, *, follow_links: bool) -> list[str]:
435 """What discovery lists under ``root``, POSIX and relative to ``root``'s parent
436 (``root.parent`` resolved: the listing is of canonical paths)."""
437 config = SourceDiscoverConfig(
438 src_dir=root, gitignore=False, follow_links=follow_links
439 )
440 base = root.parent.resolve()
441 return [
442 path.relative_to(base).as_posix()
443 for path in SourceDiscover(config).source_paths
444 ]
445
446
447def test_a_file_link_inside_the_root_is_listed_once(tmp_path: Path) -> None:
448 """``b_link.cpp`` is ``sub/b.cpp``: one entry, the target's, not one per path to it.
449
450 A symlinked file is listed even with ``follow_links = false`` (``is_file()``
451 follows the link) -- ubCode skips it; a parity gap kept as it is. A link to a file
452 OUTSIDE the root is not listed (#2062).
453 """
454 assert _listing(_links_tree(tmp_path), follow_links=False) == [
455 "root/a.cpp",
456 "root/sub/b.cpp",
457 ]
458
459
460def test_a_followed_directory_link_inside_the_root_lists_each_file_once(
461 tmp_path: Path,
462) -> None:
463 """With ``follow_links = true`` ``sub/b.cpp`` is reached three ways (``sub/``,
464 ``b_link.cpp`` and ``dirlink_in/``): still one entry. The followed link to outside
465 the root lists nothing (#2062)."""
466 assert _listing(_links_tree(tmp_path), follow_links=True) == [
467 "root/a.cpp",
468 "root/sub/b.cpp",
469 ]
470
471
472def test_a_link_free_listing_is_sorted_by_full_path(tmp_path: Path) -> None:
473 """Without links nothing is deduplicated, and the order is the full path's: ``-``
474 and ``.`` sort before the separator, ``_`` after it (on POSIX and on Windows)."""
475 root = tmp_path / "root"
476 for relative in [
477 "b.cpp",
478 "sub_x.cpp",
479 "z/y/x.cpp",
480 "sub/c.cpp",
481 "a.cpp",
482 "sub.cpp",
483 "sub-x/d.cpp",
484 ]:
485 path = root / relative
486 path.parent.mkdir(parents=True, exist_ok=True)
487 path.write_text("// x\n", encoding="utf-8")
488
489 assert _listing(root, follow_links=False) == [
490 "root/a.cpp",
491 "root/b.cpp",
492 "root/sub-x/d.cpp",
493 "root/sub.cpp",
494 "root/sub/c.cpp",
495 "root/sub_x.cpp",
496 "root/z/y/x.cpp",
497 ]
498 assert _listing(root, follow_links=True) == _listing(root, follow_links=False)
499
500
501@pytest.mark.parametrize("follow_links", [False, True])
502def test_a_link_to_outside_the_root_is_not_listed(
503 tmp_path: Path, follow_links: bool
504) -> None:
505 """Neither a file link to outside the root nor a followed directory link's files
506 are listed, either way (#2062): every record and copy is relative to the root."""
507 listing = _listing(_links_tree(tmp_path), follow_links=follow_links)
508 assert not any(path.startswith("outside/") for path in listing)
509 assert not any("link" in path for path in listing)