1import re
2from dataclasses import dataclass
3
4from sphinx_codelinks.analyse.models import WarningSubTypeEnum
5from sphinx_codelinks.config import ESCAPE, UNIX_NEWLINE, OneLineCommentStyle
6
7# Tags of Python docstring conventions (Epydoc, Doxygen, Sphinx field names)
8# that a one-line start sequence such as ``@`` also matches: ``@param a: the
9# first, thing`` would otherwise become a need with the id ``thing``.
10PYTHON_DOCSTRING_TAGS = frozenset(
11 {
12 "arg",
13 "argument",
14 "attention",
15 "author",
16 "brief",
17 "bug",
18 "cvar",
19 "deprecated",
20 "details",
21 "except",
22 "exception",
23 "invariant",
24 "ivar",
25 "key",
26 "keyword",
27 "kwarg",
28 "kwparam",
29 "note",
30 "param",
31 "parameter",
32 "post",
33 "pre",
34 "raise",
35 "raises",
36 "retval",
37 "return",
38 "returns",
39 "rtype",
40 "see",
41 "since",
42 "summary",
43 "throw",
44 "throws",
45 "todo",
46 "tparam",
47 "type",
48 "var",
49 "vartype",
50 "version",
51 "warning",
52 "yield",
53 "yields",
54 "ytype",
55 }
56)
57
58
59#: What follows the tag of an Epydoc field: ``:`` directly (``@return: text``), or
60#: spaces or tabs, one word of characters other than space, tab and ``:``, and ``:``,
61#: as in ``@param a: text``. Locale-free: ubCode spells the same.
62_FIELD_SHAPE = re.compile(r":|[ \t]+[^ \t:]+:")
63
64
65def docstring_tag(line: str, start_sequence: str) -> str | None:
66 """The docstring tag a docstring line starts with, if any.
67
68 The line starts with one when, after comment decoration and whitespace (no
69 alphanumeric character), it holds ``start_sequence`` directly followed by a
70 tag of :data:`PYTHON_DOCSTRING_TAGS` in the Epydoc field shape: ``:``
71 directly, or spaces or tabs, one word without space, tab or ``:``, and ``:``.
72 A line such as ``@todo fix it, IMPL_1, impl`` has no field shape and is left to
73 the one-line parser.
74 """
75 if not start_sequence:
76 return None
77 idx = line.find(start_sequence)
78 if idx == -1 or any(char.isalnum() for char in line[:idx]):
79 return None
80 rest = line[idx + len(start_sequence) :]
81 word_end = 0
82 while word_end < len(rest) and rest[word_end].isalpha():
83 word_end += 1
84 tag = rest[:word_end]
85 if tag not in PYTHON_DOCSTRING_TAGS or not _FIELD_SHAPE.match(rest, word_end):
86 return None
87 return tag
88
89
90# Block tags of JSDoc, TSDoc and TypeScript's JSDoc support, and the inline tags that can
91# start a line, which a one-line start sequence such as ``@`` also matches: ``@param a
92# the first, thing`` would otherwise become a need with the id ``thing``.
93JSDOC_TAGS = frozenset(
94 {
95 "abstract",
96 "access",
97 "alias",
98 "alpha",
99 "arg",
100 "argument",
101 "async",
102 "augments",
103 "author",
104 "beta",
105 "borrows",
106 "callback",
107 "class",
108 "classdesc",
109 "const",
110 "constant",
111 "constructor",
112 "constructs",
113 "copyright",
114 "decorator",
115 "default",
116 "defaultValue",
117 "defaultvalue",
118 "deprecated",
119 "desc",
120 "description",
121 "emits",
122 "enum",
123 "event",
124 "eventProperty",
125 "example",
126 "exception",
127 "experimental",
128 "exports",
129 "extends",
130 "external",
131 "file",
132 "fileoverview",
133 "fires",
134 "func",
135 "function",
136 "generator",
137 "global",
138 "hideconstructor",
139 "host",
140 "ignore",
141 "implements",
142 "import",
143 "inheritDoc",
144 "inheritdoc",
145 "inner",
146 "instance",
147 "interface",
148 "internal",
149 "jsx",
150 "jsxFrag",
151 "jsxImportSource",
152 "jsxRuntime",
153 "kind",
154 "label",
155 "lends",
156 "license",
157 "link",
158 "linkcode",
159 "linkplain",
160 "listens",
161 "member",
162 "memberof",
163 "method",
164 "mixes",
165 "mixin",
166 "modifies",
167 "module",
168 "name",
169 "namespace",
170 "overload",
171 "override",
172 "overview",
173 "package",
174 "packageDocumentation",
175 "param",
176 "private",
177 "privateRemarks",
178 "prop",
179 "property",
180 "protected",
181 "public",
182 "readonly",
183 "remarks",
184 "requires",
185 "return",
186 "returns",
187 "satisfies",
188 "sealed",
189 "see",
190 "since",
191 "static",
192 "summary",
193 "template",
194 "this",
195 "throws",
196 "todo",
197 "tutorial",
198 "type",
199 "typeParam",
200 "typedef",
201 "var",
202 "variation",
203 "version",
204 "virtual",
205 "yield",
206 "yields",
207 }
208)
209
210
211#: What ends a JSDoc tag: a space, a tab, a ``{type}``, the ``}`` of an inline tag, or the
212#: end of the line. Locale-free: ubCode spells the same.
213_JSDOC_TAG_END = re.compile(r"[ \t\n{}]|$")
214
215
216def jsdoc_tag(line: str, start_sequence: str) -> str | None:
217 """The JSDoc tag a line of a JSDoc comment starts with, if any.
218
219 The line starts with one when, after comment decoration and whitespace (no
220 alphanumeric character), it holds ``start_sequence`` directly followed by a tag of
221 :data:`JSDOC_TAGS` that ends there: at a space, a tab, a ``{``, a ``}`` or the end of
222 the line (``@param a text``, ``@param{number} a``, ``{@link Foo}``, ``@returns``).
223 """
224 if not start_sequence:
225 return None
226 idx = line.find(start_sequence)
227 if idx == -1 or any(char.isalnum() for char in line[:idx]):
228 return None
229 rest = line[idx + len(start_sequence) :]
230 word_end = 0
231 while word_end < len(rest) and rest[word_end].isalpha():
232 word_end += 1
233 tag = rest[:word_end]
234 if tag not in JSDOC_TAGS or not _JSDOC_TAG_END.match(rest, word_end):
235 return None
236 return tag
237
238
239#: What follows PowerShell's ``@`` sigil in a hashtable (``@{ }``) or an array
240#: subexpression (``@( )``) literal.
241POWERSHELL_LITERAL_OPENERS = ("{", "(")
242
243
[docs]244# @PowerShell hashtable and array literals are no markers, IMPL_PS_10, impl, [FE_POWERSHELL]
245def opens_powershell_literal(line: str, start_sequence: str) -> bool:
246 """Whether the first ``start_sequence`` of ``line`` is directly followed by ``{``
247 or ``(``, as the ``@`` of PowerShell's hashtable and array literals is: such a
248 line is commented-out code or ``.EXAMPLE`` help, never a marker.
249 """
250 if not start_sequence:
251 return False
252 idx = line.find(start_sequence)
253 if idx == -1:
254 return False
255 return line[idx + len(start_sequence) : idx + len(start_sequence) + 1] in (
256 POWERSHELL_LITERAL_OPENERS
257 )
258
259
260# Doxygen's special commands, by name, as the Doxygen 1.19.0 manual lists them: a line
261# of a Doxygen comment that a one-line start sequence such as ``@`` also matches (``@param
262# clk Clock input, rising edge``) would otherwise become a need with the id ``rising
263# edge``. The list the shared corpus file ``doxygen.yaml`` writes in its header, which
264# ``test_doxygen_commands_are_the_corpus_list`` holds equal.
265DOXYGEN_COMMANDS = frozenset(
266 {
267 "a",
268 "addindex",
269 "addtogroup",
270 "anchor",
271 "arg",
272 "attention",
273 "author",
274 "authors",
275 "b",
276 "brief",
277 "bug",
278 "c",
279 "callergraph",
280 "callgraph",
281 "category",
282 "cite",
283 "class",
284 "code",
285 "collaborationgraph",
286 "concept",
287 "cond",
288 "copybrief",
289 "copydetails",
290 "copydoc",
291 "copyright",
292 "date",
293 "def",
294 "defgroup",
295 "deprecated",
296 "details",
297 "diafile",
298 "dir",
299 "directorygraph",
300 "docbookinclude",
301 "docbookonly",
302 "dontinclude",
303 "dot",
304 "dotfile",
305 "doxyconfig",
306 "e",
307 "else",
308 "elseif",
309 "em",
310 "emoji",
311 "endcode",
312 "endcond",
313 "enddocbookonly",
314 "enddot",
315 "endhtmlonly",
316 "endif",
317 "endinternal",
318 "endlatexonly",
319 "endlink",
320 "endmanonly",
321 "endmermaid",
322 "endmsc",
323 "endparblock",
324 "endrtfonly",
325 "endsecreflist",
326 "enduml",
327 "endverbatim",
328 "endxmlonly",
329 "enum",
330 "example",
331 "exception",
332 "extends",
333 "file",
334 "fileinfo",
335 "fn",
336 "groupgraph",
337 "headerfile",
338 "hidecallergraph",
339 "hidecallgraph",
340 "hidecollaborationgraph",
341 "hidedirectorygraph",
342 "hideenumvalues",
343 "hidegroupgraph",
344 "hideincludedbygraph",
345 "hideincludegraph",
346 "hideinheritancegraph",
347 "hideinitializer",
348 "hideinlinesource",
349 "hiderefby",
350 "hiderefs",
351 "htmlinclude",
352 "htmlonly",
353 "idlexcept",
354 "if",
355 "ifnot",
356 "image",
357 "implements",
358 "important",
359 "include",
360 "includedbygraph",
361 "includedoc",
362 "includegraph",
363 "includelineno",
364 "ingroup",
365 "inheritancegraph",
366 "interface",
367 "internal",
368 "invariant",
369 "latexinclude",
370 "latexonly",
371 "li",
372 "line",
373 "lineinfo",
374 "link",
375 "mainpage",
376 "maninclude",
377 "manonly",
378 "memberof",
379 "mermaid",
380 "mermaidfile",
381 "module",
382 "msc",
383 "mscfile",
384 "n",
385 "name",
386 "namespace",
387 "noop",
388 "nosubgrouping",
389 "note",
390 "overload",
391 "p",
392 "package",
393 "page",
394 "par",
395 "paragraph",
396 "param",
397 "parblock",
398 "plantumlfile",
399 "post",
400 "pre",
401 "private",
402 "privatesection",
403 "property",
404 "protected",
405 "protectedsection",
406 "protocol",
407 "public",
408 "publicsection",
409 "pure",
410 "qualifier",
411 "raisewarning",
412 "ref",
413 "refitem",
414 "related",
415 "relatedalso",
416 "relates",
417 "relatesalso",
418 "remark",
419 "remarks",
420 "requirement",
421 "result",
422 "return",
423 "returns",
424 "retval",
425 "rtfinclude",
426 "rtfonly",
427 "sa",
428 "satisfies",
429 "secreflist",
430 "section",
431 "see",
432 "short",
433 "showdate",
434 "showenumvalues",
435 "showinitializer",
436 "showinlinesource",
437 "showrefby",
438 "showrefs",
439 "since",
440 "skip",
441 "skipline",
442 "snippet",
443 "snippetdoc",
444 "snippetlineno",
445 "startuml",
446 "static",
447 "struct",
448 "subpage",
449 "subparagraph",
450 "subsection",
451 "subsubparagraph",
452 "subsubsection",
453 "tableofcontents",
454 "test",
455 "throw",
456 "throws",
457 "todo",
458 "tparam",
459 "typedef",
460 "union",
461 "until",
462 "var",
463 "verbatim",
464 "verbinclude",
465 "verifies",
466 "version",
467 "vhdlflow",
468 "warning",
469 "weakgroup",
470 "xmlinclude",
471 "xmlonly",
472 "xrefitem",
473 }
474)
475
476
477#: What ends a Doxygen command: a space, a tab, a ``[`` as in ``@param[in]``, a ``{`` as
478#: in ``@code{.vhd}``, or the end of the line. Locale-free.
479_DOXYGEN_COMMAND_END = re.compile(r"[ \t\n\[{]|$")
480
481
482def doxygen_tag(line: str, start_sequence: str) -> str | None:
483 """The Doxygen command a line of a Doxygen comment starts with, if any.
484
485 The line starts with one when, after comment decoration and whitespace (no
486 alphanumeric character), it holds ``start_sequence`` directly followed by a command of
487 :data:`DOXYGEN_COMMANDS` that ends there: at a space, a tab, a ``[``, a ``{`` or the end
488 of the line (``@param clk text``, ``@param[in] clk``, ``@code{.vhd}``, ``@brief``).
489 """
490 if not start_sequence:
491 return None
492 idx = line.find(start_sequence)
493 if idx == -1 or any(char.isalnum() for char in line[:idx]):
494 return None
495 rest = line[idx + len(start_sequence) :]
496 word_end = 0
497 while word_end < len(rest) and rest[word_end].isalpha():
498 word_end += 1
499 tag = rest[:word_end]
500 if tag not in DOXYGEN_COMMANDS or not _DOXYGEN_COMMAND_END.match(rest, word_end):
501 return None
502 return tag
503
504
505@dataclass
506class OnelineParserInvalidWarning:
507 """Invalid oneline comments."""
508
509 sub_type: WarningSubTypeEnum
510 msg: str
511
512
[docs]513# @One-line comment parser for traceability markers, IMPL_OLP_1, impl, [FE_DEF, FE_CMT]
514def oneline_parser( # handel warnings
515 oneline: str, oneline_config: OneLineCommentStyle
516) -> dict[str, str | list[str] | int] | OnelineParserInvalidWarning | None:
517 """
518 Extract the string from the custom one-line comment style with the following steps.
519
520 - Locate the start and end sequences
521 - extract the string between them
522 - return ``None`` for an empty content, whatever the start sequence
523 - on a one-character start sequence, return ``None`` for a content without the
524 field separator where more than one field is required: not a marker, and not a
525 warning either. The rule is ubCode's, and so are its words: "when a
526 single-character start_sequence (like `@`) matches but the content has no field
527 delimiter at all, this is likely a stray match
528 (e.g., Doxygen `@param`, `@brief`, `@return`) rather than a real marker.
529 Treating these as NoMatch avoids noisy TooFewFields warnings on codebases with
530 Doxygen annotations. Multi-character sequences (like `[[`) are specific enough
531 that a missing delimiter should still produce a warning."
532 - apply custom_split to split the strings into a list of fields by `field_split_char`
533 - check the number of required fields and the max number of the given fields
534 - split the strings located in the field with `type: list[str]` to a list of string
535 - introduce the default values to those fields which are not given
536 """
537 # find indices start and end char
538 start_idx = oneline.find(oneline_config.start_sequence)
539 end_idx = oneline.rfind(oneline_config.end_sequence)
540 if start_idx == -1 or end_idx == -1:
541 # start or end sequences do not exist
542 return None
543
544 # A marker whose end sequence is the newline extends to the end of the
545 # line, so an unanchored start sequence would swallow trailing prose. Anchor
546 # such a marker to the start of the comment content: everything preceding
547 # the start sequence must be comment decoration (`//`, `#`, `*`, ...) and
548 # whitespace. A word character before it means the start sequence is part of
549 # free-form prose (e.g. `// see @author, ...`) and the line is ignored
550 # (issue #88). Explicitly-bounded markers (e.g. `[[ ... ]]`) are
551 # self-delimiting and may appear anywhere, so they are exempt.
552 if oneline_config.end_sequence == UNIX_NEWLINE and any(
553 char.isalnum() for char in oneline[:start_idx]
554 ):
555 return None
556
557 # extract the string wrapped by start and end
558 start_idx = start_idx + len(oneline_config.start_sequence)
559 string = oneline[start_idx:end_idx].strip()
560
561 # numbers of needs_fields which are required
562 cnt_required_fields = oneline_config.get_cnt_required_fields()
563
564 if not string:
565 # nothing between the start and end sequences: not a marker either
566 return None
567
568 # a documentation tag such as `@param x`, not a marker (see the docstring)
569 if (
570 len(oneline_config.start_sequence) == 1
571 and oneline_config.field_split_char not in string
572 and cnt_required_fields > 1
573 ):
574 return None
575 # indices of the field which has type:list[str]
576 positions_list_str = oneline_config.get_pos_list_str()
577
578 min_fields = cnt_required_fields
579 max_fields = len(oneline_config.needs_fields)
580
581 string_fields = [
582 _field.strip(" ")
583 for _field in custom_split(
584 string, oneline_config.field_split_char, positions_list_str
585 )
586 ]
587 if len(string_fields) < min_fields:
588 return OnelineParserInvalidWarning(
589 sub_type=WarningSubTypeEnum.too_few_fields,
590 msg=f"{len(string_fields)} given fields, minimum is {min_fields}",
591 )
592
593 if len(string_fields) > max_fields:
594 return OnelineParserInvalidWarning(
595 sub_type=WarningSubTypeEnum.too_many_fields,
596 msg=f"{len(string_fields)} given fields, maximum is {max_fields}",
597 )
598 resolved: dict[str, str | list[str] | int] = {}
599 for idx in range(len(oneline_config.needs_fields)):
600 field_name: str = oneline_config.needs_fields[idx]["name"]
601 if len(string_fields) > idx:
602 # given fields
603 if is_newline_in_field(string_fields[idx]):
604 # the case where the field contains a new line character
605 return OnelineParserInvalidWarning(
606 sub_type=WarningSubTypeEnum.newline_in_field,
607 msg=f"Field '{field_name}' contains a newline character",
608 )
609 if oneline_config.needs_fields[idx]["type"] == "str":
610 resolved[field_name] = string_fields[idx]
611 elif oneline_config.needs_fields[idx]["type"] == "list[str]":
612 # find the indices of "[" and "]"
613 list_start_idx = string_fields[idx].find("[")
614 list_end_idx = string_fields[idx].rfind("]")
615 if list_start_idx == -1 or list_end_idx == -1:
616 # brackets are not found
617 return OnelineParserInvalidWarning(
618 sub_type=WarningSubTypeEnum.missing_square_brackets,
619 msg=f"Field '{field_name}' with 'type': '{oneline_config.needs_fields[idx]['type']}' must be given with '[]' brackets",
620 )
621
622 if list_start_idx != 0 or list_end_idx != len(string_fields[idx]) - 1:
623 # brackets are found but not at the beginning and the end
624 return OnelineParserInvalidWarning(
625 sub_type=WarningSubTypeEnum.not_start_or_end_with_square_brackets,
626 msg=f"Field '{field_name}' with 'type': '{oneline_config.needs_fields[idx]['type']}' must start with '[' and end with ']'",
627 )
628
629 string_items = string_fields[idx][list_start_idx + 1 : list_end_idx]
630
631 if not string_items.strip():
632 # the case where the empty string ("") or only spaces between "[" "]"
633 resolved[field_name] = []
634 else:
635 items = [_item.strip() for _item in custom_split(string_items, ",")]
636 resolved[field_name] = [item.strip() for item in items]
637 else:
638 # for not given fields, introduce the default
639 default = oneline_config.needs_fields[idx].get("default")
640 if default is None:
641 continue
642 resolved[field_name] = default
643
644 resolved["start_column"] = start_idx
645 resolved["end_column"] = end_idx
646 return resolved
647
648
649def custom_split(
650 string: str, delimiter: str, positions_list_str: list[int] | None = None
651) -> list[str]:
652 """
653 A string shall be split with the following conditions:
654
655 - To use special chars in literal , escape ('\') must be used
656 - String shall be split by the given delimiter
657 - In a field with `type: str`:
658 - Special chars are delimiter, '\', '[' and ']'
659 - In a field with `type: list[str]`:
660 - Special chars are only '[' and ']'
661
662 When the string is given without any fields with `type: list[str]` (positions_list_str=None),
663 it's considered as it is in a field with `type: str`.
664 """
665 if positions_list_str is None:
666 positions_list_str = []
667 escape_chars = [delimiter, "[", "]", ESCAPE]
668 field = [] # a list of string for a field
669 fields: list[str] = [] # a list of string which contains
670 leading_escape = False
671 expect_closing_bracket = False
672
673 for char in string:
674 # +1 to locate the current field position
675 current_field_idx = len(fields) + 1
676 is_list_str_field = current_field_idx in positions_list_str
677
678 if leading_escape:
679 if char not in escape_chars:
680 # leading escape is considered as a literal
681 field.append(ESCAPE)
682 field.append(char)
683 leading_escape = False
684 continue
685
686 if char == ESCAPE and not is_list_str_field:
687 leading_escape = True
688 continue
689
690 if char == delimiter:
691 if is_list_str_field and expect_closing_bracket:
692 # delimiter occurs in the field with type:list[str]
693 field.append(char)
694 else:
695 fields.append("".join(field))
696 field = []
697 continue
698
699 if is_list_str_field:
700 if char == "[":
701 expect_closing_bracket = True
702 if char == "]":
703 expect_closing_bracket = False
704
705 field.append(char)
706
707 # add last field
708 fields.append("".join(field))
709 return fields
710
711
712def is_newline_in_field(field: str) -> bool:
713 """
714 Check if the field contains a new line character.
715 """
716 return UNIX_NEWLINE in field
Back to top