diff --git a/AGENTS.md b/AGENTS.md index a07c3a5700..0705adefe8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,6 +75,10 @@ rules. Read the complete governance document before making or reviewing changes: the stream state (`fail()`/`eof()`) after extraction and abort with a diagnostic message on malformed input rather than silently defaulting to zero. + 20. Do not introduce new `#define` macros without approval. Include + guards and `#ifdef`/`#ifndef` usages of existing macros are exempt. + 21. Do not initialize member variables from `PARAM` in `.h`/`.hpp` + files; pass the value through the constructor or a setter instead. - Use LF line endings for text files. Only `.bat` and `.cmd` files may use CRLF. - Keep source file additions deterministic: update the relevant `CMakeLists.txt` or explain why the file is generated or included indirectly. diff --git a/tools/03_code_analysis/code_quality_score.py b/tools/03_code_analysis/code_quality_score.py index 97f25bd2d7..59c0ea8219 100755 --- a/tools/03_code_analysis/code_quality_score.py +++ b/tools/03_code_analysis/code_quality_score.py @@ -44,6 +44,12 @@ the parameter list spans lines. - #include of .hpp implementation header: -2 per occurrence (cap 5) (AGENTS.md §3, §4) + - function call passing a ternary `?:`, `&&` or `||` expression as + a top-level argument: -1 per offending call (cap 5) + (AGENTS.md §14). Assign the value to a named local variable + first, then pass it. Operators nested inside an inner call's + parentheses are judged on the inner call; declarations, + definitions, control-statement headers and lambdas are excluded. - each public member variable in a class/struct: -1 - each static member variable in a class/struct: -1 (cap 10) Excludes static_assert, static member functions, and static_cast. @@ -60,6 +66,12 @@ complex functions in the same file accumulate) - file longer than 500 lines: -2 per additional 50-line block (counts all physical lines including blanks and comments; no cap) + - .h header longer than 120 lines: -1 per additional 10-line block + (cap 5; in addition to the generic 500-line file rule above) + - non-inline, non-template function body in a .h header longer than + 20 lines: -1 per additional 10-line block (cap 5; `inline` and + `template <...>` functions are exempt, short accessor bodies at or + below 20 lines are exempt by construction) - member function longer than 50 lines: -1 per additional 50-line block (counts physical lines after comments are blanked, blanks preserved) @@ -79,7 +91,7 @@ `X::UPPER` or `x.UPPER` (those are governed by the cross-layer-dependency rule or another file's class definition). - tab indentation: -1 per line (cap 5) - - `using namespace std;`: -1 per occurrence (cap 5) + - `using namespace ;` directive: -1 per occurrence (cap 5) - `goto` keyword: -2 per occurrence (cap 5) - `auto` keyword: -1 per occurrence (cap 10). Counted after comments and string literals are stripped; `auto_ptr` does not @@ -88,11 +100,23 @@ - Chinese characters in comments/code: -1 per line (cap 5) - preprocessor directive not at column 0 (indented #if/#include/...): -1 per line (cap 5) + - `#if`/`#ifdef`/`#ifndef` block inside a function call's argument + list (conditional compilation wrapping individual arguments): + -1 per opening directive (cap 5). Select the argument (or the + overload) before the call instead of wrapping arguments in the + preprocessor. - comment block above the first include guard of a .h file: -1 per contiguous comment block (a file-header is one block, so typically -1) - `@file` name does not match the actual filename: -1 - comment text duplicated across comment blocks (same >=40-char line appearing in two or more blocks): -1 per duplicated line (cap 5) + - new `#define` macro in a .h/.cpp file: -3 per definition (cap 10). + Include guards (macro name ending in `_H`/`_HPP` with no parameter + list or replacement text) and `#ifdef`/`#ifndef` usages of + existing macros are exempt (AGENTS.md rule 20). + - member variable initialized from `PARAM` in a .h/.hpp file: -5 per + declaration (cap 10). Pass the value through the constructor or a + setter instead (AGENTS.md rule 21). Usage: python3 code_quality_score.py source/source_base @@ -103,1949 +127,24 @@ python3 code_quality_score.py --min-score 80 source/ """ -from __future__ import annotations +"""Entry-point module: CLI parsing and orchestration only. + +Implementation was split into: + cqs_config.py - weights/caps/thresholds/regexes/prefix helpers + cqs_text.py - comment/string stripping, comment-block extraction + cqs_rules.py - rule-detection (find_*/analyze_*) functions + cqs_analyze.py - per-file analysis orchestration, file discovery + cqs_report.py - Finding/FileReport dataclasses, text/JSON renderers +""" import argparse -import json -import re import sys -from dataclasses import dataclass, field from pathlib import Path -from typing import List, Optional, Sequence, Tuple - - -SOURCE_EXTENSIONS = {".c", ".cc", ".cpp", ".cxx", ".cu", ".h", ".hh", ".hpp", ".hxx", ".cuh"} - -SKIP_DIRS = { - ".git", "build", "__pycache__", "node_modules", ".cache", - "third_party", "thirdparty", ".vscode", ".idea", ".trae-cn", - "Dependencies", - "test", "tests", "test_serial", "test_parallel", "test_gpu", "unit_test", "unittest", "unittests", -} - -CAPS = { - "tab_indentation": 5, - "using_namespace_std": 5, - "line_too_long": 5, - "chinese_comment": 5, - "uppercase_constant": 5, - "default_parameter": 5, - "global_dependency": 10, - "hpp_include": 5, - "friend_keyword": 5, - "unpaired_new_delete": 5, - "member_local_name_conflict": 5, -} - -WEIGHTS = { - "filename_too_long": 1, - "filename_uppercase": 1, - "hpp_implementation": 50, - "public_member_variable": 1, - "member_function_too_long": 1, - "tab_indentation": 1, - "using_namespace_std": 1, - "line_too_long": 1, - "chinese_comment": 1, - "uppercase_constant": 1, - "default_parameter": 2, - "global_dependency": 3, - "hpp_include": 2, - "friend_keyword": 1, - "unpaired_new_delete": 1, - "member_local_name_conflict": 1, - "file_too_long": 2, - "raw_new_keyword": 1, - "too_many_parameters": 1, - "high_cyclomatic_complexity": 1, - "post_cpp11_feature": 40, - "post_cpp11_per_feature": 8, - "goto_keyword": 2, - "auto_keyword": 1, - "static_member_variable": 1, - "indented_preprocessor": 1, - "comment_above_guard": 1, - "doc_file_mismatch": 1, - "duplicate_doc_block": 1, -} - -CAPS = { - "tab_indentation": 5, - "using_namespace_std": 5, - "line_too_long": 5, - "chinese_comment": 5, - "uppercase_constant": 5, - "default_parameter": 5, - "global_dependency": 10, - "hpp_include": 5, - "friend_keyword": 5, - "unpaired_new_delete": 5, - "member_local_name_conflict": 5, - "too_many_parameters": 30, - "high_cyclomatic_complexity": 30, - "post_cpp11_per_feature": 5, - "goto_keyword": 5, - "auto_keyword": 10, - "static_member_variable": 10, - "indented_preprocessor": 5, - "comment_above_guard": 3, - "duplicate_doc_block": 5, -} - -FUNCTION_LENGTH_THRESHOLD = 50 -FUNCTION_LENGTH_STEP = 50 -FILE_LENGTH_THRESHOLD = 500 -FILE_LENGTH_STEP = 50 -FUNCTION_PARAM_THRESHOLD = 7 -CYCLO_THRESHOLD = 10 -LINE_LENGTH_LIMIT = 120 -FILENAME_LENGTH_LIMIT = 20 -PASS_THRESHOLD = 60 - -CHINESE_RE = re.compile("[\u4e00-\u9fff]") -USING_NS_STD_RE = re.compile(r"\busing\s+namespace\s+std\b") -UPPERCASE_CONST_RE = re.compile(r"(?])\b[A-Z][A-Z0-9_]{2,}\b(?![.:])") -ACCESS_RE = re.compile(r"^\s*(public|private|protected)\s*:") -CLASS_OPEN_RE = re.compile(r"\b(class|struct)\s+(\w+)\b") - -GLOBAL_DEPENDENCY_RE = re.compile(r"\b(?:GlobalV::|GlobalC::|PARAM(?:\.|->|::))") -HPP_INCLUDE_RE = re.compile(r'^\s*#\s*include\s+[<"][^>"]+\.hpp[>"]') -FRIEND_RE = re.compile(r"\bfriend\b(?!\s+(?:class\s+|struct\s+)?std::)") -NEW_EXPR_RE = re.compile(r"\bnew\s+\w") -DELETE_EXPR_RE = re.compile(r"\bdelete\s*\[\s*\]?\s+\w") - -# Occurrences of `new T(...)` or `new T[...]` that are clearly owned by a -# smart-pointer wrapper and therefore should NOT count as unpaired -# new/delete leaks. Two idiomatic C++11 shapes are recognised: -# 1. `ptr.reset(new T(args))` / `ptr->reset(new T(args))` -# — reset() transfers ownership into an existing smart pointer; -# most commonly used with unique_ptr/shared_ptr when make_unique/ -# make_shared are not available (pre-C++14) or unsuitable. -# 2. `unique_ptr p(new T(args))` / `shared_ptr p(new T(args))` -# plus the allocator-taking `allocate_shared` form — direct -# construction of a smart pointer that owns the freshly-allocated -# object from day one. -# The pattern does NOT try to skip new in `raw_new_count` (that rule -# penalises *any* direct use of `new` vs. make_unique/make_shared); it -# only affects the ownership heuristic `unpaired_new`. -_OWNED_NEW_SHAPES = [ - # shape 1: .reset(new T or ->reset(new T (optionally with whitespace) - r"\.(?:reset|operator\s*=)\s*\(\s*new\s+\w", - r"->(?:reset|operator\s*=)\s*\(\s*new\s+\w", - # shape 2: unique_ptr ident ( new T - # shared_ptr ident ( new T - # allocate_shared ( alloc , new T -> not exact, but - # the simpler `allocate_shared<...> ( ... new T` heuristic - r"\b(?:unique_ptr|shared_ptr|scoped_ptr|auto_ptr)\s*<[^>]*>\s*[A-Za-z_]\w*\s*\(\s*new\s+\w", - r"\b(?:make_unique|make_shared|allocate_shared)\s*<[^>]*>\s*\([^;()]*?\bnew\s+\w", -] -OWNED_NEW_RE = re.compile("|".join(f"(?:{p})" for p in _OWNED_NEW_SHAPES)) -GOTO_RE = re.compile(r"\bgoto\b") -# `auto` type deduction keyword. `\bauto\b` does not match `auto_ptr` -# (`_` is a word character, so there is no boundary before it). -AUTO_RE = re.compile(r"\bauto\b") - -# Preprocessor directive that does not start at column 0 (leading spaces/tabs -# before the '#'). Matched against raw lines; string literals virtually never -# contain a line-start '#', so no string stripping is needed here. -INDENTED_PREPROC_RE = re.compile( - r"^[ \t]+#\s*(?:if|ifdef|ifndef|define|endif|include|undef|pragma|else|elif)\b" -) -# First include guard / pragma-once line of a header. -GUARD_RE = re.compile(r"^\s*#\s*(?:ifndef|pragma\s+once)\b") -# `@file name` inside a doxygen comment. -DOC_FILE_RE = re.compile(r"@file\s+(\S+)") -# Comment-only line (/// ... or a line inside a /* */ block is not fully -# distinguishable line-wise, so we only treat leading // or /* as comment). -COMMENT_LINE_RE = re.compile(r"^\s*(?://|/\*|\*)") - -DEFAULT_PARAM_RE = re.compile( - r"\b\w+\s*\([^();]*\b\w+\s*=(?![=>])[^();]*\)\s*" - r"(?:const\s*|noexcept\s*|override\s*|final\s*)*[;{]" -) - -# Strict declaration regex: requires `type ... name ;` or `type ... name = ...;`. -# Excludes pure assignments like `counter = counter + 1;` which lack a leading type. -DECL_RE = re.compile(r"^\s*(?:\w+[\s\*]+)+(\w+)\s*(?:=[^;]*)?;\s*$") - -BLOCK_KEYWORD_PREFIXES = ( - "public:", "private:", "protected:", - "typedef", "using", "static_assert", - "class", "struct", "friend", - "//", "/*", "*", - "return", "if ", "for ", "while ", - "switch ", "case ", "template", - "break", "continue", "goto", "default:", - "else", "do", "try", "catch", "throw", - "namespace", "enum", "union", -) - -NON_FUNCTION_KEYWORDS = { - "if", "for", "while", "switch", "sizeof", "return", "throw", - "catch", "class", "struct", "namespace", "enum", "union", - "template", "static_assert", "typedef", "using", "new", - "delete", "operator", "do", "else", "goto", "continue", - "break", "try", -} - -# Keywords that, when they appear as the last identifier in the prefix -# before a `name(args)` candidate followed by `;` or `=`, strongly -# indicate the candidate is a function call in statement/expression -# context rather than a declaration. -STATEMENT_CONTEXT_KEYWORDS = NON_FUNCTION_KEYWORDS | { - "co_await", "co_return", "co_yield", - "typeid", "noexcept", "alignof", "alignas", "decltype", - "static_cast", "const_cast", "dynamic_cast", "reinterpret_cast", - "and", "or", "not", "xor", "bitor", "bitand", "compl", -} - -# Punctuation characters that, when they are the last non-whitespace -# char before `name(args)` followed by `;` or `=`, indicate an -# expression / argument-list context (i.e. a call, not a declaration). -# `*` and `&` are intentionally NOT in this set because they also act -# as valid pointer/reference type qualifiers in a return type. The -# double-character operator forms (`&&`, `||`, `**`, etc.) are handled -# by the two-character check inside _is_declaration_prefix. -# Examples: `foo(1, bar(x), 3)` -> prefix ends with `,` -# `if (cond && ok(x))` -> prefix ends with `&&` -# `v.push_back(fn())` -> prefix ends with `.` -DECL_PREFIX_BAD_TAIL = set("=+-/%|^~!<>?([{,.;:") - -# Last-identifier regex: extracts the trailing identifier word from a -# prefix string (strips trailing punctuation and whitespace first). -_LAST_IDENT_RE = re.compile(r"([A-Za-z_]\w*)\s*$") - - -def _is_declaration_prefix(prefix_stripped: str) -> bool: - """Return True when `prefix_stripped` (the right-stripped text between - the start of the line and the candidate `name(args)` opener) is - consistent with a function declaration/definition context rather - than a call-site expression. - - A prefix is *not* a declaration prefix when: - * it is empty; - * its last non-whitespace character is punctuation that typically - appears inside expressions / argument lists (see - DECL_PREFIX_BAD_TAIL), or it is the double-character operator - form of `&&`, `||`, `**`, `->` that commonly appears inside - expressions; - * its trailing identifier token is a statement/expression - keyword (see STATEMENT_CONTEXT_KEYWORDS) such as `return`, - `throw`, `if`, `sizeof`, `static_cast`, etc. - - Constructors/destructors that open with `{` instead of `;` or `=` - are handled separately by the caller and do not go through this - check. - """ - if not prefix_stripped: - return False - tail_char = prefix_stripped[-1] - # Two-character expression operators that include `*` or `&` (which - # are not declared in the single-char bad-tail set because they are - # also valid type qualifiers). - if len(prefix_stripped) >= 2: - prev_char = prefix_stripped[-2] - double = prev_char + tail_char - if double in {"&&", "||", "**", "*&", "&*", "->"}: - return False - if tail_char in DECL_PREFIX_BAD_TAIL: - return False - m = _LAST_IDENT_RE.search(prefix_stripped) - if m and m.group(1) in STATEMENT_CONTEXT_KEYWORDS: - return False - return True - - -FUNC_NAME_RE = re.compile(r"\b([A-Za-z_]\w*)\s*\(") -QUALIFIER_RE = re.compile(r"\b(?:const|override|final|noexcept)\b") -CYCLO_KEYWORDS_RE = re.compile(r"\b(?:if|for|while|switch|case)\b|&&|\|\|") - -# Post-C++11 features with low false-positive rates. -# Each entry is (label, compiled_regex) — the label appears in the finding. -POST_CPP11_PATTERNS: List[Tuple[str, "re.Pattern[str]"]] = [ - # C++14 - ( - "std::make_unique", - re.compile(r"\bstd::make_unique\s*<"), - ), - ( - "digit-separator in numeric literal (C++14)", - re.compile(r"(?]=?|[.<>=])" - r"\s*\.\.\.\s*\)" - r"|\(\s*\.\.\.\s*(?:\+\+|\&\&|\|\||[+\-*/%^&|<>]=?|[.<>=])" - r"\s*[A-Za-z_]\w*\s*\)" - r"|\(\s*[A-Za-z_]\w*\s*(?:\+\+|\&\&|\|\||[+\-*/%^&|<>]=?|[.<>=])" - r"\s*\.\.\.\s*(?:\+\+|\&\&|\|\||[+\-*/%^&|<>]=?|[.<>=])" - r"\s*[^,)]+\s*\)" - ), - ), - ( - "std::optional (C++17)", - re.compile(r"\bstd::optional\b"), - ), - ( - "std::variant (C++17)", - re.compile(r"\bstd::variant\b"), - ), - ( - "std::any (C++17)", - re.compile(r"\bstd::any\b"), - ), - ( - "[[nodiscard]] (C++17)", - re.compile(r"\[\[nodiscard\b"), - ), - ( - "[[maybe_unused]] (C++17)", - re.compile(r"\[\[maybe_unused\b"), - ), - # C++20 - ( - "concept (C++20)", - re.compile(r"\bconcept\b"), - ), - ( - "requires (C++20)", - re.compile(r"\brequires\b"), - ), - ( - "consteval (C++20)", - re.compile(r"\bconsteval\b"), - ), - ( - "constinit (C++20)", - re.compile(r"\bconstinit\b"), - ), - ( - "coroutine co_await / co_yield / co_return (C++20)", - re.compile(r"\bco_(?:await|yield|return)\b"), - ), - ( - "std::span (C++20)", - re.compile(r"\bstd::span\b"), - ), - ( - "std::ranges (C++20)", - re.compile(r"\bstd::ranges::"), - ), - ( - "std::format (C++20)", - re.compile(r"\bstd::format\b"), - ), - # C++23 - ( - "std::expected (C++23)", - re.compile(r"\bstd::expected\b"), - ), - ( - "std::print / std::println (C++23)", - re.compile(r"\bstd::print(?:ln)?\s*\("), - ), -] - - -@dataclass -class Finding: - rule: str - line: Optional[int] - reason: str - deduction: int - - -@dataclass -class FileReport: - path: str - score: int - findings: List[Finding] = field(default_factory=list) - real_score: int = 100 - - def to_dict(self) -> dict: - return { - "path": self.path, - "score": self.score, - "real_score": self.real_score, - "findings": [ - { - "rule": f.rule, - "line": f.line, - "reason": f.reason, - "deduction": f.deduction, - } - for f in self.findings - ], - } - - -def discover_files(paths: Sequence[str]) -> List[Path]: - """Walk input paths and return source files, skipping generated dirs.""" - result: List[Path] = [] - for p in paths: - path = Path(p) - if path.is_file(): - if path.suffix in SOURCE_EXTENSIONS: - result.append(path) - elif path.is_dir(): - for sub in path.rglob("*"): - if not sub.is_file(): - continue - if any(part in SKIP_DIRS for part in sub.parts): - continue - if sub.suffix in SOURCE_EXTENSIONS: - result.append(sub) - return result - - -def _is_digit_separator(content: str, quote_pos: int) -> bool: - """Return True if the single-quote at `quote_pos` is a C++14 digit - separator (e.g. the middle `'` in `1'000'000`). - - Heuristic: a digit separator must be preceded and followed by a - character that can belong to a numeric literal — digits, - hexadecimal letters (a–f, A–F) for hex literals, or the base/type - suffix letters (uUlLbB) that typically attach to numbers after the - body. Floating-point '.' can also appear on either side (e.g. - `1'000.5` or `1.000'5`). Any other previous/next char means the - quote starts a character literal. - """ - n = len(content) - if quote_pos <= 0 or quote_pos >= n - 1: - return False - prev = content[quote_pos - 1] - nxt = content[quote_pos + 1] - numeric_chars = set("0123456789abcdefABCDEFuUlLbBxXoO.") - return prev in numeric_chars and nxt in numeric_chars - - -def strip_comments(content: str) -> str: - """Return content with comments replaced by spaces, preserving line numbers. - - Handles // line comments and /* */ block comments. String literals are - respected so that '/' inside strings is not mistaken for a comment. - - C++14 digit separators such as `1'000'000` are NOT treated as character - literals: the parser inspects the characters on both sides of a `'` and - only enters character-literal mode when the quote is genuinely opening - a char literal like `'x'` or `'\\n'`. - """ - out = [] - i = 0 - n = len(content) - in_string = False - in_char = False - while i < n: - c = content[i] - if in_string: - out.append(c) - if c == "\\" and i + 1 < n: - out.append(content[i + 1]) - i += 2 - continue - if c == '"': - in_string = False - i += 1 - continue - if in_char: - out.append(c) - if c == "\\" and i + 1 < n: - out.append(content[i + 1]) - i += 2 - continue - if c == "'": - in_char = False - i += 1 - continue - if c == '"': - in_string = True - out.append(c) - i += 1 - continue - if c == "'": - if _is_digit_separator(content, i): - out.append(c) - i += 1 - continue - in_char = True - out.append(c) - i += 1 - continue - if c == "/" and i + 1 < n: - if content[i + 1] == "/": - # line comment until newline - j = content.find("\n", i) - if j < 0: - j = n - out.append(" " * (j - i)) - i = j - continue - if content[i + 1] == "*": - # block comment until */ - j = content.find("*/", i + 2) - if j < 0: - j = n - else: - j += 2 - block = content[i:j] - # preserve newlines - out.append(re.sub(r"[^\n]", " ", block)) - i = j - continue - out.append(c) - i += 1 - return "".join(out) - - -def strip_strings(content: str) -> str: - """Return content with string and character literals erased to spaces, - preserving line numbers. - - The result still has the same number of characters and lines, but no - letter/word survives inside "..." or '...'. This prevents false matches - on keywords that happen to appear inside string literals. - - C++14 digit separators such as `1'000'000` are left alone: the parser - inspects the characters on both sides of a `'` and only enters - character-literal mode when the quote genuinely opens a char literal - like `'x'` or `'\\n'`. - """ - out = [] - i = 0 - n = len(content) - in_string = False - in_char = False - while i < n: - c = content[i] - if in_string: - if c == "\\" and i + 1 < n: - # keep the escape sequence length but blank it out - out.append(" ") - if content[i + 1] == "\n": - out.append("\n") - else: - out.append(" ") - i += 2 - continue - if c == '"': - in_string = False - out.append('"') # keep the boundary marker (safest for line numbers) - i += 1 - continue - if c == "\n": - out.append("\n") - else: - out.append(" ") - i += 1 - continue - if in_char: - if c == "\\" and i + 1 < n: - out.append(" ") - if content[i + 1] == "\n": - out.append("\n") - else: - out.append(" ") - i += 2 - continue - if c == "'": - in_char = False - out.append("'") - i += 1 - continue - if c == "\n": - out.append("\n") - else: - out.append(" ") - i += 1 - continue - if c == '"': - in_string = True - out.append('"') - i += 1 - continue - if c == "'": - if _is_digit_separator(content, i): - out.append(c) - i += 1 - continue - in_char = True - out.append("'") - i += 1 - continue - out.append(c) - i += 1 - return "".join(out) - - -def extract_comment_blocks(content: str) -> List[List[str]]: - """Extract normalised comment lines, grouped by comment block. - - Returns a list of blocks; each block is a list of normalised comment - lines (comment markers stripped, whitespace collapsed, empty lines - dropped). A 'block' is a maximal run of `///` lines, or one `/* ... */` - region. String literals are respected so that `//` inside a string is - not treated as a comment opener. - """ - blocks: List[List[str]] = [] - current: List[str] = [] - i = 0 - n = len(content) - in_string = False - in_char = False - - def flush() -> None: - nonlocal current - if current: - blocks.append(current) - current = [] - - while i < n: - c = content[i] - if in_string: - if c == "\\" and i + 1 < n: - i += 2 - continue - if c == '"': - in_string = False - i += 1 - continue - if in_char: - if c == "\\" and i + 1 < n: - i += 2 - continue - if c == "'": - in_char = False - i += 1 - continue - if c == '"': - in_string = True - i += 1 - continue - if c == "'": - if _is_digit_separator(content, i): - i += 1 - continue - in_char = True - i += 1 - continue - if c == "/" and i + 1 < n and content[i + 1] == "/": - # line comment: collect the text after // (or /// for doxygen) - j = content.find("\n", i) - if j < 0: - j = n - text = content[i + 2:j].lstrip("/ \t").strip() - if text: - current.append(" ".join(text.split())) - i = j - continue - if c == "/" and i + 1 < n and content[i + 1] == "*": - # block comment: collect until */ - j = content.find("*/", i + 2) - if j < 0: - j = n - 2 - body = content[i + 2:j] - block: List[str] = [] - for ln in body.split("\n"): - # strip leading comment markers: '*', '/', whitespace, and - # any leftover '///' from doxygen-style lines - ln = re.sub(r"^[\s*/]+", "", ln).strip() - if ln: - block.append(" ".join(ln.split())) - if block: - flush() - blocks.append(block) - i = j + 2 - continue - if c == "\n": - # a blank/non-comment line ends a /// run; check next non-empty - # cheaply by flushing on newline only when current is non-empty - # and the following line does not start with optional ws + // - rest = content[i + 1:i + 200] - m = re.match(r"\s*//", rest) - if not m: - flush() - i += 1 - continue - if not c.isspace(): - flush() - i += 1 - flush() - return blocks - - -def find_class_blocks(code: str) -> List[Tuple[int, int, str, str]]: - """Find top-level class/struct blocks. Returns list of - (start_line_1indexed, end_line_1indexed, kind, name). - - Walks brace matching starting from each `class X {` / `struct X {` opener. - Excludes `template ` type parameters and `enum class` scoped enums. - """ - blocks: List[Tuple[int, int, str, str]] = [] - i = 0 - n = len(code) - while i < n: - m = CLASS_OPEN_RE.search(code, i) - if not m: - break - # reject template type parameters: `template ` or `<..., class T, ...>` - # look at non-whitespace chars before the match - pre_start = m.start() - 1 - while pre_start >= 0 and code[pre_start] in " \t\n\r": - pre_start -= 1 - if pre_start >= 0 and code[pre_start] in "<,": - i = m.end() - continue - # reject `enum class` / `enum struct` scoped enumerations - # look backwards for `enum` keyword (with optional whitespace) - enum_check_pos = pre_start - while enum_check_pos >= 0 and code[enum_check_pos] in " \t\n\r": - enum_check_pos -= 1 - if ( - enum_check_pos >= 3 - and code[enum_check_pos - 3:enum_check_pos + 1].lower() == "enum" - and (enum_check_pos == 3 or not code[enum_check_pos - 4].isalnum()) - ): - i = m.end() - continue - # find the opening brace after the class/struct header - # allow inheritance clauses: class X : public Y { ... } - brace_pos = code.find("{", m.end()) - if brace_pos < 0: - break - # reject if there's a `;` before the brace (forward declaration) - if ";" in code[m.end():brace_pos]: - i = m.end() - continue - depth = 1 - j = brace_pos + 1 - in_string = False - in_char = False - while j < n and depth > 0: - c = code[j] - if in_string: - if c == "\\": - j += 2 - continue - if c == '"': - in_string = False - j += 1 - continue - if in_char: - if c == "\\": - j += 2 - continue - if c == "'": - in_char = False - j += 1 - continue - if c == '"': - in_string = True - j += 1 - continue - if c == "'": - in_char = True - j += 1 - continue - if c == "{": - depth += 1 - elif c == "}": - depth -= 1 - if depth == 0: - break - j += 1 - if depth == 0: - start_line = code[:m.start()].count("\n") + 1 - end_line = code[:j].count("\n") + 1 - blocks.append((start_line, end_line, m.group(1), m.group(2))) - i = j + 1 - else: - i = m.end() - return blocks - - -def detect_access_at_line(class_lines: List[str], target_idx: int, default_access: str) -> str: - """Determine the active access specifier for line `target_idx` inside class.""" - access = default_access - for k in range(target_idx + 1): - m = ACCESS_RE.match(class_lines[k]) - if m: - access = m.group(1) - return access - - -def is_public_member_var(line: str) -> bool: - """Heuristic: does this stripped code line look like a public member variable - declaration (not a function, not a typedef, not a nested class)?""" - s = line.strip() - if not s.endswith(";"): - return False - if "(" in s or ")" in s: - return False - if "{" in s or "}" in s: - return False - if "=" in s and "(" in s: - return False - bad_prefixes = ( - "public:", "private:", "protected:", - "typedef", "using", "static_assert", - "class", "struct", "friend", - "//", "/*", "*", - "return", "if ", "for ", "while ", "switch ", "case ", - "template", - ) - if s.startswith(bad_prefixes): - return False - # skip macro-like lines (all caps with parens already excluded above) - # require at least one identifier character - if not re.search(r"[A-Za-z_]", s): - return False - return True - - -def is_static_member_var(line: str) -> bool: - """Heuristic: does this stripped code line look like a static member - variable declaration inside a class/struct body? - - Targets `static ;` and `static = ...;` - while excluding static member functions, static_assert, and - static_cast (the latter two carry parentheses). - """ - s = line.strip() - if not s.endswith(";"): - return False - if not s.startswith("static "): - return False - if s.startswith("static_assert"): - return False - if "(" in s or ")" in s: - return False - if "{" in s or "}" in s: - return False - return True - - -def _match_var_decl(stripped_line: str) -> Optional[str]: - """If line looks like a variable declaration `type name;` or - `type name = ...;`, return the variable name; else None. - - Excludes pure assignments (e.g. `counter = counter + 1;`) which lack a - leading type token. This is a best-effort heuristic, not a full parser. - """ - s = stripped_line.strip() - if not s or s.startswith(BLOCK_KEYWORD_PREFIXES): - return None - m = DECL_RE.match(s) - return m.group(1) if m else None - - -# Keywords that, when they appear on a class-body line that does not yet -# end with ';', mark the start of a declaration that *may* span several -# source lines. Following lines (up to the terminating ';') should not be -# mistaken for individual member-variable declarations. This catches -# multi-line `using`/`typedef`/`template`/namespace-qualified type aliases -# such as: -# using value_type -# = std::vector; -# Without the lookahead, the second line is inspected in isolation and -# reads as `= std::vector;` — a bogus public-static member hit. -_MULTILINE_DECL_STARTERS = ( - # Both the space-separated form (`using value_type = ...`) and the - # bare-keyword-alone form (`typedef` at the end of a line with a - # line-break before the type) are valid. Using tuples of both - # spellings catches multi-line aliases regardless of where the - # author wraps the line. - "using", "typedef", "template", "typename", - "namespace", "extern", "friend", -) - - -def _mark_continued_decl_lines(class_lines: List[str]) -> "set": - """Return the set of 0-based line indices inside a class body that - belong to a multi-line declaration started by a previous line (and - therefore must not be treated as standalone member declarations). - - A line is considered a "continuation line" when: - * there exists a strictly earlier non-empty line at the same or - smaller brace-nesting depth that begins with a declaration - starter keyword (`using`, `typedef`, `template`, `typename`, - `namespace`, `extern`, `friend`); - * the earlier starter line does NOT terminate with ';' (i.e. the - declaration has not been closed yet); - * no ';' has been seen on any intervening line at the starter's - brace-nesting depth since the starter was opened. - - The heuristic is deliberately conservative: lines are only marked as - continuations inside the same brace-depth run so that real member - declarations following a `using` block are never incorrectly - suppressed. - """ - marked: set = set() - depth = 0 - # stack of (depth_at_starter, starter_line_idx) for currently-open - # multi-line declaration starters. Multiple starters may coexist at - # different brace-nesting levels; a semicolon at depth D closes all - # starters that were opened at exactly D (conservative, but this - # matches how class bodies declare one item per terminating ';'). - open_starters: List[Tuple[int, int]] = [] - for idx, line in enumerate(class_lines): - prev_depth = depth - stripped_l = line.strip() - # (1) Mark as continuation BEFORE processing any ';' on this line: - # the terminating-semicolon line is still part of the multi-line - # declaration and must be suppressed in the public/static-member - # heuristics. The starter line itself is never marked; every - # later line that runs while any starter is open is marked. - if open_starters and any(idx > sidx for _, sidx in open_starters): - marked.add(idx) - # (2) Now close starters at prev_depth if the current line - # contains a ';' at that depth. Closure happens AFTER the - # marking step so the line that carries the terminating ';' is - # still counted as the final continuation of the declaration. - if stripped_l and ";" in stripped_l: - closed_depths = {d for d, _ in open_starters if d == prev_depth} - if closed_depths: - open_starters = [ - (d, sidx) for (d, sidx) in open_starters - if d not in closed_depths - ] - # (3) Advance brace-depth for the current line (in/out of - # functions, nested classes, etc.). Starter-opening happens - # after depth tracking so the starter is recorded at its - # prev_depth, which is the same depth at which the terminating - # ';' will be processed. - depth += line.count("{") - line.count("}") - # (4) Lines that begin with declaration-starter keywords but do - # NOT contain ';' will (in a well-formed class body) continue - # across at least one more line before reaching the terminating - # ';'. Record them so subsequent lines can be marked. - if prev_depth >= 1 and stripped_l and ";" not in stripped_l \ - and stripped_l.startswith(_MULTILINE_DECL_STARTERS): - open_starters.append((prev_depth, idx)) - return marked - - -def _skip_trailing_return_type(s: str, start: int) -> int: - """Advance past `-> ReturnType` starting at position `start`. - - `start` must point at the `-` in `->`. Scans over the return type by - tracking bracket nesting; stops and returns the position of the first - post-type character. Recognises two stop conditions at depth 0: - - (a) current char is a direct terminator (`;`, `{`, `=`) or a - qualifier keyword starts here — stop on the spot so the caller - consumes it on the next loop iteration. - (b) current char is whitespace followed by a terminator or qualifier - keyword — stop at the whitespace position so the caller's own - whitespace-skip advances past it cleanly. - - `:` is NOT a terminator because qualified return types like - `std::vector` contain `::` scope-resolution colons, and in any - case a trailing-return-type function cannot also be a constructor - (which is the only case that introduces a member initializer list). - - Caller re-runs qualifier / suffix scanning after this helper returns. - """ - i = start + 2 # skip past `->` - n = len(s) - # skip whitespace between `->` and the type - while i < n and s[i] in " \t\n": - i += 1 - depth = 0 - while i < n: - c = s[i] - if c in "([{<": - depth += 1 - elif c in ")]}>": - depth = max(0, depth - 1) - if depth == 0: - # (a) terminator/qualifier attached directly to the type, e.g. `int;` - if c in ";{=": - break - if QUALIFIER_RE.match(s, i): - break - # (b) terminator/qualifier after whitespace, e.g. `int const` / `int ;` - if c in " \t\n": - next_word_start = i + 1 - while next_word_start < n and s[next_word_start] in " \t\n": - next_word_start += 1 - if next_word_start >= n: - break - nc = s[next_word_start] - if nc in ";{=": - break - if QUALIFIER_RE.match(s, next_word_start): - break - i += 1 - return i - - -def _skip_member_initializer_list(s: str, start: int) -> int: - """Advance past `: member(args), member{args}, ...` constructor initializer - list starting at position `start`. - - `start` must point at the `:` that introduces the list (and be preceded by - `)` — caller ensures this is not a `::` scope-resolution colon). Scans - commas between members while respecting nesting; returns the position of - the opening `{` (or `;`, `=`, or end-of-string if body-less). - - Heuristic: at depth 0, `{` can either open a brace-initializer for a - member (e.g. `a{1,2}`) or open the function body itself. The function - body `{` is distinguished by looking at the previous non-whitespace - character: if it is `)` or `}` (a previous initializer ended normally) - or `,` / `:` (list-level punctuation), then `{` belongs to the function - body and scanning stops. Otherwise the `{` is consumed as a member - brace-initializer (depth increases normally). - """ - i = start + 1 # skip past `:` - n = len(s) - depth = 0 - while i < n: - c = s[i] - # Decide `{` at depth 0 before bumping depth. - if depth == 0 and c == "{": - prev = i - 1 - while prev >= 0 and s[prev] in " \t\n": - prev -= 1 - prev_c = s[prev] if prev >= 0 else "" - # `{` after full initializer / separator = function body opener - if prev_c in ")}:,": - break - if c in "([{<": - depth += 1 - elif c in ")]}>": - depth = max(0, depth - 1) - if depth == 0 and c in ";=": - break - i += 1 - return i - - -def count_function_params(params_str: str) -> int: - """Count top-level parameters by counting commas at bracket depth 0. - - Tracks (), [] {} and <> as nesting (so commas inside std::map or - function-pointer params are not counted as param separators). - """ - s = params_str.strip() - if not s or s == "void": - return 0 - depth = 0 - count = 1 - for c in s: - if c in "([{<": - depth += 1 - elif c in ")]}>": - depth = max(0, depth - 1) - elif c == "," and depth == 0: - count += 1 - return count - - -def find_long_function_signatures( - content: str, threshold: int -) -> List[Tuple[int, str, int]]: - """Find function declarations/definitions whose parameter count exceeds threshold. - - Returns list of (line_no, function_name, param_count). Best-effort: - requires either a return-type prefix before the function name (for `;` - and `=` endings) or a `{` ending (for constructors/destructors which - have no return type). - """ - stripped = strip_comments(content) - n = len(stripped) - findings: List[Tuple[int, str, int]] = [] - - i = 0 - while i < n: - m = FUNC_NAME_RE.search(stripped, i) - if not m: - break - name = m.group(1) - if name in NON_FUNCTION_KEYWORDS: - i = m.end() - continue - - # inspect the prefix before `name(` to reject calls/lambdas/macros - line_start = stripped.rfind("\n", 0, m.start()) + 1 - prefix = stripped[line_start:m.start()] - prefix_stripped = prefix.rstrip() - prefix_lstripped = prefix.lstrip() - - # skip preprocessor lines - if prefix_lstripped.startswith("#"): - i = m.end() - continue - # skip lambdas: prefix ends with `]` - if prefix_stripped.endswith("]"): - i = m.end() - continue - # skip method calls: prefix ends with `.` or `->` - if prefix_stripped.endswith(".") or prefix_stripped.endswith("->"): - i = m.end() - continue - # skip assignment-result calls: prefix ends with `=` (but not `==`) - if prefix_stripped.endswith("=") and not prefix_stripped.endswith("=="): - i = m.end() - continue - # skip function-pointer typedefs - if "typedef" in prefix_stripped: - i = m.end() - continue - - # match parens to find params string - paren_open = m.end() - 1 - depth = 1 - j = paren_open + 1 - while j < n and depth > 0: - c = stripped[j] - if c == "(": - depth += 1 - elif c == ")": - depth -= 1 - if depth == 0: - break - j += 1 - if depth != 0: - i = m.end() - continue - params_str = stripped[paren_open + 1:j] - - # after `)`: optional qualifiers (const/override/final/noexcept), - # optional trailing-return type (`-> ReturnType`), optional - # constructor member initializer list (`: a(x), b{y}`), then ; { or = - pos = j + 1 - while pos < n and stripped[pos] in " \t\n": - pos += 1 - while True: - # consume qualifiers - mq = QUALIFIER_RE.match(stripped, pos) - if mq: - pos = mq.end() - while pos < n and stripped[pos] in " \t\n": - pos += 1 - continue - # consume trailing return type: `-> ReturnType` - if ( - pos + 1 < n - and stripped[pos] == "-" - and stripped[pos + 1] == ">" - ): - pos = _skip_trailing_return_type(stripped, pos) - while pos < n and stripped[pos] in " \t\n": - pos += 1 - continue - # consume constructor member initializer list: `: members...` - # but avoid scope-resolution `::` — the prior non-whitespace - # char must be `)` (the closing param paren we just matched) - if pos < n and stripped[pos] == ":": - pre_colon = pos - 1 - while pre_colon >= 0 and stripped[pre_colon] in " \t\n": - pre_colon -= 1 - if pre_colon == j: - pos = _skip_member_initializer_list(stripped, pos) - while pos < n and stripped[pos] in " \t\n": - pos += 1 - continue - break - if pos >= n or stripped[pos] not in ";{=": - i = j + 1 - continue - - # For `;` and `=`, the preceding prefix must look like a genuine - # declaration context (return type + optional specifiers) rather - # than the tail of an expression / argument list. - # - `int f(int);` prefix `int` -> declaration ✓ - # - `return f(1, 2, 3, ...);` prefix `return` -> call ✗ - # - `foo(1, bar(x), 3);` prefix `foo(1, ` -> call ✗ (ends with `,(`) - # - Constructor/destructor bodies terminate with `{` and bypass - # this check entirely. - if stripped[pos] in ";=" and not _is_declaration_prefix(prefix_stripped): - i = j + 1 - continue - - param_count = count_function_params(params_str) - if param_count > threshold: - line_no = stripped[:paren_open].count("\n") + 1 - findings.append((line_no, name, param_count)) - - i = j + 1 - - return findings - - -def find_function_bodies(content: str) -> List[Tuple[int, str, int, int]]: - """Find function definitions (with body), not just declarations. - - Returns list of (signature_line_no, function_name, body_start_pos, - body_end_pos) where positions are absolute offsets in stripped content. - Reuses the prefix/reject logic from find_long_function_signatures. - """ - stripped = strip_comments(content) - n = len(stripped) - bodies: List[Tuple[int, str, int, int]] = [] - - i = 0 - while i < n: - m = FUNC_NAME_RE.search(stripped, i) - if not m: - break - name = m.group(1) - if name in NON_FUNCTION_KEYWORDS: - i = m.end() - continue - - line_start = stripped.rfind("\n", 0, m.start()) + 1 - prefix = stripped[line_start:m.start()] - prefix_stripped = prefix.rstrip() - prefix_lstripped = prefix.lstrip() - - if prefix_lstripped.startswith("#"): - i = m.end() - continue - if prefix_stripped.endswith("]"): - i = m.end() - continue - if prefix_stripped.endswith(".") or prefix_stripped.endswith("->"): - i = m.end() - continue - if prefix_stripped.endswith("=") and not prefix_stripped.endswith("=="): - i = m.end() - continue - if "typedef" in prefix_stripped: - i = m.end() - continue - - paren_open = m.end() - 1 - depth = 1 - j = paren_open + 1 - while j < n and depth > 0: - c = stripped[j] - if c == "(": - depth += 1 - elif c == ")": - depth -= 1 - if depth == 0: - break - j += 1 - if depth != 0: - i = m.end() - continue - - # find what comes after `)`: skip whitespace, qualifiers, optional - # trailing-return type (`-> ReturnType`), and optional constructor - # member initializer list (`: a(x), b{y}`). - pos = j + 1 - while pos < n and stripped[pos] in " \t\n": - pos += 1 - while True: - # consume qualifiers - mq = QUALIFIER_RE.match(stripped, pos) - if mq: - pos = mq.end() - while pos < n and stripped[pos] in " \t\n": - pos += 1 - continue - # consume trailing return type: `-> ReturnType` - if ( - pos + 1 < n - and stripped[pos] == "-" - and stripped[pos + 1] == ">" - ): - pos = _skip_trailing_return_type(stripped, pos) - while pos < n and stripped[pos] in " \t\n": - pos += 1 - continue - # consume constructor member initializer list: `: members...` - # but avoid scope-resolution `::` — the prior non-whitespace - # char must be `)` (the closing param paren we just matched) - if pos < n and stripped[pos] == ":": - pre_colon = pos - 1 - while pre_colon >= 0 and stripped[pre_colon] in " \t\n": - pre_colon -= 1 - if pre_colon == j: - pos = _skip_member_initializer_list(stripped, pos) - while pos < n and stripped[pos] in " \t\n": - pos += 1 - continue - break - - # we need a `{` body (not `;` declaration, not `= 0` pure virtual) - if pos >= n or stripped[pos] != "{": - i = j + 1 - continue - - # match braces to find body end - body_open = pos - depth = 1 - body_close = body_open + 1 - while body_close < n and depth > 0: - c = stripped[body_close] - if c == "{": - depth += 1 - elif c == "}": - depth -= 1 - if depth == 0: - break - body_close += 1 - if depth != 0: - i = j + 1 - continue - - sig_line_no = stripped[:paren_open].count("\n") + 1 - bodies.append((sig_line_no, name, body_open, body_close)) - i = body_close + 1 - - return bodies - - -def find_high_complexity_functions( - content: str, threshold: int -) -> List[Tuple[int, str, int]]: - """Find functions whose cyclomatic complexity exceeds threshold. - - Cyclomatic complexity counts: if, for, while, switch, case, &&, ||. - Returns list of (line_no, function_name, complexity). - - String and character literals are blanked before the regex runs so - that control-flow words appearing inside messages (e.g. - `const char* msg = "if not ok return";`) are not counted as real - control flow statements. - """ - stripped = strip_strings(strip_comments(content)) - findings: List[Tuple[int, str, int]] = [] - for sig_line, name, body_open, body_close in find_function_bodies(content): - body = stripped[body_open + 1:body_close] - complexity = len(CYCLO_KEYWORDS_RE.findall(body)) - if complexity > threshold: - findings.append((sig_line, name, complexity)) - return findings - - -def find_post_cpp11_features( - content: str, -) -> List[Tuple[int, str]]: - """Find the first occurrence of each post-C++11 feature. - - Uses the patterns in POST_CPP11_PATTERNS against content with both - comments AND string literals blanked out. String blanking prevents - false matches on keywords embedded in user-facing messages (e.g. - `"X requires Y"` in a WARNING_QUIT call). - - Returns a list of (line_no, feature_label) — one entry per distinct - detected feature so the finding message is informative (the deduction - is one-shot -80 per file regardless of how many features hit). - """ - stripped = strip_strings(strip_comments(content)) - hits: List[Tuple[int, str]] = [] - for label, regex in POST_CPP11_PATTERNS: - m = regex.search(stripped) - if m: - line_no = stripped[:m.start()].count("\n") + 1 - hits.append((line_no, label)) - return hits - - -def analyze_class_blocks( - content: str, -) -> Tuple[List[Finding], List[Finding], List[Finding], List[Finding]]: - """Analyze class/struct blocks for public member variables, static member - variables, long member functions, and member/local name conflicts. - - Returns (public_member_findings, static_member_findings, - long_function_findings, name_conflict_findings). - """ - pub_findings: List[Finding] = [] - static_findings: List[Finding] = [] - long_func_findings: List[Finding] = [] - conflict_findings: List[Finding] = [] - - stripped = strip_comments(content) - lines = stripped.split("\n") - - for start_line, end_line, kind, name in find_class_blocks(stripped): - default_access = "private" if kind == "class" else "public" - body_start = start_line - 1 - body_end = end_line - 1 - class_lines = lines[body_start:body_end + 1] - - # pre-compute access per line (for public member variable rule) - access_per_line: List[str] = [] - cur_access = default_access - for k in range(len(class_lines)): - m = ACCESS_RE.match(class_lines[k]) - if m: - cur_access = m.group(1) - access_per_line.append(cur_access) - - # pass 0: identify lines belonging to multi-line declarations that - # start with keywords such as `using`, `typedef`, `template`, etc. - # Continuation lines (e.g. the "= std::vector;" half of a - # two-line `using value_type = ...;` alias) would otherwise be - # treated as standalone member declarations when the per-line - # heuristics run below. - continued_idxs = _mark_continued_decl_lines(class_lines) - - # pass 1: collect all member variable names (any access section). - # depth starts at 0; the class header line brings it to 1. Only lines - # that begin AND end at depth 1 are class-body member declarations - # (lines that open a function body bring depth from 1 to 2). - member_var_names: set = set() - depth = 0 - for idx, line in enumerate(class_lines): - prev_depth = depth - depth += line.count("{") - line.count("}") - if prev_depth == 1 and depth == 1 and idx not in continued_idxs: - var_name = _match_var_decl(line) - if var_name: - member_var_names.add(var_name) - - # pass 2: walk class body for findings - depth = 0 - func_start_abs = -1 - for idx in range(len(class_lines)): - line = class_lines[idx] - abs_line = body_start + idx + 1 # 1-indexed absolute line - prev_depth = depth - - # public member variable: only at depth 1 (class body, not in a - # function). Lines flagged as continuations of an earlier - # multi-line declaration are skipped regardless of how they - # look in isolation — e.g. ` = std::vector;` after a - # two-line `using value_type` alias must not count as a member. - # Only `class` bodies are penalised: in C++ a `struct` has - # public access by default and public data members are a - # legitimate, intended usage (POD aggregate, value types, - # mixin tags). Treating them as a code-quality issue would - # penalise perfectly idiomatic C++. - if prev_depth == 1 and access_per_line[idx] == "public" \ - and kind == "class" \ - and idx not in continued_idxs: - if is_public_member_var(line): - pub_findings.append(Finding( - rule="public_member_variable", - line=abs_line, - reason=f"public member in {kind} {name}: {line.strip()}", - deduction=WEIGHTS["public_member_variable"], - )) - - # static member variable: depth 1, any access section. - # Excludes static_assert, static member functions, and - # static_cast via is_static_member_var's heuristic. Continued - # multi-line declarations are skipped for the same reason - # documented above for the public-member rule. - if prev_depth == 1 and idx not in continued_idxs \ - and is_static_member_var(line): - static_findings.append(Finding( - rule="static_member_variable", - line=abs_line, - reason=f"static member in {kind} {name}: {line.strip()}", - deduction=WEIGHTS["static_member_variable"], - )) - - # member/local name conflict: only at depth >= 2 (function body) - if prev_depth >= 2: - var_name = _match_var_decl(line) - if var_name and var_name in member_var_names: - conflict_findings.append(Finding( - rule="member_local_name_conflict", - line=abs_line, - reason=( - f"local variable '{var_name}' in {kind} {name} " - f"shadows a member variable" - ), - deduction=WEIGHTS["member_local_name_conflict"], - )) - - # apply brace counting for this line - depth = prev_depth + line.count("{") - line.count("}") - - # function body start: transition from depth 1 to >=2 - if prev_depth == 1 and depth >= 2 and func_start_abs < 0: - func_start_abs = abs_line - # function body end: transition from >=2 back to 1 - if prev_depth >= 2 and depth == 1 and func_start_abs > 0: - func_end_abs = abs_line - func_lines = func_end_abs - func_start_abs + 1 - if func_lines > FUNCTION_LENGTH_THRESHOLD: - excess = func_lines - FUNCTION_LENGTH_THRESHOLD - blocks = (excess + FUNCTION_LENGTH_STEP - 1) // FUNCTION_LENGTH_STEP - long_func_findings.append(Finding( - rule="member_function_too_long", - line=func_start_abs, - reason=( - f"member function in {kind} {name} spans {func_lines} lines " - f"(exceeds {FUNCTION_LENGTH_THRESHOLD})" - ), - deduction=blocks * WEIGHTS["member_function_too_long"], - )) - func_start_abs = -1 - - return pub_findings, static_findings, long_func_findings, conflict_findings - - -def analyze_file(path: Path) -> FileReport: - """Analyze one file and return its FileReport.""" - findings: List[Finding] = [] - name = path.name - stem = path.stem - suffix = path.suffix - - # filename rules - if len(stem) > FILENAME_LENGTH_LIMIT: - findings.append(Finding( - rule="filename_too_long", - line=None, - reason=( - f"filename '{name}' stem has {len(stem)} chars " - f"(limit {FILENAME_LENGTH_LIMIT})" - ), - deduction=WEIGHTS["filename_too_long"], - )) - if any(c.isupper() for c in stem): - findings.append(Finding( - rule="filename_uppercase", - line=None, - reason=f"filename '{name}' contains uppercase letters", - deduction=WEIGHTS["filename_uppercase"], - )) - if suffix == ".hpp": - findings.append(Finding( - rule="hpp_implementation", - line=None, - reason=".hpp implementation header prohibited (use .cpp + .h split)", - deduction=WEIGHTS["hpp_implementation"], - )) - - # read content - try: - content = path.read_text(encoding="utf-8", errors="replace") - except OSError as e: - findings.append(Finding( - rule="read_error", - line=None, - reason=f"failed to read file: {e}", - deduction=0, - )) - return FileReport(path=str(path), score=100, findings=findings) - - lines = content.split("\n") - - # file length rule (no cap: large files legitimately scale deduction) - total_lines = len(lines) - if total_lines > FILE_LENGTH_THRESHOLD: - excess = total_lines - FILE_LENGTH_THRESHOLD - blocks = (excess + FILE_LENGTH_STEP - 1) // FILE_LENGTH_STEP - findings.append(Finding( - rule="file_too_long", - line=None, - reason=( - f"file has {total_lines} lines " - f"(exceeds {FILE_LENGTH_THRESHOLD} by {excess})" - ), - deduction=blocks * WEIGHTS["file_too_long"], - )) - - # line-based rules with caps - tab_count = sum(1 for l in lines if "\t" in l) - long_line_count = sum(1 for l in lines if len(l) > LINE_LENGTH_LIMIT) - using_ns_count = sum( - 1 for l in lines if USING_NS_STD_RE.search(strip_comments(l)) - ) - chinese_count = sum(1 for l in lines if CHINESE_RE.search(l)) - - stripped_content = strip_comments(content) - # uppercase constants: strip strings too (so default-value strings like - # "ABACUS" do not trigger), and the regex excludes member accesses - # (e.g. GlobalV::MY_RANK, x.UPPER) via a negative lookbehind. - stripped_for_upper = strip_strings(stripped_content) - upper_const_count = sum( - len(UPPERCASE_CONST_RE.findall(l)) - for l in stripped_for_upper.split("\n") - ) - - # interface & dependency rules - global_dep_count = len(GLOBAL_DEPENDENCY_RE.findall(stripped_content)) - hpp_include_count = sum(1 for l in lines if HPP_INCLUDE_RE.match(l)) - friend_count = len(FRIEND_RE.findall(stripped_content)) - goto_count = len(GOTO_RE.findall(stripped_content)) - # `auto` keyword: counted on comment-and-string-stripped content so - # occurrences inside comments or string literals do not deduct points. - auto_count = len(AUTO_RE.findall(stripped_for_upper)) - - # default parameter: scan the full stripped content so multi-line - # declarations (parameter list spanning multiple lines) are detected; - # `[^();]*` in the regex already matches newlines. - default_param_count = len(DEFAULT_PARAM_RE.findall(stripped_content)) - - # unpaired new/delete (file-level): rough heuristic, cap applied below. - # All `new` expressions are counted first. `delete` expressions are the - # obvious symmetric ownership cancel, but there is a second class of - # cancellations that never produces a `delete` keyword at all: a - # C++11 (or later) smart pointer that owns the object from construction. - # Recognised ownership-transfer patterns are aggregated in OWNED_NEW_RE - # and subtracted before comparing with `delete` so the leak heuristic - # stays meaningful on modern code that prefers std::unique_ptr / - # std::shared_ptr to manual delete. Examples that now cancel: - # ptr.reset(new Foo()); - # std::unique_ptr p(new Foo(arg)); - # These cancellations deliberately do NOT touch `raw_new_count` below - # because the "raw_new_keyword" rule charges per direct `new` regardless - # of ownership — it is the stylistic counterpart to std::make_unique / - # std::make_shared, so `reset(new Foo)` and `unique_ptr(new T)` - # must still contribute to that count. - new_count = len(NEW_EXPR_RE.findall(stripped_content)) - delete_count = len(DELETE_EXPR_RE.findall(stripped_content)) - owned_new_count = len(OWNED_NEW_RE.findall(stripped_content)) - cancelled_new = delete_count + owned_new_count - unpaired_new = max(0, new_count - cancelled_new) - - # raw `new` keyword usage: each occurrence costs 1 (no cap) - raw_new_count = new_count - - def append_capped(rule: str, count: int) -> None: - if count == 0: - return - cap = CAPS.get(rule) - actual = min(count, cap) if cap else count - cap_text = f" (capped at {cap})" if cap and count > cap else "" - findings.append(Finding( - rule=rule, - line=None, - reason=f"{count} occurrence(s){cap_text}", - deduction=actual * WEIGHTS[rule], - )) - - append_capped("tab_indentation", tab_count) - append_capped("line_too_long", long_line_count) - append_capped("using_namespace_std", using_ns_count) - append_capped("chinese_comment", chinese_count) - append_capped("uppercase_constant", upper_const_count) - append_capped("default_parameter", default_param_count) - append_capped("global_dependency", global_dep_count) - append_capped("hpp_include", hpp_include_count) - append_capped("friend_keyword", friend_count) - append_capped("unpaired_new_delete", unpaired_new) - append_capped("raw_new_keyword", raw_new_count) - append_capped("goto_keyword", goto_count) - append_capped("auto_keyword", auto_count) - - # --- documentation style rules ------------------------------------- - # indented preprocessor directive (raw lines, `#` not at column 0) - indented_preproc_count = sum(1 for l in lines if INDENTED_PREPROC_RE.match(l)) - append_capped("indented_preprocessor", indented_preproc_count) - - # comment block(s) above the first include guard (headers only). One - # contiguous comment block counts once regardless of its length. - if path.suffix == ".h": - comment_above_guard = 0 - in_block = False - for l in lines: - if GUARD_RE.match(l): - break - if COMMENT_LINE_RE.match(l): - if not in_block: - comment_above_guard += 1 - in_block = True - elif l.strip(): - in_block = False - append_capped("comment_above_guard", comment_above_guard) - - # `@file xxx` name vs the actual filename - doc_file_match = DOC_FILE_RE.search(content) - if doc_file_match and doc_file_match.group(1) != path.name: - findings.append(Finding( - rule="doc_file_mismatch", - line=None, - reason=( - f"@file names '{doc_file_match.group(1)}' but the file is " - f"'{path.name}'" - ), - deduction=WEIGHTS["doc_file_mismatch"], - )) - - # duplicated comment text across comment blocks (e.g. the same formula - # repeated in the file-header and in a function doxygen block). Only - # lines of >= 40 chars are compared. Excluded from comparison: - # - doxygen command lines (@param/@return/...): repeated `@param pv` - # across overloads is normal, not redundant prose; - # - decorative separator lines (=== Part N ===): >50% non-alnum chars; - # - ALL-CAPS banner lines: visual section headers, not content. - def _is_dup_candidate(l: str) -> bool: - if len(l) < 40 or re.match(r"@\w+", l): - return False - alnum = sum(1 for ch in l if ch.isalnum()) - if alnum * 2 < len(l): - return False - if l.isupper(): - return False - return True - - comment_blocks = extract_comment_blocks(content) - if len(comment_blocks) > 1: - seen: dict = {} - duplicate_doc_lines = 0 - for block in comment_blocks: - block_lines = set(l for l in block if _is_dup_candidate(l)) - for l in block_lines: - if l in seen: - duplicate_doc_lines += 1 - else: - seen[l] = True - append_capped("duplicate_doc_block", duplicate_doc_lines) - - # too-many-parameters rule: each function's deduction is capped - # individually (per-function cap, not per-file). A file with many - # over-parameterised functions accumulates deductions across all of - # them, instead of one bad function hiding the rest. - long_param_funcs = find_long_function_signatures(content, FUNCTION_PARAM_THRESHOLD) - cap_params = CAPS.get("too_many_parameters") - for line_no, fname, pcount in long_param_funcs: - excess = pcount - FUNCTION_PARAM_THRESHOLD - raw_deduction = excess * WEIGHTS["too_many_parameters"] - if cap_params is not None and raw_deduction > cap_params: - per_deduction = cap_params - cap_text = f" (capped at {cap_params})" - else: - per_deduction = raw_deduction - cap_text = "" - findings.append(Finding( - rule="too_many_parameters", - line=line_no, - reason=( - f"function '{fname}' has {pcount} parameters " - f"(exceeds {FUNCTION_PARAM_THRESHOLD} by {excess}){cap_text}" - ), - deduction=per_deduction, - )) - - # high cyclomatic complexity rule: each function's deduction is capped - # individually (per-function cap, not per-file). - high_cyclo_funcs = find_high_complexity_functions(content, CYCLO_THRESHOLD) - cap_cyclo = CAPS.get("high_cyclomatic_complexity") - for line_no, fname, cyclo in high_cyclo_funcs: - excess = cyclo - CYCLO_THRESHOLD - raw_deduction = excess * WEIGHTS["high_cyclomatic_complexity"] - if cap_cyclo is not None and raw_deduction > cap_cyclo: - per_deduction = cap_cyclo - cap_text = f" (capped at {cap_cyclo})" - else: - per_deduction = raw_deduction - cap_text = "" - findings.append(Finding( - rule="high_cyclomatic_complexity", - line=line_no, - reason=( - f"function '{fname}' has cyclomatic complexity {cyclo} " - f"(exceeds {CYCLO_THRESHOLD} by {excess}){cap_text}" - ), - deduction=per_deduction, - )) - - # post-C++11 feature rule: base deduction for any usage + per-feature - # incremental deduction (capped). Total cap matches the previous - # one-shot -80, but a single misuse no longer swamps the whole file. - post_cpp11_hits = find_post_cpp11_features(content) - if post_cpp11_hits: - feature_list = ", ".join( - f"'{label}' (line {ln})" for ln, label in post_cpp11_hits - ) - findings.append(Finding( - rule="post_cpp11_feature", - line=post_cpp11_hits[0][0], - reason=( - f"uses post-C++11 feature(s): {feature_list} " - f"(repo baseline is C++11)" - ), - deduction=WEIGHTS["post_cpp11_feature"], - )) - num_features = len(post_cpp11_hits) - cap_features = CAPS.get("post_cpp11_per_feature", num_features) - capped = min(num_features, cap_features) - if capped > 0: - cap_text = ( - f" (capped at {cap_features})" if num_features > cap_features else "" - ) - findings.append(Finding( - rule="post_cpp11_per_feature", - line=post_cpp11_hits[0][0], - reason=( - f"{num_features} distinct post-C++11 feature(s) detected" - f"{cap_text}" - ), - deduction=capped * WEIGHTS["post_cpp11_per_feature"], - )) - - # class-based rules - ( - pub_findings, - static_findings, - long_func_findings, - conflict_findings, - ) = analyze_class_blocks(content) - findings.extend(pub_findings) - # static member findings: cap per file - if len(static_findings) > CAPS.get("static_member_variable", len(static_findings)): - cap_static = CAPS["static_member_variable"] - static_findings = static_findings[:cap_static] - findings.extend(static_findings) - findings.extend(long_func_findings) - # name conflict findings already respect cap via per-file cap on the rule - if len(conflict_findings) > CAPS.get("member_local_name_conflict", len(conflict_findings)): - cap = CAPS["member_local_name_conflict"] - conflict_findings = conflict_findings[:cap] - findings.extend(conflict_findings) - - # compute score: real_score is the unclamped value (may be negative); - # score is clamped to a minimum of 0 for display. - real_score = 100 - for f in findings: - real_score -= f.deduction - score = max(0, real_score) - - return FileReport( - path=str(path), - score=score, - findings=findings, - real_score=real_score, - ) - - -def get_module(path: str) -> str: - """Group a file path into its module for rollup purposes. - - Returns the `source/source_*` prefix when present (e.g. - `source/source_lcao`); otherwise falls back to the file's parent - directory. Backslashes are normalised to forward slashes. - """ - parts = path.replace("\\", "/").split("/") - for i, p in enumerate(parts): - if ( - p == "source" - and i + 1 < len(parts) - and parts[i + 1].startswith("source_") - ): - return f"source/{parts[i + 1]}" - return "/".join(parts[:-1]) if len(parts) > 1 else path - - -def compute_module_rollup( - reports: List[FileReport], -) -> List[Tuple[str, int, float, int]]: - """Group reports by module and return rows sorted by avg score ascending. - - Each row is (module, file_count, average_score, passing_count). - Worst module (lowest average) appears first. - """ - groups: dict = {} - for r in reports: - m = get_module(r.path) - groups.setdefault(m, []).append(r) - rows: List[Tuple[str, int, float, int]] = [] - for module, rs in groups.items(): - n = len(rs) - avg = sum(r.score for r in rs) / n if n else 0.0 - passing = sum(1 for r in rs if r.score >= PASS_THRESHOLD) - rows.append((module, n, avg, passing)) - rows.sort(key=lambda x: x[2]) - return rows - - -def render_text(reports: List[FileReport], min_score: Optional[int]) -> str: - """Render reports as plain text.""" - if not reports: - return "No files to analyze.\n" - - visible = reports if min_score is None else [r for r in reports if r.score <= min_score] - sorted_reports = sorted(visible, key=lambda r: (r.score, r.real_score)) - - out: List[str] = [] - width = 70 - out.append("=" * width) - out.append("Code Quality Score Report") - out.append("=" * width) - - rollup = compute_module_rollup(reports) - if rollup: - out.append("") - out.append( - "Per-module rollup (sorted by average score, worst first):" - ) - out.append(f" {'Module':<40} {'Files':>5} {'Avg':>6} {'Pass':>12}") - out.append("-" * width) - for module, n, avg, passing in rollup: - module_disp = module if len(module) <= 40 else module[-37:] + "..." - pass_str = f"{passing}/{n}" - out.append( - f" {module_disp:<40} {n:>5} {avg:>6.1f} {pass_str:>12}" - ) - out.append("") - - out.append(f"{'Score':>6} {'File'}") - out.append("-" * width) - for r in sorted_reports: - suffix = f" (real_score={r.real_score})" if r.real_score < 0 else "" - out.append(f"{r.score:>6} {r.path}{suffix}") - out.append("") - - for r in sorted_reports: - if not r.findings: - continue - out.append("-" * width) - out.append(f"File: {r.path} (score: {r.score})") - out.append("-" * width) - for f in r.findings: - loc = f"line {f.line}" if f.line else "file" - out.append(f" [{f.rule}] {loc}: {f.reason} (-{f.deduction})") - out.append("") - - total = len(reports) - visible_n = len(visible) - avg = sum(r.score for r in reports) / total if total else 0.0 - passing = sum(1 for r in reports if r.score >= PASS_THRESHOLD) - out.append("=" * width) - out.append(f"Files scanned: {total}") - out.append(f"Files shown: {visible_n}") - out.append(f"Average score: {avg:.1f}") - out.append(f"Passing (>= {PASS_THRESHOLD}): {passing}/{total}") - out.append("=" * width) - return "\n".join(out) + "\n" - +from typing import Optional, Sequence -def render_json(reports: List[FileReport], min_score: Optional[int]) -> str: - visible = reports if min_score is None else [r for r in reports if r.score <= min_score] - rollup = compute_module_rollup(reports) - return json.dumps({ - "summary": { - "total_scanned": len(reports), - "total_shown": len(visible), - "average_score": ( - sum(r.score for r in reports) / len(reports) if reports else 0.0 - ), - "passing": sum(1 for r in reports if r.score >= PASS_THRESHOLD), - "pass_threshold": PASS_THRESHOLD, - "by_module": [ - { - "module": module, - "file_count": n, - "average_score": avg, - "passing": passing, - } - for module, n, avg, passing in rollup - ], - }, - "files": [r.to_dict() for r in sorted(visible, key=lambda r: (r.score, r.real_score))], - }, indent=2) +from cqs_analyze import analyze_file, discover_files +from cqs_config import PASS_THRESHOLD +from cqs_report import render_json, render_text def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace: diff --git a/tools/03_code_analysis/cqs_analyze.py b/tools/03_code_analysis/cqs_analyze.py new file mode 100644 index 0000000000..74ff7a44b6 --- /dev/null +++ b/tools/03_code_analysis/cqs_analyze.py @@ -0,0 +1,496 @@ +"""Per-file analysis orchestration for the ABACUS code-quality scorer.""" + +import re +from pathlib import Path +from typing import List, Sequence, Set, Tuple + +from cqs_config import ( + AUTO_RE, CAPS, CHINESE_RE, COMMENT_LINE_RE, CYCLO_THRESHOLD, + DEFAULT_PARAM_RE, DELETE_EXPR_RE, DOC_FILE_RE, FILE_LENGTH_STEP, + FILE_LENGTH_THRESHOLD, FILENAME_LENGTH_LIMIT, FRIEND_RE, + FUNCTION_PARAM_THRESHOLD, GLOBAL_DEPENDENCY_RE, GOTO_RE, GUARD_RE, + HEADER_FUNC_BODY_STEP, HEADER_FUNC_BODY_THRESHOLD, + HEADER_LENGTH_STEP, HEADER_LENGTH_THRESHOLD, HPP_INCLUDE_RE, + INDENTED_PREPROC_RE, LINE_LENGTH_LIMIT, MACRO_DEFINE_RE, NEW_EXPR_RE, + OWNED_NEW_RE, SKIP_DIRS, SOURCE_EXTENSIONS, + UPPERCASE_CONST_RE, + USING_NS_STD_RE, WEIGHTS, +) +from cqs_report import FileReport, Finding +from cqs_rules import ( + analyze_class_blocks, find_complex_call_arguments, + find_function_bodies, find_header_function_bodies, + find_high_complexity_functions, find_ifdef_in_call_args, + find_long_function_signatures, find_param_member_inits, + find_post_cpp11_features, +) +from cqs_text import extract_comment_blocks, strip_comments, strip_strings + + +def discover_files(paths: Sequence[str]) -> List[Path]: + """Walk input paths and return source files, skipping generated dirs.""" + result: List[Path] = [] + for p in paths: + path = Path(p) + if path.is_file(): + if path.suffix in SOURCE_EXTENSIONS: + result.append(path) + elif path.is_dir(): + for sub in path.rglob("*"): + if not sub.is_file(): + continue + if any(part in SKIP_DIRS for part in sub.parts): + continue + if sub.suffix in SOURCE_EXTENSIONS: + result.append(sub) + return result + + +def analyze_file(path: Path) -> FileReport: + """Analyze one file and return its FileReport.""" + findings: List[Finding] = [] + name = path.name + stem = path.stem + suffix = path.suffix + + # filename rules + if len(stem) > FILENAME_LENGTH_LIMIT: + findings.append(Finding( + rule="filename_too_long", + line=None, + reason=( + f"filename '{name}' stem has {len(stem)} chars " + f"(limit {FILENAME_LENGTH_LIMIT})" + ), + deduction=WEIGHTS["filename_too_long"], + )) + if any(c.isupper() for c in stem): + findings.append(Finding( + rule="filename_uppercase", + line=None, + reason=f"filename '{name}' contains uppercase letters", + deduction=WEIGHTS["filename_uppercase"], + )) + if suffix == ".hpp": + findings.append(Finding( + rule="hpp_implementation", + line=None, + reason=".hpp implementation header prohibited (use .cpp + .h split)", + deduction=WEIGHTS["hpp_implementation"], + )) + + # read content + try: + content = path.read_text(encoding="utf-8", errors="replace") + except OSError as e: + findings.append(Finding( + rule="read_error", + line=None, + reason=f"failed to read file: {e}", + deduction=0, + )) + return FileReport(path=str(path), score=100, findings=findings) + + lines = content.split("\n") + + # file length rule (no cap: large files legitimately scale deduction) + total_lines = len(lines) + if total_lines > FILE_LENGTH_THRESHOLD: + excess = total_lines - FILE_LENGTH_THRESHOLD + blocks = (excess + FILE_LENGTH_STEP - 1) // FILE_LENGTH_STEP + findings.append(Finding( + rule="file_too_long", + line=None, + reason=( + f"file has {total_lines} lines " + f"(exceeds {FILE_LENGTH_THRESHOLD} by {excess})" + ), + deduction=blocks * WEIGHTS["file_too_long"], + )) + + # .h header length rule (cap 5: headers should stay compact) + if path.suffix == ".h" and total_lines > HEADER_LENGTH_THRESHOLD: + header_excess = total_lines - HEADER_LENGTH_THRESHOLD + header_blocks = ( + (header_excess + HEADER_LENGTH_STEP - 1) // HEADER_LENGTH_STEP + ) + header_deduction = min( + header_blocks * WEIGHTS["header_too_long"], + CAPS["header_too_long"], + ) + findings.append(Finding( + rule="header_too_long", + line=None, + reason=( + f".h header has {total_lines} lines " + f"(exceeds {HEADER_LENGTH_THRESHOLD} by {header_excess})" + ), + deduction=header_deduction, + )) + + # non-inline, non-template function bodies in .h headers + if path.suffix == ".h": + for fn_line, fn_name, fn_lines in find_header_function_bodies( + content + ): + excess = fn_lines - HEADER_FUNC_BODY_THRESHOLD + blocks = ( + (excess + HEADER_FUNC_BODY_STEP - 1) + // HEADER_FUNC_BODY_STEP + ) + deduction = min( + blocks * WEIGHTS["header_function_body"], + CAPS["header_function_body"], + ) + findings.append(Finding( + rule="header_function_body", + line=fn_line, + reason=( + f"non-inline function `{fn_name}` defined in .h header " + f"spans {fn_lines} lines " + f"(exceeds {HEADER_FUNC_BODY_THRESHOLD}; move to .cpp)" + ), + deduction=deduction, + )) + + # line-based rules with caps + tab_count = sum(1 for l in lines if "\t" in l) + long_line_count = sum(1 for l in lines if len(l) > LINE_LENGTH_LIMIT) + using_ns_count = sum( + 1 for l in lines if USING_NS_STD_RE.search(strip_comments(l)) + ) + chinese_count = sum(1 for l in lines if CHINESE_RE.search(l)) + + stripped_content = strip_comments(content) + # uppercase constants: strip strings too (so default-value strings like + # "ABACUS" do not trigger), and the regex excludes member accesses + # (e.g. GlobalV::MY_RANK, x.UPPER) via a negative lookbehind. + stripped_for_upper = strip_strings(stripped_content) + upper_const_count = sum( + len(UPPERCASE_CONST_RE.findall(l)) + for l in stripped_for_upper.split("\n") + ) + + # interface & dependency rules + global_dep_count = len(GLOBAL_DEPENDENCY_RE.findall(stripped_content)) + hpp_include_count = sum(1 for l in lines if HPP_INCLUDE_RE.match(l)) + friend_count = len(FRIEND_RE.findall(stripped_content)) + goto_count = len(GOTO_RE.findall(stripped_content)) + # `auto` keyword: counted on comment-and-string-stripped content so + # occurrences inside comments or string literals do not deduct points. + auto_count = len(AUTO_RE.findall(stripped_for_upper)) + + # default parameter: scan the full stripped content so multi-line + # declarations (parameter list spanning multiple lines) are detected; + # `[^();]*` in the regex already matches newlines. + default_param_count = len(DEFAULT_PARAM_RE.findall(stripped_content)) + + # unpaired new/delete (file-level): rough heuristic, cap applied below. + # All `new` expressions are counted first. `delete` expressions are the + # obvious symmetric ownership cancel, but there is a second class of + # cancellations that never produces a `delete` keyword at all: a + # C++11 (or later) smart pointer that owns the object from construction. + # Recognised ownership-transfer patterns are aggregated in OWNED_NEW_RE + # and subtracted before comparing with `delete` so the leak heuristic + # stays meaningful on modern code that prefers std::unique_ptr / + # std::shared_ptr to manual delete. Examples that now cancel: + # ptr.reset(new Foo()); + # std::unique_ptr p(new Foo(arg)); + # These cancellations deliberately do NOT touch `raw_new_count` below + # because the "raw_new_keyword" rule charges per direct `new` regardless + # of ownership — it is the stylistic counterpart to std::make_unique / + # std::make_shared, so `reset(new Foo)` and `unique_ptr(new T)` + # must still contribute to that count. + new_count = len(NEW_EXPR_RE.findall(stripped_content)) + delete_count = len(DELETE_EXPR_RE.findall(stripped_content)) + owned_new_count = len(OWNED_NEW_RE.findall(stripped_content)) + cancelled_new = delete_count + owned_new_count + unpaired_new = max(0, new_count - cancelled_new) + + # raw `new` keyword usage: each occurrence costs 1 (no cap) + raw_new_count = new_count + + def append_capped(rule: str, count: int) -> None: + if count == 0: + return + cap = CAPS.get(rule) + actual = min(count, cap) if cap else count + cap_text = f" (capped at {cap})" if cap and count > cap else "" + findings.append(Finding( + rule=rule, + line=None, + reason=f"{count} occurrence(s){cap_text}", + deduction=actual * WEIGHTS[rule], + )) + + append_capped("tab_indentation", tab_count) + append_capped("line_too_long", long_line_count) + append_capped("using_namespace_std", using_ns_count) + append_capped("chinese_comment", chinese_count) + append_capped("uppercase_constant", upper_const_count) + append_capped("default_parameter", default_param_count) + append_capped("global_dependency", global_dep_count) + append_capped("hpp_include", hpp_include_count) + append_capped("friend_keyword", friend_count) + append_capped("unpaired_new_delete", unpaired_new) + append_capped("raw_new_keyword", raw_new_count) + append_capped("goto_keyword", goto_count) + append_capped("auto_keyword", auto_count) + + # --- documentation style rules ------------------------------------- + # indented preprocessor directive (raw lines, `#` not at column 0) + indented_preproc_count = sum(1 for l in lines if INDENTED_PREPROC_RE.match(l)) + append_capped("indented_preprocessor", indented_preproc_count) + + # complex expressions (?:, &&, ||) written inline in a call's + # argument list: one finding per offending call, cap per file + complex_arg_hits = find_complex_call_arguments(content) + cap_complex = CAPS.get("complex_argument_expression", len(complex_arg_hits)) + for hit_idx, (line_no, callee) in enumerate(complex_arg_hits): + if hit_idx >= cap_complex: + break + findings.append(Finding( + rule="complex_argument_expression", + line=line_no, + reason=( + f"call to '{callee}' embeds a ternary/&&/|| expression " + f"in its argument list (assign to a named local first)" + ), + deduction=WEIGHTS["complex_argument_expression"], + )) + + # #if/#ifdef/#ifndef block wrapping individual call arguments + ifdef_arg_lines = find_ifdef_in_call_args(content) + cap_ifdef = CAPS.get("ifdef_in_call_args", len(ifdef_arg_lines)) + for hit_idx, line_no in enumerate(ifdef_arg_lines): + if hit_idx >= cap_ifdef: + break + findings.append(Finding( + rule="ifdef_in_call_args", + line=line_no, + reason=( + "#if/#ifdef block inside a function call's argument list " + "(select the argument before the call instead)" + ), + deduction=WEIGHTS["ifdef_in_call_args"], + )) + + # new `#define` macros in .h/.cpp (guard-style macros exempt) + if path.suffix in (".h", ".cpp"): + macro_hits = 0 + cap_macro = CAPS.get("new_macro_definition", 0) + for idx, l in enumerate(lines): + if macro_hits >= cap_macro: + break + m = MACRO_DEFINE_RE.match(l) + if not m: + continue + name = m.group(1) + rest = l[m.end():].strip() + # include guard: NAME ends in _H/_HPP and carries no + # parameter list or replacement text + if (name.endswith("_H") or name.endswith("_HPP")) and not rest: + continue + macro_hits += 1 + findings.append(Finding( + rule="new_macro_definition", + line=idx + 1, + reason=( + f"new `#define {name}` macro requires approval " + "(AGENTS.md rule 20)" + ), + deduction=WEIGHTS["new_macro_definition"], + )) + + # member variables initialized from PARAM in .h/.hpp (rule 21) + if path.suffix in (".h", ".hpp"): + param_init_lines = find_param_member_inits(content) + cap_param_init = CAPS.get("param_member_init", 0) + for hit_idx, line_no in enumerate(param_init_lines): + if hit_idx >= cap_param_init: + break + findings.append(Finding( + rule="param_member_init", + line=line_no, + reason=( + "member variable initialized from PARAM in a header " + "(pass the value through the constructor instead; " + "AGENTS.md rule 21)" + ), + deduction=WEIGHTS["param_member_init"], + )) + + # comment block(s) above the first include guard (headers only). One + # contiguous comment block counts once regardless of its length. + if path.suffix == ".h": + comment_above_guard = 0 + in_block = False + for l in lines: + if GUARD_RE.match(l): + break + if COMMENT_LINE_RE.match(l): + if not in_block: + comment_above_guard += 1 + in_block = True + elif l.strip(): + in_block = False + append_capped("comment_above_guard", comment_above_guard) + + # `@file xxx` name vs the actual filename + doc_file_match = DOC_FILE_RE.search(content) + if doc_file_match and doc_file_match.group(1) != path.name: + findings.append(Finding( + rule="doc_file_mismatch", + line=None, + reason=( + f"@file names '{doc_file_match.group(1)}' but the file is " + f"'{path.name}'" + ), + deduction=WEIGHTS["doc_file_mismatch"], + )) + + # duplicated comment text across comment blocks (e.g. the same formula + # repeated in the file-header and in a function doxygen block). Only + # lines of >= 40 chars are compared. Excluded from comparison: + # - doxygen command lines (@param/@return/...): repeated `@param pv` + # across overloads is normal, not redundant prose; + # - decorative separator lines (=== Part N ===): >50% non-alnum chars; + # - ALL-CAPS banner lines: visual section headers, not content. + def _is_dup_candidate(l: str) -> bool: + if len(l) < 40 or re.match(r"@\w+", l): + return False + alnum = sum(1 for ch in l if ch.isalnum()) + if alnum * 2 < len(l): + return False + if l.isupper(): + return False + return True + + comment_blocks = extract_comment_blocks(content) + if len(comment_blocks) > 1: + seen: dict = {} + duplicate_doc_lines = 0 + for block in comment_blocks: + block_lines = set(l for l in block if _is_dup_candidate(l)) + for l in block_lines: + if l in seen: + duplicate_doc_lines += 1 + else: + seen[l] = True + append_capped("duplicate_doc_block", duplicate_doc_lines) + + # too-many-parameters rule: each function's deduction is capped + # individually (per-function cap, not per-file). A file with many + # over-parameterised functions accumulates deductions across all of + # them, instead of one bad function hiding the rest. + long_param_funcs = find_long_function_signatures(content, FUNCTION_PARAM_THRESHOLD) + cap_params = CAPS.get("too_many_parameters") + for line_no, fname, pcount in long_param_funcs: + excess = pcount - FUNCTION_PARAM_THRESHOLD + raw_deduction = excess * WEIGHTS["too_many_parameters"] + if cap_params is not None and raw_deduction > cap_params: + per_deduction = cap_params + cap_text = f" (capped at {cap_params})" + else: + per_deduction = raw_deduction + cap_text = "" + findings.append(Finding( + rule="too_many_parameters", + line=line_no, + reason=( + f"function '{fname}' has {pcount} parameters " + f"(exceeds {FUNCTION_PARAM_THRESHOLD} by {excess}){cap_text}" + ), + deduction=per_deduction, + )) + + # high cyclomatic complexity rule: each function's deduction is capped + # individually (per-function cap, not per-file). + high_cyclo_funcs = find_high_complexity_functions(content, CYCLO_THRESHOLD) + cap_cyclo = CAPS.get("high_cyclomatic_complexity") + for line_no, fname, cyclo in high_cyclo_funcs: + excess = cyclo - CYCLO_THRESHOLD + raw_deduction = excess * WEIGHTS["high_cyclomatic_complexity"] + if cap_cyclo is not None and raw_deduction > cap_cyclo: + per_deduction = cap_cyclo + cap_text = f" (capped at {cap_cyclo})" + else: + per_deduction = raw_deduction + cap_text = "" + findings.append(Finding( + rule="high_cyclomatic_complexity", + line=line_no, + reason=( + f"function '{fname}' has cyclomatic complexity {cyclo} " + f"(exceeds {CYCLO_THRESHOLD} by {excess}){cap_text}" + ), + deduction=per_deduction, + )) + + # post-C++11 feature rule: base deduction for any usage + per-feature + # incremental deduction (capped). Total cap matches the previous + # one-shot -80, but a single misuse no longer swamps the whole file. + post_cpp11_hits = find_post_cpp11_features(content) + if post_cpp11_hits: + feature_list = ", ".join( + f"'{label}' (line {ln})" for ln, label in post_cpp11_hits + ) + findings.append(Finding( + rule="post_cpp11_feature", + line=post_cpp11_hits[0][0], + reason=( + f"uses post-C++11 feature(s): {feature_list} " + f"(repo baseline is C++11)" + ), + deduction=WEIGHTS["post_cpp11_feature"], + )) + num_features = len(post_cpp11_hits) + cap_features = CAPS.get("post_cpp11_per_feature", num_features) + capped = min(num_features, cap_features) + if capped > 0: + cap_text = ( + f" (capped at {cap_features})" if num_features > cap_features else "" + ) + findings.append(Finding( + rule="post_cpp11_per_feature", + line=post_cpp11_hits[0][0], + reason=( + f"{num_features} distinct post-C++11 feature(s) detected" + f"{cap_text}" + ), + deduction=capped * WEIGHTS["post_cpp11_per_feature"], + )) + + # class-based rules + ( + pub_findings, + static_findings, + long_func_findings, + conflict_findings, + ) = analyze_class_blocks(content) + findings.extend(pub_findings) + # static member findings: cap per file + if len(static_findings) > CAPS.get("static_member_variable", len(static_findings)): + cap_static = CAPS["static_member_variable"] + static_findings = static_findings[:cap_static] + findings.extend(static_findings) + findings.extend(long_func_findings) + # name conflict findings already respect cap via per-file cap on the rule + if len(conflict_findings) > CAPS.get("member_local_name_conflict", len(conflict_findings)): + cap = CAPS["member_local_name_conflict"] + conflict_findings = conflict_findings[:cap] + findings.extend(conflict_findings) + + # compute score: real_score is the unclamped value (may be negative); + # score is clamped to a minimum of 0 for display. + real_score = 100 + for f in findings: + real_score -= f.deduction + score = max(0, real_score) + + return FileReport( + path=str(path), + score=score, + findings=findings, + real_score=real_score, + ) diff --git a/tools/03_code_analysis/cqs_config.py b/tools/03_code_analysis/cqs_config.py new file mode 100644 index 0000000000..909dd146c0 --- /dev/null +++ b/tools/03_code_analysis/cqs_config.py @@ -0,0 +1,382 @@ +"""Shared configuration for the ABACUS code-quality scorer. + +Single source of truth for: source extensions, skip dirs, per-rule +weights/caps, numeric thresholds, all compiled regexes, and the two +prefix-classification helpers used by multiple rule detectors. + +Rules documentation lives in the entry script's module docstring +(`code_quality_score.py`) — keep both in sync when adding a rule. +""" + +from __future__ import annotations + +import re +from typing import List, Tuple + +SOURCE_EXTENSIONS = { + ".c", ".cc", ".cpp", ".cxx", ".cu", ".h", ".hh", ".hpp", ".hxx", ".cuh", +} + +SKIP_DIRS = { + ".git", "build", "__pycache__", "node_modules", ".cache", + "third_party", "thirdparty", ".vscode", ".idea", ".trae-cn", + "Dependencies", + "test", "tests", "test_serial", "test_parallel", "test_gpu", + "unit_test", "unittest", "unittests", +} + +WEIGHTS = { + "filename_too_long": 1, + "filename_uppercase": 1, + "hpp_implementation": 50, + "public_member_variable": 1, + "member_function_too_long": 1, + "tab_indentation": 1, + "using_namespace_std": 1, + "line_too_long": 1, + "chinese_comment": 1, + "uppercase_constant": 1, + "default_parameter": 2, + "global_dependency": 3, + "hpp_include": 2, + "friend_keyword": 1, + "unpaired_new_delete": 1, + "member_local_name_conflict": 1, + "file_too_long": 2, + "raw_new_keyword": 1, + "too_many_parameters": 1, + "high_cyclomatic_complexity": 1, + "post_cpp11_feature": 40, + "post_cpp11_per_feature": 8, + "goto_keyword": 2, + "auto_keyword": 1, + "static_member_variable": 1, + "indented_preprocessor": 1, + "complex_argument_expression": 1, + "ifdef_in_call_args": 1, + "comment_above_guard": 1, + "doc_file_mismatch": 1, + "duplicate_doc_block": 1, + "header_too_long": 1, + "header_function_body": 1, + "new_macro_definition": 3, + "param_member_init": 5, +} + +CAPS = { + "tab_indentation": 5, + "using_namespace_std": 5, + "line_too_long": 5, + "chinese_comment": 5, + "uppercase_constant": 5, + "default_parameter": 5, + "global_dependency": 10, + "hpp_include": 5, + "friend_keyword": 5, + "unpaired_new_delete": 5, + "member_local_name_conflict": 5, + "too_many_parameters": 30, + "high_cyclomatic_complexity": 30, + "post_cpp11_per_feature": 5, + "goto_keyword": 5, + "auto_keyword": 10, + "static_member_variable": 10, + "indented_preprocessor": 5, + "complex_argument_expression": 5, + "ifdef_in_call_args": 5, + "comment_above_guard": 3, + "duplicate_doc_block": 5, + "header_too_long": 5, + "header_function_body": 5, + "new_macro_definition": 10, + "param_member_init": 10, +} + +FUNCTION_LENGTH_THRESHOLD = 50 +FUNCTION_LENGTH_STEP = 50 +FILE_LENGTH_THRESHOLD = 500 +FILE_LENGTH_STEP = 50 +HEADER_LENGTH_THRESHOLD = 120 +HEADER_LENGTH_STEP = 10 +HEADER_FUNC_BODY_THRESHOLD = 20 +HEADER_FUNC_BODY_STEP = 10 +FUNCTION_PARAM_THRESHOLD = 7 +CYCLO_THRESHOLD = 10 +LINE_LENGTH_LIMIT = 120 +FILENAME_LENGTH_LIMIT = 20 +PASS_THRESHOLD = 60 + +CHINESE_RE = re.compile("[一-鿿]") +USING_NS_STD_RE = re.compile(r"\busing\s+namespace\s+\w[\w:]*\s*;") +UPPERCASE_CONST_RE = re.compile(r"(?])\b[A-Z][A-Z0-9_]{2,}\b(?![.:])") +ACCESS_RE = re.compile(r"^\s*(public|private|protected)\s*:") +CLASS_OPEN_RE = re.compile(r"\b(class|struct)\s+(\w+)\b") + +GLOBAL_DEPENDENCY_RE = re.compile(r"\b(?:GlobalV::|GlobalC::|PARAM(?:\.|->|::))") +# Member-variable declaration initialized from PARAM inside a class body +# (`.h`/`.hpp` rule 21). Matches `Type name = PARAM...;`, +# `Type name = PARAM...;` and `Type name{PARAM...};`, also when +# the declaration is split across lines (`\s*` spans newlines). The gap +# between the `=`/`{` and `PARAM` excludes `;{}()` so a match can never +# cross a statement boundary or dive into another brace/paren level; +# class-scope filtering (member function bodies, parameter lists) is +# done by the caller via brace/paren depth. +PARAM_MEMBER_INIT_RE = re.compile( + r"(?:static\s+)?(?:const\s+)?[\w:<>,\s*&]+?\s+\w+\s*" + r"[={]\s*[^;={}()]*?PARAM\s*(?:\.|->|::)" +) +HPP_INCLUDE_RE = re.compile(r'^\s*#\s*include\s+[<"][^>"]+\.hpp[>"]') +FRIEND_RE = re.compile(r"\bfriend\b(?!\s+(?:class\s+|struct\s+)?std::)") +NEW_EXPR_RE = re.compile(r"\bnew\s+\w") +DELETE_EXPR_RE = re.compile(r"\bdelete\s*\[\s*\]?\s+\w") + +# Occurrences of `new T(...)` or `new T[...]` that are clearly owned by a +# smart-pointer wrapper and therefore should NOT count as unpaired +# new/delete leaks. Two idiomatic C++11 shapes are recognised: +# 1. `ptr.reset(new T(args))` / `ptr->reset(new T(args))` +# — reset() transfers ownership into an existing smart pointer; +# most commonly used with unique_ptr/shared_ptr when make_unique/ +# make_shared are not available (pre-C++14) or unsuitable. +# 2. `unique_ptr p(new T(args))` / `shared_ptr p(new T(args))` +# plus the allocator-taking `allocate_shared` form — direct +# construction of a smart pointer that owns the freshly-allocated +# object from day one. +# The pattern does NOT try to skip new in `raw_new_count` (that rule +# penalises *any* direct use of `new` vs. make_unique/make_shared); it +# only affects the ownership heuristic `unpaired_new`. +_OWNED_NEW_SHAPES = [ + # shape 1: .reset(new T or ->reset(new T (optionally with whitespace) + r"\.(?:reset|operator\s*=)\s*\(\s*new\s+\w", + r"->(?:reset|operator\s*=)\s*\(\s*new\s+\w", + # shape 2: unique_ptr ident ( new T + # shared_ptr ident ( new T + # allocate_shared ( alloc , new T -> not exact, but + # the simpler `allocate_shared<...> ( ... new T` heuristic + r"\b(?:unique_ptr|shared_ptr|scoped_ptr|auto_ptr)\s*<[^>]*>\s*[A-Za-z_]\w*\s*\(\s*new\s+\w", + r"\b(?:make_unique|make_shared|allocate_shared)\s*<[^>]*>\s*\([^;()]*?\bnew\s+\w", +] +OWNED_NEW_RE = re.compile("|".join(f"(?:{p})" for p in _OWNED_NEW_SHAPES)) +GOTO_RE = re.compile(r"\bgoto\b") +# `auto` type deduction keyword. `\bauto\b` does not match `auto_ptr` +# (`_` is a word character, so there is no boundary before it). +AUTO_RE = re.compile(r"\bauto\b") + +# Preprocessor directive that does not start at column 0 (leading spaces/tabs +# before the '#'). Matched against raw lines; string literals virtually never +# contain a line-start '#', so no string stripping is needed here. +INDENTED_PREPROC_RE = re.compile( + r"^[ \t]+#\s*(?:if|ifdef|ifndef|define|endif|include|undef|pragma|else|elif)\b" +) +# First include guard / pragma-once line of a header. +GUARD_RE = re.compile(r"^\s*#\s*(?:ifndef|pragma\s+once)\b") +# New `#define NAME ...` directive; NAME is captured so guard-style +# macros (NAME ending in _H / _HPP with no parameter list or value) can +# be exempted by the caller. +MACRO_DEFINE_RE = re.compile(r"^\s*#\s*define\s+([A-Za-z_]\w*)") +# `@file name` inside a doxygen comment. +DOC_FILE_RE = re.compile(r"@file\s+(\S+)") +# Comment-only line (/// ... or a line inside a /* */ block is not fully +# distinguishable line-wise, so we only treat leading // or /* as comment). +COMMENT_LINE_RE = re.compile(r"^\s*(?://|/\*|\*)") + +DEFAULT_PARAM_RE = re.compile( + r"\b\w+\s*\([^();]*\b\w+\s*=(?![=>])[^();]*\)\s*" + r"(?:const\s*|noexcept\s*|override\s*|final\s*)*[;{]" +) + +# Strict declaration regex: requires `type ... name ;` or `type ... name = ...;`. +# Excludes pure assignments like `counter = counter + 1;` which lack a leading type. +DECL_RE = re.compile(r"^\s*(?:\w+[\s\*]+)+(\w+)\s*(?:=[^;]*)?;\s*$") + +BLOCK_KEYWORD_PREFIXES = ( + "public:", "private:", "protected:", + "typedef", "using", "static_assert", + "class", "struct", "friend", + "//", "/*", "*", + "return", "if ", "for ", "while ", + "switch ", "case ", "template", + "break", "continue", "goto", "default:", + "else", "do", "try", "catch", "throw", + "namespace", "enum", "union", +) + +NON_FUNCTION_KEYWORDS = { + "if", "for", "while", "switch", "sizeof", "return", "throw", + "catch", "class", "struct", "namespace", "enum", "union", + "template", "static_assert", "typedef", "using", "new", + "delete", "operator", "do", "else", "goto", "continue", + "break", "try", +} + +# Keywords that, when they appear as the last identifier in the prefix +# before a `name(args)` candidate followed by `;` or `=`, strongly +# indicate the candidate is a function call in statement/expression +# context rather than a declaration. +STATEMENT_CONTEXT_KEYWORDS = NON_FUNCTION_KEYWORDS | { + "co_await", "co_return", "co_yield", + "typeid", "noexcept", "alignof", "alignas", "decltype", + "static_cast", "const_cast", "dynamic_cast", "reinterpret_cast", + "and", "or", "not", "xor", "bitor", "bitand", "compl", +} + +# Punctuation characters that, when they are the last non-whitespace +# char before `name(args)` followed by `;` or `=`, indicate an +# expression / argument-list context (i.e. a call, not a declaration). +# `*` and `&` are intentionally NOT in this set because they also act +# as valid pointer/reference type qualifiers in a return type. The +# double-character operator forms (`&&`, `||`, `**`, etc.) are handled +# by the two-character check inside _is_declaration_prefix. +# Examples: `foo(1, bar(x), 3)` -> prefix ends with `,` +# `if (cond && ok(x))` -> prefix ends with `&&` +# `v.push_back(fn())` -> prefix ends with `.` +DECL_PREFIX_BAD_TAIL = set("=+-/%|^~!<>?([{,.;:") + +# Last-identifier regex: extracts the trailing identifier word from a +# prefix string (strips trailing punctuation and whitespace first). +_LAST_IDENT_RE = re.compile(r"([A-Za-z_]\w*)\s*$") + + +def _is_declaration_prefix(prefix_stripped: str) -> bool: + """Return True when `prefix_stripped` (the right-stripped text between + the start of the line and the candidate `name(args)` opener) is + consistent with a function declaration/definition context rather + than a call-site expression. + + A prefix is *not* a declaration prefix when: + * it is empty; + * its last non-whitespace character is punctuation that typically + appears inside expressions / argument lists (see + DECL_PREFIX_BAD_TAIL), or it is the double-character operator + form of `&&`, `||`, `**`, `->` that commonly appears inside + expressions; + * its trailing identifier token is a statement/expression + keyword (see STATEMENT_CONTEXT_KEYWORDS) such as `return`, + `throw`, `if`, `sizeof`, `static_cast`, etc. + + Constructors/destructors that open with `{` instead of `;` or `=` + are handled separately by the caller and do not go through this + check. + """ + if not prefix_stripped: + return False + tail_char = prefix_stripped[-1] + # Two-character expression operators that include `*` or `&` (which + # are not declared in the single-char bad-tail set because they are + # also valid type qualifiers). + if len(prefix_stripped) >= 2: + prev_char = prefix_stripped[-2] + double = prev_char + tail_char + if double in {"&&", "||", "**", "*&", "&*", "->"}: + return False + if tail_char in DECL_PREFIX_BAD_TAIL: + return False + m = _LAST_IDENT_RE.search(prefix_stripped) + if m and m.group(1) in STATEMENT_CONTEXT_KEYWORDS: + return False + return True + + +FUNC_NAME_RE = re.compile(r"\b([A-Za-z_]\w*)\s*\(") +QUALIFIER_RE = re.compile(r"\b(?:const|override|final|noexcept)\b") +CYCLO_KEYWORDS_RE = re.compile(r"\b(?:if|for|while|switch|case)\b|&&|\|\|") + +# Preprocessor conditional opener / closer, matched against raw lines. +PREPROC_IF_RE = re.compile(r"^\s*#\s*(?:if|ifdef|ifndef)\b") +PREPROC_ENDIF_RE = re.compile(r"^\s*#\s*endif\b") + +# Post-C++11 features with low false-positive rates. +# Each entry is (label, compiled_regex) — the label appears in the finding. +POST_CPP11_PATTERNS: List[Tuple[str, "re.Pattern[str]"]] = [ + # C++14 + ( + "std::make_unique", + re.compile(r"\bstd::make_unique\s*<"), + ), + ( + "digit-separator in numeric literal (C++14)", + re.compile(r"(?]=?|[.<>=])" + r"\s*\.\.\.\s*\)" + r"|\(\s*\.\.\.\s*(?:\+\+|\&\&|\|\||[+\-*/%^&|<>]=?|[.<>=])" + r"\s*[A-Za-z_]\w*\s*\)" + r"|\(\s*[A-Za-z_]\w*\s*(?:\+\+|\&\&|\|\||[+\-*/%^&|<>]=?|[.<>=])" + r"\s*\.\.\.\s*(?:\+\+|\&\&|\|\||[+\-*/%^&|<>]=?|[.<>=])" + r"\s*[^,)]+\s*\)" + ), + ), + ( + "std::optional (C++17)", + re.compile(r"\bstd::optional\b"), + ), + ( + "std::variant (C++17)", + re.compile(r"\bstd::variant\b"), + ), + ( + "std::any (C++17)", + re.compile(r"\bstd::any\b"), + ), + ( + "[[nodiscard]] (C++17)", + re.compile(r"\[\[nodiscard\b"), + ), + ( + "[[maybe_unused]] (C++17)", + re.compile(r"\[\[maybe_unused\b"), + ), + # C++20 + ( + "concept (C++20)", + re.compile(r"\bconcept\b"), + ), + ( + "requires (C++20)", + re.compile(r"\brequires\b"), + ), + ( + "consteval (C++20)", + re.compile(r"\bconsteval\b"), + ), + ( + "constinit (C++20)", + re.compile(r"\bconstinit\b"), + ), + ( + "coroutine co_await / co_yield / co_return (C++20)", + re.compile(r"\bco_(?:await|yield|return)\b"), + ), + ( + "std::span (C++20)", + re.compile(r"\bstd::span\b"), + ), + ( + "std::ranges (C++20)", + re.compile(r"\bstd::ranges::"), + ), + ( + "std::format (C++20)", + re.compile(r"\bstd::format\b"), + ), + # C++23 + ( + "std::expected (C++23)", + re.compile(r"\bstd::expected\b"), + ), + ( + "std::print / std::println (C++23)", + re.compile(r"\bstd::print(?:ln)?\s*\("), + ), +] diff --git a/tools/03_code_analysis/cqs_report.py b/tools/03_code_analysis/cqs_report.py new file mode 100644 index 0000000000..e8f443c635 --- /dev/null +++ b/tools/03_code_analysis/cqs_report.py @@ -0,0 +1,169 @@ +"""Report data model and renderers for the ABACUS code-quality scorer.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from cqs_config import PASS_THRESHOLD + + +@dataclass +class Finding: + rule: str + line: Optional[int] + reason: str + deduction: int + + +@dataclass +class FileReport: + path: str + score: int + findings: List[Finding] = field(default_factory=list) + real_score: int = 100 + + def to_dict(self) -> dict: + return { + "path": self.path, + "score": self.score, + "real_score": self.real_score, + "findings": [ + { + "rule": f.rule, + "line": f.line, + "reason": f.reason, + "deduction": f.deduction, + } + for f in self.findings + ], + } + + +def get_module(path: str) -> str: + """Group a file path into its module for rollup purposes. + + Returns the `source/source_*` prefix when present (e.g. + `source/source_lcao`); otherwise falls back to the file's parent + directory. Backslashes are normalised to forward slashes. + """ + parts = path.replace("\\", "/").split("/") + for i, p in enumerate(parts): + if ( + p == "source" + and i + 1 < len(parts) + and parts[i + 1].startswith("source_") + ): + return f"source/{parts[i + 1]}" + return "/".join(parts[:-1]) if len(parts) > 1 else path + + +def compute_module_rollup( + reports: List[FileReport], +) -> List[Tuple[str, int, float, int]]: + """Group reports by module and return rows sorted by avg score ascending. + + Each row is (module, file_count, average_score, passing_count). + Worst module (lowest average) appears first. + """ + groups: dict = {} + for r in reports: + m = get_module(r.path) + groups.setdefault(m, []).append(r) + rows: List[Tuple[str, int, float, int]] = [] + for module, rs in groups.items(): + n = len(rs) + avg = sum(r.score for r in rs) / n if n else 0.0 + passing = sum(1 for r in rs if r.score >= PASS_THRESHOLD) + rows.append((module, n, avg, passing)) + rows.sort(key=lambda x: x[2]) + return rows + + +def render_text(reports: List[FileReport], min_score: Optional[int]) -> str: + """Render reports as plain text.""" + if not reports: + return "No files to analyze.\n" + + visible = reports if min_score is None else [r for r in reports if r.score <= min_score] + sorted_reports = sorted(visible, key=lambda r: (r.score, r.real_score)) + + out: List[str] = [] + width = 70 + out.append("=" * width) + out.append("Code Quality Score Report") + out.append("=" * width) + + rollup = compute_module_rollup(reports) + if rollup: + out.append("") + out.append( + "Per-module rollup (sorted by average score, worst first):" + ) + out.append(f" {'Module':<40} {'Files':>5} {'Avg':>6} {'Pass':>12}") + out.append("-" * width) + for module, n, avg, passing in rollup: + module_disp = module if len(module) <= 40 else module[-37:] + "..." + pass_str = f"{passing}/{n}" + out.append( + f" {module_disp:<40} {n:>5} {avg:>6.1f} {pass_str:>12}" + ) + out.append("") + + out.append(f"{'Score':>6} {'File'}") + out.append("-" * width) + for r in sorted_reports: + suffix = f" (real_score={r.real_score})" if r.real_score < 0 else "" + out.append(f"{r.score:>6} {r.path}{suffix}") + out.append("") + + for r in sorted_reports: + if not r.findings: + continue + out.append("-" * width) + out.append(f"File: {r.path} (score: {r.score})") + out.append("-" * width) + for f in r.findings: + loc = f"line {f.line}" if f.line else "file" + out.append(f" [{f.rule}] {loc}: {f.reason} (-{f.deduction})") + out.append("") + + total = len(reports) + visible_n = len(visible) + avg = sum(r.score for r in reports) / total if total else 0.0 + passing = sum(1 for r in reports if r.score >= PASS_THRESHOLD) + out.append("=" * width) + out.append(f"Files scanned: {total}") + out.append(f"Files shown: {visible_n}") + out.append(f"Average score: {avg:.1f}") + out.append(f"Passing (>= {PASS_THRESHOLD}): {passing}/{total}") + out.append("=" * width) + return "\n".join(out) + "\n" + + +def render_json(reports: List[FileReport], min_score: Optional[int]) -> str: + visible = reports if min_score is None else [r for r in reports if r.score <= min_score] + rollup = compute_module_rollup(reports) + return json.dumps({ + "summary": { + "total_scanned": len(reports), + "total_shown": len(visible), + "average_score": ( + sum(r.score for r in reports) / len(reports) if reports else 0.0 + ), + "passing": sum(1 for r in reports if r.score >= PASS_THRESHOLD), + "pass_threshold": PASS_THRESHOLD, + "by_module": [ + { + "module": module, + "file_count": n, + "average_score": avg, + "passing": passing, + } + for module, n, avg, passing in rollup + ], + }, + "files": [r.to_dict() for r in sorted(visible, key=lambda r: (r.score, r.real_score))], + }, indent=2) + diff --git a/tools/03_code_analysis/cqs_rules.py b/tools/03_code_analysis/cqs_rules.py new file mode 100644 index 0000000000..e71dcbc1de --- /dev/null +++ b/tools/03_code_analysis/cqs_rules.py @@ -0,0 +1,1234 @@ +"""Rule-detection functions for the ABACUS code-quality scorer. + +Every public `find_*`/`analyze_*` function scans source text and returns +violation locations (line numbers plus optional detail strings). Rule +weights, caps and regexes live in `cqs_config`. +""" + +from __future__ import annotations + +import re +from typing import Dict, List, Optional, Set, Tuple + +from cqs_config import ( + ACCESS_RE, AUTO_RE, BLOCK_KEYWORD_PREFIXES, CLASS_OPEN_RE, + CYCLO_KEYWORDS_RE, CYCLO_THRESHOLD, DECL_RE, DEFAULT_PARAM_RE, + FUNC_NAME_RE, FUNCTION_LENGTH_STEP, FUNCTION_LENGTH_THRESHOLD, + FUNCTION_PARAM_THRESHOLD, + GLOBAL_DEPENDENCY_RE, HEADER_FUNC_BODY_THRESHOLD, HPP_INCLUDE_RE, + INDENTED_PREPROC_RE, + NEW_EXPR_RE, NON_FUNCTION_KEYWORDS, PARAM_MEMBER_INIT_RE, + POST_CPP11_PATTERNS, + PREPROC_IF_RE, QUALIFIER_RE, STATEMENT_CONTEXT_KEYWORDS, WEIGHTS, + _is_declaration_prefix, _LAST_IDENT_RE, +) +from cqs_report import Finding +from cqs_text import strip_comments, strip_strings + + +def find_class_blocks(code: str) -> List[Tuple[int, int, str, str]]: + """Find top-level class/struct blocks. Returns list of + (start_line_1indexed, end_line_1indexed, kind, name). + + Walks brace matching starting from each `class X {` / `struct X {` opener. + Excludes `template ` type parameters and `enum class` scoped enums. + """ + blocks: List[Tuple[int, int, str, str]] = [] + i = 0 + n = len(code) + while i < n: + m = CLASS_OPEN_RE.search(code, i) + if not m: + break + # reject template type parameters: `template ` or `<..., class T, ...>` + # look at non-whitespace chars before the match + pre_start = m.start() - 1 + while pre_start >= 0 and code[pre_start] in " \t\n\r": + pre_start -= 1 + if pre_start >= 0 and code[pre_start] in "<,": + i = m.end() + continue + # reject `enum class` / `enum struct` scoped enumerations + # look backwards for `enum` keyword (with optional whitespace) + enum_check_pos = pre_start + while enum_check_pos >= 0 and code[enum_check_pos] in " \t\n\r": + enum_check_pos -= 1 + if ( + enum_check_pos >= 3 + and code[enum_check_pos - 3:enum_check_pos + 1].lower() == "enum" + and (enum_check_pos == 3 or not code[enum_check_pos - 4].isalnum()) + ): + i = m.end() + continue + # find the opening brace after the class/struct header + # allow inheritance clauses: class X : public Y { ... } + brace_pos = code.find("{", m.end()) + if brace_pos < 0: + break + # reject if there's a `;` before the brace (forward declaration) + if ";" in code[m.end():brace_pos]: + i = m.end() + continue + depth = 1 + j = brace_pos + 1 + in_string = False + in_char = False + while j < n and depth > 0: + c = code[j] + if in_string: + if c == "\\": + j += 2 + continue + if c == '"': + in_string = False + j += 1 + continue + if in_char: + if c == "\\": + j += 2 + continue + if c == "'": + in_char = False + j += 1 + continue + if c == '"': + in_string = True + j += 1 + continue + if c == "'": + in_char = True + j += 1 + continue + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + break + j += 1 + if depth == 0: + start_line = code[:m.start()].count("\n") + 1 + end_line = code[:j].count("\n") + 1 + blocks.append((start_line, end_line, m.group(1), m.group(2))) + i = j + 1 + else: + i = m.end() + return blocks + + +def detect_access_at_line(class_lines: List[str], target_idx: int, default_access: str) -> str: + """Determine the active access specifier for line `target_idx` inside class.""" + access = default_access + for k in range(target_idx + 1): + m = ACCESS_RE.match(class_lines[k]) + if m: + access = m.group(1) + return access + + +def is_public_member_var(line: str) -> bool: + """Heuristic: does this stripped code line look like a public member variable + declaration (not a function, not a typedef, not a nested class)?""" + s = line.strip() + if not s.endswith(";"): + return False + if "(" in s or ")" in s: + return False + if "{" in s or "}" in s: + return False + if "=" in s and "(" in s: + return False + bad_prefixes = ( + "public:", "private:", "protected:", + "typedef", "using", "static_assert", + "class", "struct", "friend", + "//", "/*", "*", + "return", "if ", "for ", "while ", "switch ", "case ", + "template", + ) + if s.startswith(bad_prefixes): + return False + # skip macro-like lines (all caps with parens already excluded above) + # require at least one identifier character + if not re.search(r"[A-Za-z_]", s): + return False + return True + + +def is_static_member_var(line: str) -> bool: + """Heuristic: does this stripped code line look like a static member + variable declaration inside a class/struct body? + + Targets `static ;` and `static = ...;` + while excluding static member functions, static_assert, and + static_cast (the latter two carry parentheses). + """ + s = line.strip() + if not s.endswith(";"): + return False + if not s.startswith("static "): + return False + if s.startswith("static_assert"): + return False + if "(" in s or ")" in s: + return False + if "{" in s or "}" in s: + return False + return True + + +def _match_var_decl(stripped_line: str) -> Optional[str]: + """If line looks like a variable declaration `type name;` or + `type name = ...;`, return the variable name; else None. + + Excludes pure assignments (e.g. `counter = counter + 1;`) which lack a + leading type token. This is a best-effort heuristic, not a full parser. + """ + s = stripped_line.strip() + if not s or s.startswith(BLOCK_KEYWORD_PREFIXES): + return None + m = DECL_RE.match(s) + return m.group(1) if m else None + + +# Keywords that, when they appear on a class-body line that does not yet +# end with ';', mark the start of a declaration that *may* span several +# source lines. Following lines (up to the terminating ';') should not be +# mistaken for individual member-variable declarations. This catches +# multi-line `using`/`typedef`/`template`/namespace-qualified type aliases +# such as: +# using value_type +# = std::vector; +# Without the lookahead, the second line is inspected in isolation and +# reads as `= std::vector;` — a bogus public-static member hit. +_MULTILINE_DECL_STARTERS = ( + # Both the space-separated form (`using value_type = ...`) and the + # bare-keyword-alone form (`typedef` at the end of a line with a + # line-break before the type) are valid. Using tuples of both + # spellings catches multi-line aliases regardless of where the + # author wraps the line. + "using", "typedef", "template", "typename", + "namespace", "extern", "friend", +) + + +def _mark_continued_decl_lines(class_lines: List[str]) -> "set": + """Return the set of 0-based line indices inside a class body that + belong to a multi-line declaration started by a previous line (and + therefore must not be treated as standalone member declarations). + + A line is considered a "continuation line" when: + * there exists a strictly earlier non-empty line at the same or + smaller brace-nesting depth that begins with a declaration + starter keyword (`using`, `typedef`, `template`, `typename`, + `namespace`, `extern`, `friend`); + * the earlier starter line does NOT terminate with ';' (i.e. the + declaration has not been closed yet); + * no ';' has been seen on any intervening line at the starter's + brace-nesting depth since the starter was opened. + + The heuristic is deliberately conservative: lines are only marked as + continuations inside the same brace-depth run so that real member + declarations following a `using` block are never incorrectly + suppressed. + """ + marked: set = set() + depth = 0 + # stack of (depth_at_starter, starter_line_idx) for currently-open + # multi-line declaration starters. Multiple starters may coexist at + # different brace-nesting levels; a semicolon at depth D closes all + # starters that were opened at exactly D (conservative, but this + # matches how class bodies declare one item per terminating ';'). + open_starters: List[Tuple[int, int]] = [] + for idx, line in enumerate(class_lines): + prev_depth = depth + stripped_l = line.strip() + # (1) Mark as continuation BEFORE processing any ';' on this line: + # the terminating-semicolon line is still part of the multi-line + # declaration and must be suppressed in the public/static-member + # heuristics. The starter line itself is never marked; every + # later line that runs while any starter is open is marked. + if open_starters and any(idx > sidx for _, sidx in open_starters): + marked.add(idx) + # (2) Now close starters at prev_depth if the current line + # contains a ';' at that depth. Closure happens AFTER the + # marking step so the line that carries the terminating ';' is + # still counted as the final continuation of the declaration. + if stripped_l and ";" in stripped_l: + closed_depths = {d for d, _ in open_starters if d == prev_depth} + if closed_depths: + open_starters = [ + (d, sidx) for (d, sidx) in open_starters + if d not in closed_depths + ] + # (3) Advance brace-depth for the current line (in/out of + # functions, nested classes, etc.). Starter-opening happens + # after depth tracking so the starter is recorded at its + # prev_depth, which is the same depth at which the terminating + # ';' will be processed. + depth += line.count("{") - line.count("}") + # (4) Lines that begin with declaration-starter keywords but do + # NOT contain ';' will (in a well-formed class body) continue + # across at least one more line before reaching the terminating + # ';'. Record them so subsequent lines can be marked. + if prev_depth >= 1 and stripped_l and ";" not in stripped_l \ + and stripped_l.startswith(_MULTILINE_DECL_STARTERS): + open_starters.append((prev_depth, idx)) + return marked + + +def _skip_trailing_return_type(s: str, start: int) -> int: + """Advance past `-> ReturnType` starting at position `start`. + + `start` must point at the `-` in `->`. Scans over the return type by + tracking bracket nesting; stops and returns the position of the first + post-type character. Recognises two stop conditions at depth 0: + + (a) current char is a direct terminator (`;`, `{`, `=`) or a + qualifier keyword starts here — stop on the spot so the caller + consumes it on the next loop iteration. + (b) current char is whitespace followed by a terminator or qualifier + keyword — stop at the whitespace position so the caller's own + whitespace-skip advances past it cleanly. + + `:` is NOT a terminator because qualified return types like + `std::vector` contain `::` scope-resolution colons, and in any + case a trailing-return-type function cannot also be a constructor + (which is the only case that introduces a member initializer list). + + Caller re-runs qualifier / suffix scanning after this helper returns. + """ + i = start + 2 # skip past `->` + n = len(s) + # skip whitespace between `->` and the type + while i < n and s[i] in " \t\n": + i += 1 + depth = 0 + while i < n: + c = s[i] + if c in "([{<": + depth += 1 + elif c in ")]}>": + depth = max(0, depth - 1) + if depth == 0: + # (a) terminator/qualifier attached directly to the type, e.g. `int;` + if c in ";{=": + break + if QUALIFIER_RE.match(s, i): + break + # (b) terminator/qualifier after whitespace, e.g. `int const` / `int ;` + if c in " \t\n": + next_word_start = i + 1 + while next_word_start < n and s[next_word_start] in " \t\n": + next_word_start += 1 + if next_word_start >= n: + break + nc = s[next_word_start] + if nc in ";{=": + break + if QUALIFIER_RE.match(s, next_word_start): + break + i += 1 + return i + + +def _skip_member_initializer_list(s: str, start: int) -> int: + """Advance past `: member(args), member{args}, ...` constructor initializer + list starting at position `start`. + + `start` must point at the `:` that introduces the list (and be preceded by + `)` — caller ensures this is not a `::` scope-resolution colon). Scans + commas between members while respecting nesting; returns the position of + the opening `{` (or `;`, `=`, or end-of-string if body-less). + + Heuristic: at depth 0, `{` can either open a brace-initializer for a + member (e.g. `a{1,2}`) or open the function body itself. The function + body `{` is distinguished by looking at the previous non-whitespace + character: if it is `)` or `}` (a previous initializer ended normally) + or `,` / `:` (list-level punctuation), then `{` belongs to the function + body and scanning stops. Otherwise the `{` is consumed as a member + brace-initializer (depth increases normally). + """ + i = start + 1 # skip past `:` + n = len(s) + depth = 0 + while i < n: + c = s[i] + # Decide `{` at depth 0 before bumping depth. + if depth == 0 and c == "{": + prev = i - 1 + while prev >= 0 and s[prev] in " \t\n": + prev -= 1 + prev_c = s[prev] if prev >= 0 else "" + # `{` after full initializer / separator = function body opener + if prev_c in ")}:,": + break + if c in "([{<": + depth += 1 + elif c in ")]}>": + depth = max(0, depth - 1) + if depth == 0 and c in ";=": + break + i += 1 + return i + + +def count_function_params(params_str: str) -> int: + """Count top-level parameters by counting commas at bracket depth 0. + + Tracks (), [] {} and <> as nesting (so commas inside std::map or + function-pointer params are not counted as param separators). + """ + s = params_str.strip() + if not s or s == "void": + return 0 + depth = 0 + count = 1 + for c in s: + if c in "([{<": + depth += 1 + elif c in ")]}>": + depth = max(0, depth - 1) + elif c == "," and depth == 0: + count += 1 + return count + + +def find_long_function_signatures( + content: str, threshold: int +) -> List[Tuple[int, str, int]]: + """Find function declarations/definitions whose parameter count exceeds threshold. + + Returns list of (line_no, function_name, param_count). Best-effort: + requires either a return-type prefix before the function name (for `;` + and `=` endings) or a `{` ending (for constructors/destructors which + have no return type). + """ + stripped = strip_comments(content) + n = len(stripped) + findings: List[Tuple[int, str, int]] = [] + + i = 0 + while i < n: + m = FUNC_NAME_RE.search(stripped, i) + if not m: + break + name = m.group(1) + if name in NON_FUNCTION_KEYWORDS: + i = m.end() + continue + + # inspect the prefix before `name(` to reject calls/lambdas/macros + line_start = stripped.rfind("\n", 0, m.start()) + 1 + prefix = stripped[line_start:m.start()] + prefix_stripped = prefix.rstrip() + prefix_lstripped = prefix.lstrip() + + # skip preprocessor lines + if prefix_lstripped.startswith("#"): + i = m.end() + continue + # skip lambdas: prefix ends with `]` + if prefix_stripped.endswith("]"): + i = m.end() + continue + # skip method calls: prefix ends with `.` or `->` + if prefix_stripped.endswith(".") or prefix_stripped.endswith("->"): + i = m.end() + continue + # skip assignment-result calls: prefix ends with `=` (but not `==`) + if prefix_stripped.endswith("=") and not prefix_stripped.endswith("=="): + i = m.end() + continue + # skip function-pointer typedefs + if "typedef" in prefix_stripped: + i = m.end() + continue + + # match parens to find params string + paren_open = m.end() - 1 + depth = 1 + j = paren_open + 1 + while j < n and depth > 0: + c = stripped[j] + if c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth == 0: + break + j += 1 + if depth != 0: + i = m.end() + continue + params_str = stripped[paren_open + 1:j] + + # after `)`: optional qualifiers (const/override/final/noexcept), + # optional trailing-return type (`-> ReturnType`), optional + # constructor member initializer list (`: a(x), b{y}`), then ; { or = + pos = j + 1 + while pos < n and stripped[pos] in " \t\n": + pos += 1 + while True: + # consume qualifiers + mq = QUALIFIER_RE.match(stripped, pos) + if mq: + pos = mq.end() + while pos < n and stripped[pos] in " \t\n": + pos += 1 + continue + # consume trailing return type: `-> ReturnType` + if ( + pos + 1 < n + and stripped[pos] == "-" + and stripped[pos + 1] == ">" + ): + pos = _skip_trailing_return_type(stripped, pos) + while pos < n and stripped[pos] in " \t\n": + pos += 1 + continue + # consume constructor member initializer list: `: members...` + # but avoid scope-resolution `::` — the prior non-whitespace + # char must be `)` (the closing param paren we just matched) + if pos < n and stripped[pos] == ":": + pre_colon = pos - 1 + while pre_colon >= 0 and stripped[pre_colon] in " \t\n": + pre_colon -= 1 + if pre_colon == j: + pos = _skip_member_initializer_list(stripped, pos) + while pos < n and stripped[pos] in " \t\n": + pos += 1 + continue + break + if pos >= n or stripped[pos] not in ";{=": + i = j + 1 + continue + + # For `;` and `=`, the preceding prefix must look like a genuine + # declaration context (return type + optional specifiers) rather + # than the tail of an expression / argument list. + # - `int f(int);` prefix `int` -> declaration ✓ + # - `return f(1, 2, 3, ...);` prefix `return` -> call ✗ + # - `foo(1, bar(x), 3);` prefix `foo(1, ` -> call ✗ (ends with `,(`) + # - Constructor/destructor bodies terminate with `{` and bypass + # this check entirely. + if stripped[pos] in ";=" and not _is_declaration_prefix(prefix_stripped): + i = j + 1 + continue + + param_count = count_function_params(params_str) + if param_count > threshold: + line_no = stripped[:paren_open].count("\n") + 1 + findings.append((line_no, name, param_count)) + + i = j + 1 + + return findings + + +def find_function_bodies(content: str) -> List[Tuple[int, str, int, int]]: + """Find function definitions (with body), not just declarations. + + Returns list of (signature_line_no, function_name, body_start_pos, + body_end_pos) where positions are absolute offsets in stripped content. + Reuses the prefix/reject logic from find_long_function_signatures. + """ + stripped = strip_comments(content) + n = len(stripped) + bodies: List[Tuple[int, str, int, int]] = [] + + i = 0 + while i < n: + m = FUNC_NAME_RE.search(stripped, i) + if not m: + break + name = m.group(1) + if name in NON_FUNCTION_KEYWORDS: + i = m.end() + continue + + line_start = stripped.rfind("\n", 0, m.start()) + 1 + prefix = stripped[line_start:m.start()] + prefix_stripped = prefix.rstrip() + prefix_lstripped = prefix.lstrip() + + if prefix_lstripped.startswith("#"): + i = m.end() + continue + if prefix_stripped.endswith("]"): + i = m.end() + continue + if prefix_stripped.endswith(".") or prefix_stripped.endswith("->"): + i = m.end() + continue + if prefix_stripped.endswith("=") and not prefix_stripped.endswith("=="): + i = m.end() + continue + if "typedef" in prefix_stripped: + i = m.end() + continue + + paren_open = m.end() - 1 + depth = 1 + j = paren_open + 1 + while j < n and depth > 0: + c = stripped[j] + if c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth == 0: + break + j += 1 + if depth != 0: + i = m.end() + continue + + # find what comes after `)`: skip whitespace, qualifiers, optional + # trailing-return type (`-> ReturnType`), and optional constructor + # member initializer list (`: a(x), b{y}`). + pos = j + 1 + while pos < n and stripped[pos] in " \t\n": + pos += 1 + while True: + # consume qualifiers + mq = QUALIFIER_RE.match(stripped, pos) + if mq: + pos = mq.end() + while pos < n and stripped[pos] in " \t\n": + pos += 1 + continue + # consume trailing return type: `-> ReturnType` + if ( + pos + 1 < n + and stripped[pos] == "-" + and stripped[pos + 1] == ">" + ): + pos = _skip_trailing_return_type(stripped, pos) + while pos < n and stripped[pos] in " \t\n": + pos += 1 + continue + # consume constructor member initializer list: `: members...` + # but avoid scope-resolution `::` — the prior non-whitespace + # char must be `)` (the closing param paren we just matched) + if pos < n and stripped[pos] == ":": + pre_colon = pos - 1 + while pre_colon >= 0 and stripped[pre_colon] in " \t\n": + pre_colon -= 1 + if pre_colon == j: + pos = _skip_member_initializer_list(stripped, pos) + while pos < n and stripped[pos] in " \t\n": + pos += 1 + continue + break + + # we need a `{` body (not `;` declaration, not `= 0` pure virtual) + if pos >= n or stripped[pos] != "{": + i = j + 1 + continue + + # match braces to find body end + body_open = pos + depth = 1 + body_close = body_open + 1 + while body_close < n and depth > 0: + c = stripped[body_close] + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + break + body_close += 1 + if depth != 0: + i = j + 1 + continue + + sig_line_no = stripped[:paren_open].count("\n") + 1 + bodies.append((sig_line_no, name, body_open, body_close)) + i = body_close + 1 + + return bodies + + +_TEMPLATE_DECL_RE = re.compile(r"\btemplate\b\s*<") +_INLINE_RE = re.compile(r"\binline\b") +# how many consecutive non-blank lines above a signature are examined +# when looking for a (possibly multi-line) `template <...>` header +_TEMPLATE_HEADER_SCAN_LINES = 16 + + +def find_header_function_bodies( + content: str, +) -> List[Tuple[int, str, int]]: + """Find non-inline, non-template function bodies in a .h header that + exceed HEADER_FUNC_BODY_THRESHOLD lines. + + Exempts functions declared `inline` and template functions (they + must live in the header): the `template <...>` header is recognised + on the signature line itself, on the consecutive non-blank lines + above the signature (in any line-splitting), and for member + functions of a templated class. Short getter/setter-style bodies at + or below the threshold are exempt by construction. Returns a list of + (signature_line_no, function_name, body_line_count). + """ + stripped = strip_comments(content) + lines = stripped.split("\n") + class_blocks = find_class_blocks(stripped) + results: List[Tuple[int, str, int]] = [] + for sig_line, name, body_open, body_close in find_function_bodies( + content + ): + body_lines = stripped[body_open:body_close].count("\n") + if body_lines <= HEADER_FUNC_BODY_THRESHOLD: + continue + # exempt `inline ... name(...) {`: check the declaration text + decl_text = _declaration_prefix_before(stripped, body_open) + if _INLINE_RE.search(decl_text): + continue + if _is_template_context(lines, sig_line, decl_text, class_blocks): + continue + results.append((sig_line, name, body_lines)) + return results + + +def _is_template_context( + lines: List[str], + sig_line: int, + decl_text: str, + class_blocks: List[Tuple[int, int, str, str]], +) -> bool: + """Return True when the function whose signature starts on 1-based + `sig_line` is introduced by a `template <...>` header. + + Recognised forms (formatting must not change the result): + * the `template <...>` header shares the signature line: + `template void f(T& v) { ... }`; + * it sits on the consecutive non-blank lines above the signature, + split across any number of lines — as long as no `;` appears + between the `template` keyword and the signature (a `;` would + close an unrelated declaration in between); + * the function is a member of a class introduced by a + `template <...>` header. + """ + if _TEMPLATE_DECL_RE.search(decl_text): + return True + # consecutive non-blank lines above the signature; only the text + # after the last `;` belongs to this declaration's header + buf = "" + idx = sig_line - 2 # line index just above the signature + steps = 0 + while ( + idx >= 0 and lines[idx].strip() + and steps < _TEMPLATE_HEADER_SCAN_LINES + ): + buf = lines[idx] + "\n" + buf + if _TEMPLATE_DECL_RE.search(buf.rsplit(";", 1)[-1]): + return True + idx -= 1 + steps += 1 + # member of a templated class: check the innermost enclosing class + for start_line, end_line, _kind, _name in class_blocks: + if start_line <= sig_line <= end_line: + cls_decl = lines[start_line - 1] + jdx = start_line - 2 + steps = 0 + while ( + jdx >= 0 and lines[jdx].strip() + and steps < _TEMPLATE_HEADER_SCAN_LINES + ): + cls_decl = lines[jdx] + "\n" + cls_decl + jdx -= 1 + steps += 1 + if _TEMPLATE_DECL_RE.search(cls_decl.rsplit(";", 1)[-1]): + return True + break + return False + + +def find_param_member_inits(content: str) -> List[int]: + """Find class member variables initialized from PARAM in a header + (AGENTS.md rule 17 / 21). + + Scans only inside top-level class/struct bodies and only at class + scope: a match must sit at the class body's own brace level (not + inside a member function body or a nested class) and outside any + parentheses (not a function default argument). The whole file is + comment- AND string-stripped first so declarations inside comment + blocks or string literals are never matched. + + Returns a list of 1-indexed line numbers of offending declarations. + """ + stripped = strip_strings(strip_comments(content)) + line_starts = [0] + for line in stripped.split("\n"): + line_starts.append(line_starts[-1] + len(line) + 1) + hits: List[int] = [] + for start_line, end_line, _kind, _name in find_class_blocks(stripped): + body_lo = line_starts[start_line - 1] + body_hi = line_starts[end_line] + body = stripped[body_lo:body_hi] + # the class body opens with the first `{` of the block (an + # inheritance clause carries no braces) + brace_pos = body.find("{") + if brace_pos < 0: + continue + for m in PARAM_MEMBER_INIT_RE.finditer(body): + seg = body[brace_pos + 1:m.start()] + brace_depth = 1 + seg.count("{") - seg.count("}") + paren_depth = seg.count("(") - seg.count(")") + if brace_depth != 1 or paren_depth != 0: + continue + # the type part of the pattern absorbs the whitespace and + # newline preceding the declaration, so skip it to report + # the line where the declaration text starts + decl_start = m.start() + while decl_start < len(body) and body[decl_start] in " \t\n": + decl_start += 1 + line_no = stripped[:body_lo + decl_start].count("\n") + 1 + hits.append(line_no) + return hits + + +def _declaration_prefix_before(stripped: str, body_open: int) -> str: + """Return the declaration text preceding a function body `{`, walking + back over the parameter list to the start of the signature line.""" + depth = 0 + i = body_open + while i > 0: + c = stripped[i - 1] + if c == ")": + depth += 1 + elif c == "(": + depth -= 1 + if depth == 0: + break + i -= 1 + line_start = stripped.rfind("\n", 0, i) + 1 + return stripped[line_start:body_open] + + +def find_high_complexity_functions( + content: str, threshold: int +) -> List[Tuple[int, str, int]]: + """Find functions whose cyclomatic complexity exceeds threshold. + + Cyclomatic complexity counts: if, for, while, switch, case, &&, ||. + Returns list of (line_no, function_name, complexity). + + String and character literals are blanked before the regex runs so + that control-flow words appearing inside messages (e.g. + `const char* msg = "if not ok return";`) are not counted as real + control flow statements. + """ + stripped = strip_strings(strip_comments(content)) + findings: List[Tuple[int, str, int]] = [] + for sig_line, name, body_open, body_close in find_function_bodies(content): + body = stripped[body_open + 1:body_close] + complexity = len(CYCLO_KEYWORDS_RE.findall(body)) + if complexity > threshold: + findings.append((sig_line, name, complexity)) + return findings + + +def find_post_cpp11_features( + content: str, +) -> List[Tuple[int, str]]: + """Find the first occurrence of each post-C++11 feature. + + Uses the patterns in POST_CPP11_PATTERNS against content with both + comments AND string literals blanked out. String blanking prevents + false matches on keywords embedded in user-facing messages (e.g. + `"X requires Y"` in a WARNING_QUIT call). + + Returns a list of (line_no, feature_label) — one entry per distinct + detected feature so the finding message is informative (the deduction + is one-shot -80 per file regardless of how many features hit). + """ + stripped = strip_strings(strip_comments(content)) + hits: List[Tuple[int, str]] = [] + for label, regex in POST_CPP11_PATTERNS: + m = regex.search(stripped) + if m: + line_no = stripped[:m.start()].count("\n") + 1 + hits.append((line_no, label)) + return hits + + +# Preprocessor conditional opener / closer, matched against raw lines. +PREPROC_IF_RE = re.compile(r"^\s*#\s*(?:if|ifdef|ifndef)\b") +PREPROC_ENDIF_RE = re.compile(r"^\s*#\s*endif\b") + + +def find_complex_call_arguments(content: str) -> List[Tuple[int, str]]: + """Find function calls that embed a ternary `?:`, `&&` or `||` + expression in a top-level argument (AGENTS.md §14). + + Reuses the prefix/reject logic of find_long_function_signatures to + skip declarations, definitions, control-statement headers, lambdas + and method-call chains. After matching a call's parentheses, only + `?`, `&&`, `||` at the call's own paren depth are attributed to + this call; operators nested inside an inner call's parentheses are + judged when the inner call is visited (the scan resumes inside the + outer call's argument list so nested callees are never skipped). + Angle brackets `<>` of template argument lists inside the argument + list are treated as nesting only in their "glued" form (`Foo`); + a spaced `<`/`>` is a comparison (`a < b ? c : d`) and does not + nest, so the ternary is still attributed to the call. + + Returns a list of (line_no, callee_name) for each offending call. + """ + stripped = strip_strings(strip_comments(content)) + n = len(stripped) + hits: List[Tuple[int, str]] = [] + + i = 0 + while i < n: + m = FUNC_NAME_RE.search(stripped, i) + if not m: + break + name = m.group(1) + if name in NON_FUNCTION_KEYWORDS: + i = m.end() + continue + + line_start = stripped.rfind("\n", 0, m.start()) + 1 + prefix = stripped[line_start:m.start()] + prefix_stripped = prefix.rstrip() + prefix_lstripped = prefix.lstrip() + + if prefix_lstripped.startswith("#"): + i = m.end() + continue + if prefix_stripped.endswith("]"): + i = m.end() + continue + if prefix_stripped.endswith(".") or prefix_stripped.endswith("->"): + i = m.end() + continue + # NOTE: do NOT skip on a trailing `=` — `int b = add(...)` is a + # call site, not a declaration; declarations are filtered later + # by the return-type-prefix check against _is_declaration_prefix. + if "typedef" in prefix_stripped: + i = m.end() + continue + + # match parens to find the argument list + paren_open = m.end() - 1 + depth = 1 + j = paren_open + 1 + while j < n and depth > 0: + c = stripped[j] + if c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth == 0: + break + j += 1 + if depth != 0: + i = m.end() + continue + args_str = stripped[paren_open + 1:j] + + # skip declaration/definition contexts: the prefix (text between + # the line start and the callee name) must look like a return + # type, e.g. `void f(...)` / `int add(...)` / `unsigned long + # f(...)`. A prefix ending in `)` (`(cast) f(...)`, `if (x) f(...)`) + # or `=` (`int b = add(...)`) or a statement keyword (`return + # f(...)`) is a call site and is NOT skipped — _is_declaration_prefix + # already encodes the keyword/punctuation part of this. + if not prefix_stripped.endswith(")") \ + and _is_declaration_prefix(prefix_stripped): + i = j + 1 + continue + + # scan arguments for `?`, `&&`, `||` at this call's own depth: + # parens/brackets/braces dive one level; glued template angle + # brackets (`Foo`) dive too, while spaced comparison + # operators (`a < b`) do not + arg_depth = 0 + complex_hit = False + k = 0 + while k < len(args_str): + c = args_str[k] + if c == "<": + # template opener only when glued to the preceding word + # (`Foo`); a spaced `<` is a comparison (`a < b`) + # and `<<` is a shift operator + if ( + arg_depth == 0 + and k > 0 + and (args_str[k - 1].isalnum() + or args_str[k - 1] in "_<") + and k + 1 < len(args_str) + and args_str[k + 1] not in " \t\n<" + ): + arg_depth += 1 + elif c == ">": + # template closer only when glued to the preceding word; + # a spaced `>` is a comparison (`a > b`), `->` is a + # member access and `>=`/`>>=m` are operators + if ( + arg_depth > 0 + and k > 0 + and (args_str[k - 1].isalnum() + or args_str[k - 1] in "_<>") + and (k + 1 >= len(args_str) + or args_str[k + 1] in "(),;&*> \t\n") + ): + arg_depth -= 1 + elif c in "([{": + arg_depth += 1 + elif c in ")]}": + arg_depth = max(0, arg_depth - 1) + elif arg_depth == 0: + if c == "?": + complex_hit = True + break + if (c == "&" or c == "|") and k + 1 < len(args_str) \ + and args_str[k + 1] == c: + complex_hit = True + break + k += 1 + + if complex_hit: + line_no = stripped[:paren_open].count("\n") + 1 + hits.append((line_no, name)) + + # resume INSIDE the argument list so nested callees (e.g. the + # `g` of `f(g(a ? b : c))`) are visited as their own calls + i = paren_open + 1 + + return hits + + +def find_ifdef_in_call_args(content: str) -> List[int]: + """Find `#if`/`#ifdef`/`#ifndef` directives that sit inside a + function call's argument list (conditional compilation wrapping + individual arguments). + + A single pass tracks a paren stack: each `(` opener is classified + as a *call* context or an *other* context (control-statement header + `if (/for (/while (/switch (`, declaration parameter list, etc.). + When a `#if`-family directive line is reached while the innermost + open paren is a call context, the directive is inside that call's + argument list. `#endif` lines are not penalised (counting the + opener once is enough) and carry no parens, so they are skipped. + + Runs on comment-stripped (but NOT string-stripped) content so + preprocessor lines survive; string literals virtually never contain + a line-start `#`. + + Returns a list of 1-indexed line numbers of offending directives. + """ + stripped = strip_comments(content) + hits: List[int] = [] + # stack entries: True -> paren opened by a function call + # False -> paren opened by something else + paren_stack: List[bool] = [] + + def _classify_opener(text_before: str) -> bool: + """Decide whether `(` following `text_before` opens a call's + argument list.""" + s = text_before.rstrip() + if not s: + return False + m = _LAST_IDENT_RE.search(s) + if not m: + return False + # control-statement header: `if (`, `for (`, `while (`, + # `switch (`, `sizeof(` ... — the keyword itself is the last + # identifier, with no callee between it and `(` + if m.group(1) in STATEMENT_CONTEXT_KEYWORDS: + return False + # strip the callee name and inspect what precedes it + before = s[:m.start()].rstrip() + if not before: + # bare callee in statement position: `f(` — a call (a + # declaration would carry a return type before the name) + return True + # method call: `obj.m(`, `ptr->m(` + if before.endswith(".") or before.endswith("->"): + return True + # assignment-init: `... = name(` — `name` is the CALLEE and the + # whole expression is a call, regardless of any preceding + # declaration-looking tokens. Without this, a cross-line tail + # like `int f = add` would be misread as the declaration prefix + # `type=int f =` + `name=add`. + if before.endswith("="): + return True + mb = _LAST_IDENT_RE.search(before) + if mb and mb.group(1) in STATEMENT_CONTEXT_KEYWORDS: + # expression context: `return f(`, `else f(`, `throw f(` + return True + # `type name(` — declaration/definition parameter list + return False + + line_no = 0 + # rolling window of recent non-preprocessor code so that a callee + # name ending a previous line (`int f = add(`) is still visible when + # a subsequent `(` or `#if` line is classified. Updated at the end + # of EVERY non-preprocessor line, NOT just after the paren scan. + prev_line_tail = "" + for line in stripped.split("\n"): + line_no += 1 + if PREPROC_IF_RE.match(line): + if paren_stack and paren_stack[-1]: + hits.append(line_no) + continue + if PREPROC_ENDIF_RE.match(line): + continue + # other preprocessor lines (`#else`, `#elif`, `#include` ...) + # carry no argument-list parens; skip them so they never + # disturb the stack. Do NOT touch prev_line_tail here: the code + # line before the `#if`-block is what we still need to see. + if line.lstrip().startswith("#"): + continue + for idx, c in enumerate(line): + if c == "(": + paren_stack.append( + _classify_opener(prev_line_tail + line[:idx])) + elif c == ")": + if paren_stack: + paren_stack.pop() + prev_line_tail = (prev_line_tail + "\n" + line).rstrip()[-40:] + return hits + + +def analyze_class_blocks( + content: str, +) -> Tuple[List[Finding], List[Finding], List[Finding], List[Finding]]: + """Analyze class/struct blocks for public member variables, static member + variables, long member functions, and member/local name conflicts. + + Returns (public_member_findings, static_member_findings, + long_function_findings, name_conflict_findings). + """ + pub_findings: List[Finding] = [] + static_findings: List[Finding] = [] + long_func_findings: List[Finding] = [] + conflict_findings: List[Finding] = [] + + stripped = strip_comments(content) + lines = stripped.split("\n") + + for start_line, end_line, kind, name in find_class_blocks(stripped): + default_access = "private" if kind == "class" else "public" + body_start = start_line - 1 + body_end = end_line - 1 + class_lines = lines[body_start:body_end + 1] + + # pre-compute access per line (for public member variable rule) + access_per_line: List[str] = [] + cur_access = default_access + for k in range(len(class_lines)): + m = ACCESS_RE.match(class_lines[k]) + if m: + cur_access = m.group(1) + access_per_line.append(cur_access) + + # pass 0: identify lines belonging to multi-line declarations that + # start with keywords such as `using`, `typedef`, `template`, etc. + # Continuation lines (e.g. the "= std::vector;" half of a + # two-line `using value_type = ...;` alias) would otherwise be + # treated as standalone member declarations when the per-line + # heuristics run below. + continued_idxs = _mark_continued_decl_lines(class_lines) + + # pass 1: collect all member variable names (any access section). + # depth starts at 0; the class header line brings it to 1. Only lines + # that begin AND end at depth 1 are class-body member declarations + # (lines that open a function body bring depth from 1 to 2). + member_var_names: set = set() + depth = 0 + for idx, line in enumerate(class_lines): + prev_depth = depth + depth += line.count("{") - line.count("}") + if prev_depth == 1 and depth == 1 and idx not in continued_idxs: + var_name = _match_var_decl(line) + if var_name: + member_var_names.add(var_name) + + # pass 2: walk class body for findings + depth = 0 + func_start_abs = -1 + for idx in range(len(class_lines)): + line = class_lines[idx] + abs_line = body_start + idx + 1 # 1-indexed absolute line + prev_depth = depth + + # public member variable: only at depth 1 (class body, not in a + # function). Lines flagged as continuations of an earlier + # multi-line declaration are skipped regardless of how they + # look in isolation — e.g. ` = std::vector;` after a + # two-line `using value_type` alias must not count as a member. + # Only `class` bodies are penalised: in C++ a `struct` has + # public access by default and public data members are a + # legitimate, intended usage (POD aggregate, value types, + # mixin tags). Treating them as a code-quality issue would + # penalise perfectly idiomatic C++. + if prev_depth == 1 and access_per_line[idx] == "public" \ + and kind == "class" \ + and idx not in continued_idxs: + if is_public_member_var(line): + pub_findings.append(Finding( + rule="public_member_variable", + line=abs_line, + reason=f"public member in {kind} {name}: {line.strip()}", + deduction=WEIGHTS["public_member_variable"], + )) + + # static member variable: depth 1, any access section. + # Excludes static_assert, static member functions, and + # static_cast via is_static_member_var's heuristic. Continued + # multi-line declarations are skipped for the same reason + # documented above for the public-member rule. + if prev_depth == 1 and idx not in continued_idxs \ + and is_static_member_var(line): + static_findings.append(Finding( + rule="static_member_variable", + line=abs_line, + reason=f"static member in {kind} {name}: {line.strip()}", + deduction=WEIGHTS["static_member_variable"], + )) + + # member/local name conflict: only at depth >= 2 (function body) + if prev_depth >= 2: + var_name = _match_var_decl(line) + if var_name and var_name in member_var_names: + conflict_findings.append(Finding( + rule="member_local_name_conflict", + line=abs_line, + reason=( + f"local variable '{var_name}' in {kind} {name} " + f"shadows a member variable" + ), + deduction=WEIGHTS["member_local_name_conflict"], + )) + + # apply brace counting for this line + depth = prev_depth + line.count("{") - line.count("}") + + # function body start: transition from depth 1 to >=2 + if prev_depth == 1 and depth >= 2 and func_start_abs < 0: + func_start_abs = abs_line + # function body end: transition from >=2 back to 1 + if prev_depth >= 2 and depth == 1 and func_start_abs > 0: + func_end_abs = abs_line + func_lines = func_end_abs - func_start_abs + 1 + if func_lines > FUNCTION_LENGTH_THRESHOLD: + excess = func_lines - FUNCTION_LENGTH_THRESHOLD + blocks = (excess + FUNCTION_LENGTH_STEP - 1) // FUNCTION_LENGTH_STEP + long_func_findings.append(Finding( + rule="member_function_too_long", + line=func_start_abs, + reason=( + f"member function in {kind} {name} spans {func_lines} lines " + f"(exceeds {FUNCTION_LENGTH_THRESHOLD})" + ), + deduction=blocks * WEIGHTS["member_function_too_long"], + )) + func_start_abs = -1 + + return pub_findings, static_findings, long_func_findings, conflict_findings diff --git a/tools/03_code_analysis/cqs_text.py b/tools/03_code_analysis/cqs_text.py new file mode 100644 index 0000000000..dab1a7eaa1 --- /dev/null +++ b/tools/03_code_analysis/cqs_text.py @@ -0,0 +1,283 @@ +"""Text-level helpers for the ABACUS code-quality scorer. + +Comment/string stripping that preserves line structure, comment-block +extraction for documentation rules, and include-guard helpers. +""" + +import re +from typing import List, Tuple + + +def _is_digit_separator(content: str, quote_pos: int) -> bool: + """Return True if the single-quote at `quote_pos` is a C++14 digit + separator (e.g. the middle `'` in `1'000'000`). + + Heuristic: a digit separator must be preceded and followed by a + character that can belong to a numeric literal — digits, + hexadecimal letters (a–f, A–F) for hex literals, or the base/type + suffix letters (uUlLbB) that typically attach to numbers after the + body. Floating-point '.' can also appear on either side (e.g. + `1'000.5` or `1.000'5`). Any other previous/next char means the + quote starts a character literal. + """ + n = len(content) + if quote_pos <= 0 or quote_pos >= n - 1: + return False + prev = content[quote_pos - 1] + nxt = content[quote_pos + 1] + numeric_chars = set("0123456789abcdefABCDEFuUlLbBxXoO.") + return prev in numeric_chars and nxt in numeric_chars + + +def strip_comments(content: str) -> str: + """Return content with comments replaced by spaces, preserving line numbers. + + Handles // line comments and /* */ block comments. String literals are + respected so that '/' inside strings is not mistaken for a comment. + + C++14 digit separators such as `1'000'000` are NOT treated as character + literals: the parser inspects the characters on both sides of a `'` and + only enters character-literal mode when the quote is genuinely opening + a char literal like `'x'` or `'\\n'`. + """ + out = [] + i = 0 + n = len(content) + in_string = False + in_char = False + while i < n: + c = content[i] + if in_string: + out.append(c) + if c == "\\" and i + 1 < n: + out.append(content[i + 1]) + i += 2 + continue + if c == '"': + in_string = False + i += 1 + continue + if in_char: + out.append(c) + if c == "\\" and i + 1 < n: + out.append(content[i + 1]) + i += 2 + continue + if c == "'": + in_char = False + i += 1 + continue + if c == '"': + in_string = True + out.append(c) + i += 1 + continue + if c == "'": + if _is_digit_separator(content, i): + out.append(c) + i += 1 + continue + in_char = True + out.append(c) + i += 1 + continue + if c == "/" and i + 1 < n: + if content[i + 1] == "/": + # line comment until newline + j = content.find("\n", i) + if j < 0: + j = n + out.append(" " * (j - i)) + i = j + continue + if content[i + 1] == "*": + # block comment until */ + j = content.find("*/", i + 2) + if j < 0: + j = n + else: + j += 2 + block = content[i:j] + # preserve newlines + out.append(re.sub(r"[^\n]", " ", block)) + i = j + continue + out.append(c) + i += 1 + return "".join(out) + + +def strip_strings(content: str) -> str: + """Return content with string and character literals erased to spaces, + preserving line numbers. + + The result still has the same number of characters and lines, but no + letter/word survives inside "..." or '...'. This prevents false matches + on keywords that happen to appear inside string literals. + + C++14 digit separators such as `1'000'000` are left alone: the parser + inspects the characters on both sides of a `'` and only enters + character-literal mode when the quote genuinely opens a char literal + like `'x'` or `'\\n'`. + """ + out = [] + i = 0 + n = len(content) + in_string = False + in_char = False + while i < n: + c = content[i] + if in_string: + if c == "\\" and i + 1 < n: + # keep the escape sequence length but blank it out + out.append(" ") + if content[i + 1] == "\n": + out.append("\n") + else: + out.append(" ") + i += 2 + continue + if c == '"': + in_string = False + out.append('"') # keep the boundary marker (safest for line numbers) + i += 1 + continue + if c == "\n": + out.append("\n") + else: + out.append(" ") + i += 1 + continue + if in_char: + if c == "\\" and i + 1 < n: + out.append(" ") + if content[i + 1] == "\n": + out.append("\n") + else: + out.append(" ") + i += 2 + continue + if c == "'": + in_char = False + out.append("'") + i += 1 + continue + if c == "\n": + out.append("\n") + else: + out.append(" ") + i += 1 + continue + if c == '"': + in_string = True + out.append('"') + i += 1 + continue + if c == "'": + if _is_digit_separator(content, i): + out.append(c) + i += 1 + continue + in_char = True + out.append("'") + i += 1 + continue + out.append(c) + i += 1 + return "".join(out) + + +def extract_comment_blocks(content: str) -> List[List[str]]: + """Extract normalised comment lines, grouped by comment block. + + Returns a list of blocks; each block is a list of normalised comment + lines (comment markers stripped, whitespace collapsed, empty lines + dropped). A 'block' is a maximal run of `///` lines, or one `/* ... */` + region. String literals are respected so that `//` inside a string is + not treated as a comment opener. + """ + blocks: List[List[str]] = [] + current: List[str] = [] + i = 0 + n = len(content) + in_string = False + in_char = False + + def flush() -> None: + nonlocal current + if current: + blocks.append(current) + current = [] + + while i < n: + c = content[i] + if in_string: + if c == "\\" and i + 1 < n: + i += 2 + continue + if c == '"': + in_string = False + i += 1 + continue + if in_char: + if c == "\\" and i + 1 < n: + i += 2 + continue + if c == "'": + in_char = False + i += 1 + continue + if c == '"': + in_string = True + i += 1 + continue + if c == "'": + if _is_digit_separator(content, i): + i += 1 + continue + in_char = True + i += 1 + continue + if c == "/" and i + 1 < n and content[i + 1] == "/": + # line comment: collect the text after // (or /// for doxygen) + j = content.find("\n", i) + if j < 0: + j = n + text = content[i + 2:j].lstrip("/ \t").strip() + if text: + current.append(" ".join(text.split())) + i = j + continue + if c == "/" and i + 1 < n and content[i + 1] == "*": + # block comment: collect until */ + j = content.find("*/", i + 2) + if j < 0: + j = n - 2 + body = content[i + 2:j] + block: List[str] = [] + for ln in body.split("\n"): + # strip leading comment markers: '*', '/', whitespace, and + # any leftover '///' from doxygen-style lines + ln = re.sub(r"^[\s*/]+", "", ln).strip() + if ln: + block.append(" ".join(ln.split())) + if block: + flush() + blocks.append(block) + i = j + 2 + continue + if c == "\n": + # a blank/non-comment line ends a /// run; check next non-empty + # cheaply by flushing on newline only when current is non-empty + # and the following line does not start with optional ws + // + rest = content[i + 1:i + 200] + m = re.match(r"\s*//", rest) + if not m: + flush() + i += 1 + continue + if not c.isspace(): + flush() + i += 1 + flush() + return blocks