Skip to content

Commit a431447

Browse files
committed
gh-153568: Parse rule flags in the Rule constructor
Expose memo and fastpath attributes for the C generator and simplify the atom fast-path comment.
1 parent 071ed26 commit a431447

6 files changed

Lines changed: 85 additions & 21 deletions

File tree

‎Lib/test/test_peg_generator/test_c_generator.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ def test_parser_plan_does_not_depend_on_compilation_state(self):
6363
@modulename 'sample'
6464
@bytecode '1'
6565
start[mod_ty]: expr_without_invalid 'pass' "zsoft" "asoft" ('bb' | 'aa')* ENDMARKER
66-
expr_without_invalid[expr_ty] (memo): name=expr [NUMBER] { name }
66+
expr_without_invalid[expr_ty] (memo, fastpath=custom_fastpath): name=expr [NUMBER] { name }
6767
expr[expr_ty]: expr '+' NAME | NAME
6868
""")
6969
generator.debug = True
@@ -86,6 +86,8 @@ def test_parser_plan_does_not_depend_on_compilation_state(self):
8686
rule.name = "changed"
8787
rule.type = "changed_type"
8888
rule.flags = frozenset()
89+
rule.memo = False
90+
rule.fastpath = None
8991
rule.rhs.alts[0].action = "changed_action"
9092
rule.rhs.alts.clear()
9193
generator.rules.clear()

‎Lib/test/test_peg_generator/test_c_parser.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,47 @@ def test_rule_cleanup(self) -> None:
464464
"""
465465
self.run_test(grammar_source, test_source)
466466

467+
def test_rule_fastpath(self) -> None:
468+
grammar_source = """
469+
@subheader '''
470+
#define CHECK_INVALID(expected) \\
471+
(assert(p->call_invalid_rules == (expected)), _PyPegen_dummy_name(p))
472+
473+
static int
474+
number_fastpath(Parser *p, void *result)
475+
{
476+
assert(p->call_invalid_rules == 0);
477+
expr_ty number = _PyPegen_number_token(p);
478+
if (number != NULL) {
479+
*(void **)result = number;
480+
return 1;
481+
}
482+
if (_PyPegen_expect_token(p, LPAR)) {
483+
p->error_indicator = 1;
484+
PyErr_SetString(PyExc_ValueError, "fastpath error");
485+
}
486+
if (p->error_indicator) {
487+
*(void **)result = NULL;
488+
return 1;
489+
}
490+
return 0;
491+
}
492+
'''
493+
start: enable value_without_invalid after NEWLINE ENDMARKER
494+
enable: 'enable' { (p->call_invalid_rules = 1, _PyPegen_dummy_name(p)) }
495+
value_without_invalid (memo, fastpath=number_fastpath): NAME { CHECK_INVALID(0) }
496+
after: NAME { CHECK_INVALID(1) }
497+
"""
498+
test_source = """
499+
self.check_input_strings_for_grammar(
500+
valid_cases=['enable 42 tail', 'enable name tail'],
501+
invalid_cases=['enable + tail', 'enable 42'],
502+
)
503+
with self.assertRaisesRegex(ValueError, 'fastpath error'):
504+
parse.parse_string('enable ( tail', mode=0)
505+
"""
506+
self.run_test(grammar_source, test_source)
507+
467508
def test_left_recursive_rule_cleanup(self) -> None:
468509
grammar_source = """
469510
@subheader '''

‎Lib/test/test_peg_generator/test_pegen.py‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1181,32 +1181,53 @@ def test_rule_flags(self) -> None:
11811181

11821182
# Test memo-only rule
11831183
simple_rule = rules['simple_rule']
1184-
self.assertTrue('memo' in simple_rule.flags,
1184+
self.assertTrue(simple_rule.memo,
11851185
"simple_rule should have memo")
11861186
self.assertEqual(simple_rule.flags, frozenset(['memo']),
11871187
f"simple_rule flags should be {'memo'}, got {simple_rule.flags}")
11881188

11891189
# Test multi-flag rule
11901190
multi_flag_rule = rules['multi_flag_rule']
1191-
self.assertTrue('memo' in simple_rule.flags,
1191+
self.assertTrue(multi_flag_rule.memo,
11921192
"multi_flag_rule should have memo")
11931193
self.assertEqual(multi_flag_rule.flags, frozenset({'memo', 'custom', 'test'}),
11941194
f"multi_flag_rule flags should contain memo, custom, test, got {multi_flag_rule.flags}")
11951195

11961196
# Test single custom flag rule
11971197
single_custom_rule = rules['single_custom_flag']
1198-
self.assertFalse('memo' not in simple_rule.flags,
1198+
self.assertFalse(single_custom_rule.memo,
11991199
"single_custom_flag should not have memo")
12001200
self.assertEqual(single_custom_rule.flags, frozenset(['custom']),
12011201
f"single_custom_flag flags should be {'custom'}, got {single_custom_rule.flags}")
12021202

12031203
# Test no flags rule
12041204
no_flags_rule = rules['no_flags_rule']
1205-
self.assertFalse('memo' not in simple_rule.flags,
1205+
self.assertFalse(no_flags_rule.memo,
12061206
"no_flags_rule should not have memo")
12071207
self.assertEqual(no_flags_rule.flags, frozenset(),
12081208
f"no_flags_rule flags should be the empty set, got {no_flags_rule.flags}")
12091209

1210+
for rule in rules.values():
1211+
self.assertIsNone(rule.fastpath)
1212+
1213+
def test_rule_fastpath_flags(self) -> None:
1214+
for rule_type in ("", "[expr_ty]"):
1215+
for memo in (False, True):
1216+
with self.subTest(rule_type=rule_type, memo=memo):
1217+
flags = "fastpath=custom_fastpath"
1218+
if memo:
1219+
flags = "memo, " + flags
1220+
grammar_source = f"start{rule_type} ({flags}): NAME\n"
1221+
rule = parse_string(grammar_source, GrammarParser).rules["start"]
1222+
self.assertEqual(rule.fastpath, "custom_fastpath")
1223+
self.assertIs(rule.memo, memo)
1224+
1225+
def test_rule_fastpath_requires_function(self) -> None:
1226+
with self.assertRaisesRegex(
1227+
ValueError, "rule 'start': fastpath flag needs a function"
1228+
):
1229+
parse_string("start (fastpath): NAME\n", GrammarParser)
1230+
12101231

12111232
class TestGrammarIncludes(unittest.TestCase):
12121233
def setUp(self) -> None:

‎Parser/pegen.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -327,8 +327,8 @@ _resize_tokens_array(Parser *p) {
327327
return 0;
328328
}
329329

330-
// Fast path attached to the expression precedence chain's entry rules via
331-
// the (fastpath=function) rule flag in the grammar. A bare NAME/NUMBER
330+
// Fast path attached to the expression precedence chain's entry rules.
331+
// A bare NAME/NUMBER
332332
// atom followed by a token that cannot start or continue a binary/postfix
333333
// expression is a complete expression, so it is built directly instead of
334334
// descending the whole chain. Returns 1 if it produced a result (stored

‎Tools/peg_generator/pegen/c_generator_calls.py‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,7 @@ def candidate(alt: Alt) -> Rule | None:
381381
rule = original_rules.get(alt.items[0].item.value)
382382
if rule is None or rule.name not in consuming:
383383
return None
384-
if ("memo" in rule.flags and not rule.left_recursive) or (
384+
if (rule.memo and not rule.left_recursive) or (
385385
rule.left_recursive and rule.leader
386386
):
387387
return rule
@@ -409,17 +409,6 @@ def candidate(alt: Alt) -> Rule | None:
409409
i = j
410410
self._prefixes[rule.name] = tuple(prefixes)
411411

412-
def _rule_fast_path(self, rule: Rule) -> str | None:
413-
for flag in rule.flags:
414-
name, _, func = flag.partition("=")
415-
if name == "fastpath":
416-
if not func:
417-
raise ValueError(
418-
f"rule {rule.name!r}: fastpath flag needs a function"
419-
)
420-
return func
421-
return None
422-
423412
def prepare_rule(
424413
self, rule: Rule, *, skip_actions: bool = False, shared: bool = False
425414
) -> CRule:
@@ -437,8 +426,8 @@ def prepare_rule(
437426
),
438427
left_recursive=rule.left_recursive,
439428
leader=rule.leader,
440-
memoize="memo" in rule.flags and not rule.left_recursive,
441-
fastpath=self._rule_fast_path(rule),
429+
memoize=rule.memo and not rule.left_recursive,
430+
fastpath=rule.fastpath,
442431
disable_invalid_rules=rule.name.endswith("without_invalid"),
443432
prefixes=self._prefixes.get(rule.name, ()),
444433
source=rule.source,

‎Tools/peg_generator/pegen/grammar.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,17 @@ def __init__(
9090
self.type = type
9191
self.rhs = rhs
9292
self.flags = flags or frozenset()
93+
self.memo = "memo" in self.flags
94+
self.fastpath: str | None = None
95+
for flag in self.flags:
96+
name, _, function = flag.partition("=")
97+
if name == "fastpath":
98+
if not function:
99+
raise ValueError(
100+
f"rule {self.name!r}: fastpath flag needs a function"
101+
)
102+
self.fastpath = function
103+
break
93104
self.kind = kind
94105
# The included grammar file that defines this rule, relative to the
95106
# root grammar file's directory, or None for the root grammar file itself.

0 commit comments

Comments
 (0)