Skip to content

Commit 434ad98

Browse files
committed
uop peephole cleanup
1 parent aeab76c commit 434ad98

2 files changed

Lines changed: 109 additions & 121 deletions

File tree

‎Lib/test/test_capi/test_opt.py‎

Lines changed: 42 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -5584,6 +5584,8 @@ def testfunc(*args):
55845584
uops = get_opnames(ex)
55855585

55865586
self.assertIn("_BINARY_OP_SUBSCR_INIT_CALL", uops)
5587+
# The inlined return proves the optimizer completed the callee frame.
5588+
self.assertIn("_RETURN_VALUE", uops)
55875589
self.assertNotIn("_POP_TOP_NOP", uops)
55885590

55895591
def test_load_attr_property_frame(self):
@@ -6230,8 +6232,8 @@ def negated_constant(n):
62306232
self.assertEqual(res, TIER2_THRESHOLD)
62316233
self.assertIsNotNone(ex)
62326234
uops = get_opnames(ex)
6235+
self.assertNotIn("_UNARY_NEGATIVE", uops)
62336236
self.assertNotIn("_SWAP", uops)
6234-
self.assertNotIn("_SWAP_2", uops)
62356237

62366238
def test_peephole_removes_redundant_rrot(self):
62376239
def add(a, b):
@@ -6247,25 +6249,48 @@ def testfunc(n):
62476249
self.assertEqual(res, 3 * TIER2_THRESHOLD)
62486250
self.assertIsNotNone(ex)
62496251
uops = get_opnames(ex)
6252+
self.assertIn("_RETURN_VALUE", uops)
6253+
self.assertEqual(count_ops(ex, "_BINARY_OP_ADD_INT"), 1)
62506254
self.assertNotIn("_RROT_3", uops)
62516255

6252-
def test_combine_adjacent_int_guards(self):
6253-
def testfunc(left, right, loops):
6254-
for _ in range(loops):
6255-
result = left + right
6256-
return result
6257-
6258-
result = testfunc(40, 2, TIER2_THRESHOLD)
6256+
def test_combine_adjacent_guards(self):
6257+
class StrSubclass(str):
6258+
pass
62596259

6260-
self.assertEqual(result, 42)
6261-
ex = get_first_executor(testfunc)
6262-
self.assertIsNotNone(ex)
6263-
uops = get_opnames(ex)
6264-
self.assertIn("_GUARD_TOS_AND_NOS_INT", uops)
6265-
self.assertNotIn("_GUARD_TOS_INT", uops)
6266-
self.assertNotIn("_GUARD_NOS_INT", uops)
6267-
self.assertEqual(testfunc(40.0, 2.0, 1), 42.0)
6268-
self.assertEqual(testfunc(1 << 100, 2, 1), (1 << 100) + 2)
6260+
cases = (
6261+
(int, (40, 2), ((40, 2.0), (40.0, 2), (1 << 100, 2)),
6262+
"_GUARD_TOS_AND_NOS_INT"),
6263+
(float, (1.5, 2.5), ((1.5, 2), (1, 2.5)),
6264+
"_GUARD_TOS_AND_NOS_FLOAT"),
6265+
(str, ("a", "b"),
6266+
(("a", StrSubclass("b")), (StrSubclass("a"), "b")),
6267+
"_GUARD_TOS_AND_NOS_UNICODE"),
6268+
)
6269+
for (
6270+
operand_type, good_operands, deopt_operands, combined_guard,
6271+
) in cases:
6272+
with self.subTest(operand_type=operand_type.__name__):
6273+
def testfunc(left, right, loops):
6274+
for _ in range(loops):
6275+
result = left + right
6276+
return result
6277+
6278+
with clear_executors(testfunc):
6279+
left, right = good_operands
6280+
result = testfunc(left, right, TIER2_THRESHOLD)
6281+
6282+
self.assertEqual(result, left + right)
6283+
ex = get_first_executor(testfunc)
6284+
self.assertIsNotNone(ex)
6285+
uops = get_opnames(ex)
6286+
suffix = combined_guard.removeprefix(
6287+
"_GUARD_TOS_AND_NOS_"
6288+
)
6289+
self.assertIn(combined_guard, uops)
6290+
self.assertNotIn(f"_GUARD_TOS_{suffix}", uops)
6291+
self.assertNotIn(f"_GUARD_NOS_{suffix}", uops)
6292+
for left, right in deopt_operands:
6293+
self.assertEqual(testfunc(left, right, 1), left + right)
62696294

62706295
def test_combine_adjacent_overflowed_guards(self):
62716296
def testfunc(start, loops):
@@ -6285,40 +6310,6 @@ def testfunc(start, loops):
62856310
self.assertNotIn("_GUARD_NOS_OVERFLOWED", uops)
62866311
self.assertEqual(testfunc(1 << 100, 1), -1)
62876312

6288-
def test_combine_adjacent_float_guards(self):
6289-
def testfunc(left, right, loops):
6290-
for _ in range(loops):
6291-
result = left + right
6292-
return result
6293-
6294-
result = testfunc(1.5, 2.5, TIER2_THRESHOLD)
6295-
6296-
self.assertEqual(result, 4.0)
6297-
ex = get_first_executor(testfunc)
6298-
self.assertIsNotNone(ex)
6299-
uops = get_opnames(ex)
6300-
self.assertIn("_GUARD_TOS_AND_NOS_FLOAT", uops)
6301-
self.assertNotIn("_GUARD_TOS_FLOAT", uops)
6302-
self.assertNotIn("_GUARD_NOS_FLOAT", uops)
6303-
self.assertEqual(testfunc(1, 2, 1), 3)
6304-
6305-
def test_combine_adjacent_unicode_guards(self):
6306-
def testfunc(left, right, loops):
6307-
for _ in range(loops):
6308-
result = left + right
6309-
return result
6310-
6311-
result = testfunc("a", "b", TIER2_THRESHOLD)
6312-
6313-
self.assertEqual(result, "ab")
6314-
ex = get_first_executor(testfunc)
6315-
self.assertIsNotNone(ex)
6316-
uops = get_opnames(ex)
6317-
self.assertIn("_GUARD_TOS_AND_NOS_UNICODE", uops)
6318-
self.assertNotIn("_GUARD_TOS_UNICODE", uops)
6319-
self.assertNotIn("_GUARD_NOS_UNICODE", uops)
6320-
self.assertEqual(testfunc(b"a", b"b", 1), b"ab")
6321-
63226313
def global_identity(x):
63236314
return x
63246315

‎Python/optimizer_analysis.c‎

Lines changed: 67 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -704,13 +704,6 @@ const uint16_t op_without_push[MAX_UOP_ID + 1] = {
704704
[_PUSH_NULL] = _NOP,
705705
};
706706

707-
const bool op_skip[MAX_UOP_ID + 1] = {
708-
[_NOP] = true,
709-
[_CHECK_VALIDITY] = true,
710-
[_CHECK_PERIODIC] = true,
711-
[_SET_IP] = true,
712-
};
713-
714707
const uint16_t op_without_pop[MAX_UOP_ID + 1] = {
715708
[_POP_TOP] = _NOP,
716709
[_POP_TOP_NOP] = _NOP,
@@ -719,53 +712,46 @@ const uint16_t op_without_pop[MAX_UOP_ID + 1] = {
719712
[_POP_TOP_UNICODE] = _NOP,
720713
};
721714

715+
typedef struct {
716+
uint16_t nos_guard;
717+
uint16_t fused_guard;
718+
} GuardFusion;
719+
720+
static const GuardFusion guard_fusion[MAX_UOP_ID + 1] = {
721+
[_GUARD_TOS_INT] = {_GUARD_NOS_INT, _GUARD_TOS_AND_NOS_INT},
722+
[_GUARD_TOS_OVERFLOWED] = {
723+
_GUARD_NOS_OVERFLOWED, _GUARD_TOS_AND_NOS_OVERFLOWED},
724+
[_GUARD_TOS_FLOAT] = {_GUARD_NOS_FLOAT, _GUARD_TOS_AND_NOS_FLOAT},
725+
[_GUARD_TOS_UNICODE] = {
726+
_GUARD_NOS_UNICODE, _GUARD_TOS_AND_NOS_UNICODE},
727+
};
728+
722729
/* Type analysis has already processed both guards before this cleanup pass.
723-
* Require the same side exit so removing the NOS guard preserves deoptimization
724-
* behavior. */
725-
static bool
730+
* Only combine guards that are strictly adjacent because they come from the
731+
* same macro expansion. Require the same side exit so removing the NOS guard
732+
* preserves deoptimization behavior. */
733+
static void
726734
combine_matching_tos_nos_guards(
727735
_PyUOpInstruction *buffer, int pc, int buffer_size)
728736
{
729737
if (pc + 1 >= buffer_size) {
730-
return false;
731-
}
732-
int tos_opcode = buffer[pc].opcode;
733-
int expected_nos_opcode;
734-
int combined_opcode;
735-
switch (tos_opcode) {
736-
case _GUARD_TOS_INT:
737-
expected_nos_opcode = _GUARD_NOS_INT;
738-
combined_opcode = _GUARD_TOS_AND_NOS_INT;
739-
break;
740-
case _GUARD_TOS_OVERFLOWED:
741-
expected_nos_opcode = _GUARD_NOS_OVERFLOWED;
742-
combined_opcode = _GUARD_TOS_AND_NOS_OVERFLOWED;
743-
break;
744-
case _GUARD_TOS_FLOAT:
745-
expected_nos_opcode = _GUARD_NOS_FLOAT;
746-
combined_opcode = _GUARD_TOS_AND_NOS_FLOAT;
747-
break;
748-
case _GUARD_TOS_UNICODE:
749-
expected_nos_opcode = _GUARD_NOS_UNICODE;
750-
combined_opcode = _GUARD_TOS_AND_NOS_UNICODE;
751-
break;
752-
default:
753-
return false;
738+
return;
754739
}
755-
if (buffer[pc + 1].opcode != expected_nos_opcode ||
740+
GuardFusion fusion = guard_fusion[buffer[pc].opcode];
741+
if (fusion.nos_guard == 0 ||
742+
buffer[pc + 1].opcode != fusion.nos_guard ||
756743
uop_get_target(&buffer[pc]) != uop_get_target(&buffer[pc + 1]))
757744
{
758-
return false;
745+
return;
759746
}
760-
buffer[pc].opcode = combined_opcode;
747+
buffer[pc].opcode = fusion.fused_guard;
761748
buffer[pc + 1].opcode = _NOP;
762-
return true;
763749
}
764750

765751
static int
766-
previous_non_skip_uop(_PyUOpInstruction *buffer, int pc)
752+
previous_peephole_uop(_PyUOpInstruction *buffer, int pc)
767753
{
768-
while (pc >= 0 && op_skip[buffer[pc].opcode]) {
754+
while (pc >= 0 && buffer[pc].opcode == _NOP) {
769755
pc--;
770756
}
771757
return pc;
@@ -778,14 +764,15 @@ previous_non_skip_uop(_PyUOpInstruction *buffer, int pc)
778764
static bool
779765
remove_folded_stack_shuffle(_PyUOpInstruction *buffer, int pc, int arity)
780766
{
767+
assert(arity == 2 || arity == 3);
781768
int pops[2];
782769
int idx = pc;
783770
for (int i = 0; i < arity - 1; i++) {
784-
if (idx < 0 || !op_without_pop[buffer[idx].opcode]) {
771+
if (idx < 0 || op_without_pop[buffer[idx].opcode] != _NOP) {
785772
return false;
786773
}
787774
pops[i] = idx;
788-
idx = previous_non_skip_uop(buffer, idx - 1);
775+
idx = previous_peephole_uop(buffer, idx - 1);
789776
}
790777
int shuf = idx;
791778
if (shuf < 0) {
@@ -801,12 +788,12 @@ remove_folded_stack_shuffle(_PyUOpInstruction *buffer, int pc, int arity)
801788

802789
int dead_pushes[2];
803790
for (int i = 0; i < arity; i++) {
804-
idx = previous_non_skip_uop(buffer, idx - 1);
791+
idx = previous_peephole_uop(buffer, idx - 1);
805792
if (idx < 0) {
806793
return false;
807794
}
808795
uint16_t push_op = buffer[idx].opcode;
809-
if (push_op == _COPY || !op_without_push[push_op]) {
796+
if (push_op == _COPY || op_without_push[push_op] != _NOP) {
810797
return false;
811798
}
812799
if (i > 0) {
@@ -827,15 +814,42 @@ remove_folded_stack_shuffle(_PyUOpInstruction *buffer, int pc, int arity)
827814
static bool
828815
remove_adjacent_push_pop(_PyUOpInstruction *buffer, int pc)
829816
{
830-
int last = previous_non_skip_uop(buffer, pc - 1);
831-
if (last < 0 || !op_without_push[buffer[last].opcode]) {
817+
int last = previous_peephole_uop(buffer, pc - 1);
818+
if (last < 0) {
832819
return false;
833820
}
834-
buffer[last].opcode = _NOP;
835-
buffer[pc].opcode = _NOP;
821+
uint16_t push_replacement = op_without_push[buffer[last].opcode];
822+
uint16_t pop_replacement = op_without_pop[buffer[pc].opcode];
823+
if (push_replacement == 0 || pop_replacement == 0) {
824+
return false;
825+
}
826+
buffer[last].opcode = push_replacement;
827+
buffer[pc].opcode = pop_replacement;
836828
return true;
837829
}
838830

831+
/* Cancel out pushes and pops, repeatedly. So:
832+
* _LOAD_FAST + _POP_TOP + _POP_TOP + _LOAD_CONST_INLINE_BORROW + _POP_TOP
833+
* ...becomes:
834+
* _NOP + _NOP + _POP_TOP + _NOP + _NOP */
835+
static void
836+
peephole_uops(_PyUOpInstruction *buffer, int buffer_size)
837+
{
838+
for (int pc = 0; pc < buffer_size; pc++) {
839+
combine_matching_tos_nos_guards(buffer, pc, buffer_size);
840+
int opcode = buffer[pc].opcode;
841+
while (op_without_pop[opcode]) {
842+
if (remove_folded_stack_shuffle(buffer, pc, 2) ||
843+
remove_folded_stack_shuffle(buffer, pc, 3) ||
844+
remove_adjacent_push_pop(buffer, pc))
845+
{
846+
opcode = buffer[pc].opcode;
847+
continue;
848+
}
849+
break;
850+
}
851+
}
852+
}
839853

840854
static int
841855
remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
@@ -864,31 +878,9 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
864878
buffer[pc].opcode = _NOP;
865879
}
866880
break;
867-
case _GUARD_TOS_INT:
868-
case _GUARD_TOS_OVERFLOWED:
869-
case _GUARD_TOS_FLOAT:
870-
case _GUARD_TOS_UNICODE:
871-
if (combine_matching_tos_nos_guards(buffer, pc, buffer_size)) {
872-
opcode = buffer[pc].opcode;
873-
}
874-
_Py_FALLTHROUGH;
875881
case _EXIT_TRACE:
876882
default:
877883
{
878-
// Cancel out pushes and pops, repeatedly. So:
879-
// _LOAD_FAST + _POP_TOP + _POP_TOP + _LOAD_CONST_INLINE_BORROW + _POP_TOP
880-
// ...becomes:
881-
// _NOP + _NOP + _POP_TOP + _NOP + _NOP
882-
while (op_without_pop[opcode]) {
883-
if (remove_folded_stack_shuffle(buffer, pc, 2) ||
884-
remove_folded_stack_shuffle(buffer, pc, 3) ||
885-
remove_adjacent_push_pop(buffer, pc))
886-
{
887-
opcode = buffer[pc].opcode;
888-
continue;
889-
}
890-
break;
891-
}
892884
/* _PUSH_FRAME doesn't escape or error, but it
893885
* does need the IP for the return address */
894886
bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE || opcode == _DYNAMIC_EXIT || opcode == _EXIT_TRACE);
@@ -942,6 +934,11 @@ _Py_uop_analyze_and_optimize(
942934
length = remove_unneeded_uops(output, length);
943935
assert(length > 0);
944936

937+
peephole_uops(output, length);
938+
939+
length = remove_unneeded_uops(output, length);
940+
assert(length > 0);
941+
945942
OPT_STAT_INC(optimizer_successes);
946943
return length;
947944
}

0 commit comments

Comments
 (0)