@@ -704,13 +704,6 @@ const uint16_t op_without_push[MAX_UOP_ID + 1] = {
704704 [_PUSH_NULL ] = _NOP ,
705705};
706706
707- const bool op_skip [MAX_UOP_ID + 1 ] = {
708- [_NOP ] = true,
709- [_CHECK_VALIDITY ] = true,
710- [_CHECK_PERIODIC ] = true,
711- [_SET_IP ] = true,
712- };
713-
714707const uint16_t op_without_pop [MAX_UOP_ID + 1 ] = {
715708 [_POP_TOP ] = _NOP ,
716709 [_POP_TOP_NOP ] = _NOP ,
@@ -719,53 +712,46 @@ const uint16_t op_without_pop[MAX_UOP_ID + 1] = {
719712 [_POP_TOP_UNICODE ] = _NOP ,
720713};
721714
715+ typedef struct {
716+ uint16_t nos_guard ;
717+ uint16_t fused_guard ;
718+ } GuardFusion ;
719+
720+ static const GuardFusion guard_fusion [MAX_UOP_ID + 1 ] = {
721+ [_GUARD_TOS_INT ] = {_GUARD_NOS_INT , _GUARD_TOS_AND_NOS_INT },
722+ [_GUARD_TOS_OVERFLOWED ] = {
723+ _GUARD_NOS_OVERFLOWED , _GUARD_TOS_AND_NOS_OVERFLOWED },
724+ [_GUARD_TOS_FLOAT ] = {_GUARD_NOS_FLOAT , _GUARD_TOS_AND_NOS_FLOAT },
725+ [_GUARD_TOS_UNICODE ] = {
726+ _GUARD_NOS_UNICODE , _GUARD_TOS_AND_NOS_UNICODE },
727+ };
728+
722729/* Type analysis has already processed both guards before this cleanup pass.
723- * Require the same side exit so removing the NOS guard preserves deoptimization
724- * behavior. */
725- static bool
730+ * Only combine guards that are strictly adjacent because they come from the
731+ * same macro expansion. Require the same side exit so removing the NOS guard
732+ * preserves deoptimization behavior. */
733+ static void
726734combine_matching_tos_nos_guards (
727735 _PyUOpInstruction * buffer , int pc , int buffer_size )
728736{
729737 if (pc + 1 >= buffer_size ) {
730- return false;
731- }
732- int tos_opcode = buffer [pc ].opcode ;
733- int expected_nos_opcode ;
734- int combined_opcode ;
735- switch (tos_opcode ) {
736- case _GUARD_TOS_INT :
737- expected_nos_opcode = _GUARD_NOS_INT ;
738- combined_opcode = _GUARD_TOS_AND_NOS_INT ;
739- break ;
740- case _GUARD_TOS_OVERFLOWED :
741- expected_nos_opcode = _GUARD_NOS_OVERFLOWED ;
742- combined_opcode = _GUARD_TOS_AND_NOS_OVERFLOWED ;
743- break ;
744- case _GUARD_TOS_FLOAT :
745- expected_nos_opcode = _GUARD_NOS_FLOAT ;
746- combined_opcode = _GUARD_TOS_AND_NOS_FLOAT ;
747- break ;
748- case _GUARD_TOS_UNICODE :
749- expected_nos_opcode = _GUARD_NOS_UNICODE ;
750- combined_opcode = _GUARD_TOS_AND_NOS_UNICODE ;
751- break ;
752- default :
753- return false;
738+ return ;
754739 }
755- if (buffer [pc + 1 ].opcode != expected_nos_opcode ||
740+ GuardFusion fusion = guard_fusion [buffer [pc ].opcode ];
741+ if (fusion .nos_guard == 0 ||
742+ buffer [pc + 1 ].opcode != fusion .nos_guard ||
756743 uop_get_target (& buffer [pc ]) != uop_get_target (& buffer [pc + 1 ]))
757744 {
758- return false ;
745+ return ;
759746 }
760- buffer [pc ].opcode = combined_opcode ;
747+ buffer [pc ].opcode = fusion . fused_guard ;
761748 buffer [pc + 1 ].opcode = _NOP ;
762- return true;
763749}
764750
765751static int
766- previous_non_skip_uop (_PyUOpInstruction * buffer , int pc )
752+ previous_peephole_uop (_PyUOpInstruction * buffer , int pc )
767753{
768- while (pc >= 0 && op_skip [ buffer [pc ].opcode ] ) {
754+ while (pc >= 0 && buffer [pc ].opcode == _NOP ) {
769755 pc -- ;
770756 }
771757 return pc ;
@@ -778,14 +764,15 @@ previous_non_skip_uop(_PyUOpInstruction *buffer, int pc)
778764static bool
779765remove_folded_stack_shuffle (_PyUOpInstruction * buffer , int pc , int arity )
780766{
767+ assert (arity == 2 || arity == 3 );
781768 int pops [2 ];
782769 int idx = pc ;
783770 for (int i = 0 ; i < arity - 1 ; i ++ ) {
784- if (idx < 0 || ! op_without_pop [buffer [idx ].opcode ]) {
771+ if (idx < 0 || op_without_pop [buffer [idx ].opcode ] != _NOP ) {
785772 return false;
786773 }
787774 pops [i ] = idx ;
788- idx = previous_non_skip_uop (buffer , idx - 1 );
775+ idx = previous_peephole_uop (buffer , idx - 1 );
789776 }
790777 int shuf = idx ;
791778 if (shuf < 0 ) {
@@ -801,12 +788,12 @@ remove_folded_stack_shuffle(_PyUOpInstruction *buffer, int pc, int arity)
801788
802789 int dead_pushes [2 ];
803790 for (int i = 0 ; i < arity ; i ++ ) {
804- idx = previous_non_skip_uop (buffer , idx - 1 );
791+ idx = previous_peephole_uop (buffer , idx - 1 );
805792 if (idx < 0 ) {
806793 return false;
807794 }
808795 uint16_t push_op = buffer [idx ].opcode ;
809- if (push_op == _COPY || ! op_without_push [push_op ]) {
796+ if (push_op == _COPY || op_without_push [push_op ] != _NOP ) {
810797 return false;
811798 }
812799 if (i > 0 ) {
@@ -827,15 +814,42 @@ remove_folded_stack_shuffle(_PyUOpInstruction *buffer, int pc, int arity)
827814static bool
828815remove_adjacent_push_pop (_PyUOpInstruction * buffer , int pc )
829816{
830- int last = previous_non_skip_uop (buffer , pc - 1 );
831- if (last < 0 || ! op_without_push [ buffer [ last ]. opcode ] ) {
817+ int last = previous_peephole_uop (buffer , pc - 1 );
818+ if (last < 0 ) {
832819 return false;
833820 }
834- buffer [last ].opcode = _NOP ;
835- buffer [pc ].opcode = _NOP ;
821+ uint16_t push_replacement = op_without_push [buffer [last ].opcode ];
822+ uint16_t pop_replacement = op_without_pop [buffer [pc ].opcode ];
823+ if (push_replacement == 0 || pop_replacement == 0 ) {
824+ return false;
825+ }
826+ buffer [last ].opcode = push_replacement ;
827+ buffer [pc ].opcode = pop_replacement ;
836828 return true;
837829}
838830
831+ /* Cancel out pushes and pops, repeatedly. So:
832+ * _LOAD_FAST + _POP_TOP + _POP_TOP + _LOAD_CONST_INLINE_BORROW + _POP_TOP
833+ * ...becomes:
834+ * _NOP + _NOP + _POP_TOP + _NOP + _NOP */
835+ static void
836+ peephole_uops (_PyUOpInstruction * buffer , int buffer_size )
837+ {
838+ for (int pc = 0 ; pc < buffer_size ; pc ++ ) {
839+ combine_matching_tos_nos_guards (buffer , pc , buffer_size );
840+ int opcode = buffer [pc ].opcode ;
841+ while (op_without_pop [opcode ]) {
842+ if (remove_folded_stack_shuffle (buffer , pc , 2 ) ||
843+ remove_folded_stack_shuffle (buffer , pc , 3 ) ||
844+ remove_adjacent_push_pop (buffer , pc ))
845+ {
846+ opcode = buffer [pc ].opcode ;
847+ continue ;
848+ }
849+ break ;
850+ }
851+ }
852+ }
839853
840854static int
841855remove_unneeded_uops (_PyUOpInstruction * buffer , int buffer_size )
@@ -864,31 +878,9 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
864878 buffer [pc ].opcode = _NOP ;
865879 }
866880 break ;
867- case _GUARD_TOS_INT :
868- case _GUARD_TOS_OVERFLOWED :
869- case _GUARD_TOS_FLOAT :
870- case _GUARD_TOS_UNICODE :
871- if (combine_matching_tos_nos_guards (buffer , pc , buffer_size )) {
872- opcode = buffer [pc ].opcode ;
873- }
874- _Py_FALLTHROUGH ;
875881 case _EXIT_TRACE :
876882 default :
877883 {
878- // Cancel out pushes and pops, repeatedly. So:
879- // _LOAD_FAST + _POP_TOP + _POP_TOP + _LOAD_CONST_INLINE_BORROW + _POP_TOP
880- // ...becomes:
881- // _NOP + _NOP + _POP_TOP + _NOP + _NOP
882- while (op_without_pop [opcode ]) {
883- if (remove_folded_stack_shuffle (buffer , pc , 2 ) ||
884- remove_folded_stack_shuffle (buffer , pc , 3 ) ||
885- remove_adjacent_push_pop (buffer , pc ))
886- {
887- opcode = buffer [pc ].opcode ;
888- continue ;
889- }
890- break ;
891- }
892884 /* _PUSH_FRAME doesn't escape or error, but it
893885 * does need the IP for the return address */
894886 bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE || opcode == _DYNAMIC_EXIT || opcode == _EXIT_TRACE );
@@ -942,6 +934,11 @@ _Py_uop_analyze_and_optimize(
942934 length = remove_unneeded_uops (output , length );
943935 assert (length > 0 );
944936
937+ peephole_uops (output , length );
938+
939+ length = remove_unneeded_uops (output , length );
940+ assert (length > 0 );
941+
945942 OPT_STAT_INC (optimizer_successes );
946943 return length ;
947944}
0 commit comments