Skip to content

Commit d3c7e05

Browse files
pablogsalmaurycy
andcommitted
Split interpreter control and data stacks
Co-authored-by: maurycy <5383+maurycy@users.noreply.github.com>
1 parent e3262d9 commit d3c7e05

68 files changed

Lines changed: 2667 additions & 1889 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Include/cpython/ceval.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ PyAPI_FUNC(void) PyEval_SetTraceAllThreads(Py_tracefunc, PyObject *);
1212
flag was set, else return 0. */
1313
PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
1414

15-
PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc);
15+
PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _PyControlRecord *f, int exc);
1616

1717
PyAPI_FUNC(Py_ssize_t) PyUnstable_Eval_RequestCodeExtraIndex(freefunc);
1818
// Old name -- remove when this API changes:

‎Include/cpython/pyframe.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,21 +22,21 @@ PyAPI_FUNC(PyObject*) PyFrame_GetVarString(PyFrameObject *frame, const char *nam
2222
/* The following functions are for use by debuggers and other tools
2323
* implementing custom frame evaluators with PEP 523. */
2424

25-
struct _PyInterpreterFrame;
25+
struct _PyControlRecord;
2626

2727
/* Returns the code object of the frame (strong reference).
2828
* Does not raise an exception. */
29-
PyAPI_FUNC(PyObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame);
29+
PyAPI_FUNC(PyObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyControlRecord *frame);
3030

3131
/* Returns a byte offset into the last executed instruction.
3232
* Does not raise an exception. */
33-
PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame);
33+
PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyControlRecord *frame);
3434

3535
/* Returns the currently executing line number, or -1 if there is no line number.
3636
* Does not raise an exception. */
37-
PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame);
37+
PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyControlRecord *frame);
3838

3939
/* Returns a new (strong) reference to the local variable at `index` in the
4040
* frame's localsplus array. */
4141
PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLocal(
42-
struct _PyInterpreterFrame *frame, Py_ssize_t index, PyObject **result);
42+
struct _PyControlRecord *frame, Py_ssize_t index, PyObject **result);

‎Include/cpython/pystate.h‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,6 @@ struct _ts {
121121
/* Thread state. See Include/internal/pycore_pystate.h for details. */
122122
int state;
123123

124-
int py_recursion_remaining;
125124
int py_recursion_limit;
126125
int recursion_headroom; /* Allow 50 more calls to handle any errors. */
127126

@@ -132,16 +131,11 @@ struct _ts {
132131
int what_event; /* The event currently being monitored, if any. */
133132

134133
/* Pointer to currently executing frame. */
135-
struct _PyInterpreterFrame *current_frame;
134+
struct _PyControlRecord *current_frame;
136135

137-
/* Pointer to the base frame (bottommost sentinel frame).
138-
Used by profilers to validate complete stack unwinding.
139-
Points to the embedded base_frame in _PyThreadStateImpl.
140-
The frame is embedded there rather than here because _PyInterpreterFrame
141-
is defined in internal headers that cannot be exposed in the public API. */
142-
struct _PyInterpreterFrame *base_frame;
136+
struct _PyControlRecord *base_frame;
143137

144-
struct _PyInterpreterFrame *last_profiled_frame;
138+
struct _PyControlRecord *last_profiled_frame;
145139
uintptr_t last_profiled_frame_seq;
146140

147141
Py_tracefunc c_profilefunc;
@@ -195,6 +189,11 @@ struct _ts {
195189
/* Unique thread state id. */
196190
uint64_t id;
197191

192+
struct _PyControlRecord *controlstack_top;
193+
struct _PyControlRecord *controlstack_limit;
194+
struct _PyControlRecord *controlstack_end;
195+
Py_ssize_t controlstack_extra;
196+
size_t controlstack_size;
198197
_PyStackChunk *datastack_chunk;
199198
PyObject **datastack_top;
200199
PyObject **datastack_limit;
@@ -326,7 +325,7 @@ PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
326325

327326
/* Frame evaluation API */
328327

329-
typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _PyInterpreterFrame *, int);
328+
typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _PyControlRecord *, int);
330329

331330
PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc(
332331
PyInterpreterState *interp);

‎Include/internal/pycore_ceval.h‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ extern "C" {
1414
#include "pycore_interp.h" // PyInterpreterState.eval_frame
1515
#include "pycore_pystate.h" // _PyThreadState_GET()
1616
#include "pycore_stats.h" // EVAL_CALL_STAT_INC()
17-
#include "pycore_typedefs.h" // _PyInterpreterFrame
17+
#include "pycore_typedefs.h" // _PyControlRecord
1818

1919

2020
/* Forward declarations */
@@ -115,7 +115,7 @@ extern void _PyPerfJit_WriteNamedCode(const void *code_addr,
115115
#endif
116116

117117
static inline PyObject*
118-
_PyEval_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
118+
_PyEval_EvalFrame(PyThreadState *tstate, _PyControlRecord *frame, int throwflag)
119119
{
120120
EVAL_CALL_STAT_INC(EVAL_CALL_TOTAL);
121121
if (tstate->interp->eval_frame == NULL) {
@@ -126,7 +126,7 @@ _PyEval_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwfl
126126

127127
#ifdef _Py_TIER2
128128
_Py_CODEUNIT *_PyTier2Interpreter(
129-
struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame,
129+
struct _PyExecutorObject *current_executor, _PyControlRecord *frame,
130130
_PyStackRef *stack_pointer, PyThreadState *tstate
131131
);
132132
#endif
@@ -254,7 +254,7 @@ PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(
254254
static inline void _Py_LeaveRecursiveCall(void) {
255255
}
256256

257-
extern _PyInterpreterFrame* _PyEval_GetFrame(void);
257+
extern _PyControlRecord* _PyEval_GetFrame(void);
258258

259259
extern PyObject * _PyEval_GetGlobalsFromRunningMain(PyThreadState *);
260260
extern int _PyEval_EnsureBuiltins(
@@ -293,7 +293,7 @@ PyAPI_DATA(const size_t) _Py_FunctionAttributeOffsets[];
293293

294294
PyAPI_FUNC(int) _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right);
295295
PyAPI_FUNC(int) _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right);
296-
PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *, PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest);
296+
PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(_PyControlRecord *, _PyInterpreterDataFrame *, PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest);
297297
PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg);
298298
PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj);
299299
PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
@@ -305,7 +305,7 @@ PyAPI_FUNC(PyObject *) _PyEval_LazyImportName(
305305
PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level,
306306
int lazy);
307307
PyAPI_FUNC(PyObject *) _PyEval_LazyImportFrom(
308-
PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *v, PyObject *name);
308+
PyThreadState *tstate, _PyControlRecord *frame, PyObject *v, PyObject *name);
309309
PyAPI_FUNC(PyObject *) _PyEval_ImportName(
310310
PyThreadState *tstate, PyObject *builtins, PyObject *globals,
311311
PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level);
@@ -314,18 +314,19 @@ PyObject * _PyEval_ImportNameWithImport(
314314
PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level);
315315
PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs);
316316
PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys);
317-
PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr);
318-
PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame);
317+
PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyControlRecord *frame, _Py_CODEUNIT *instr);
318+
PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyControlRecord *frame);
319319
PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp);
320-
PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame);
320+
PyAPI_FUNC(void) _PyEval_FrameClearAndPop(
321+
PyThreadState *tstate, _PyControlRecord *frame, _PyInterpreterDataFrame *fp);
321322
PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch);
322323

323324
PyAPI_FUNC(void) _PyObjectArray_Free(PyObject **array, PyObject **scratch);
324325

325326
PyAPI_FUNC(PyObject *) _PyEval_GetANext(PyObject *aiter);
326327
PyAPI_FUNC(void) _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name, _PyStackRef *writeto);
327328
PyAPI_FUNC(PyObject *) _PyEval_GetAwaitable(PyObject *iterable, int oparg);
328-
PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name);
329+
PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterDataFrame *fp, PyObject *name);
329330
PyAPI_FUNC(int)
330331
_Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args);
331332
PyAPI_FUNC(_PyStackRef) _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *null_or_index, int yield_from);
@@ -397,7 +398,7 @@ extern int _PyRunRemoteDebugger(PyThreadState *tstate);
397398
#endif
398399

399400
PyAPI_FUNC(_PyStackRef)
400-
_PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef *index_ptr);
401+
_PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyControlRecord* frame, _PyStackRef iter, _PyStackRef *index_ptr);
401402

402403
/* Special methods used by LOAD_SPECIAL */
403404
#define SPECIAL___ENTER__ 0
@@ -424,7 +425,7 @@ _Py_VectorCallInstrumentation_StackRefSteal(
424425
int total_args,
425426
_PyStackRef kwnames,
426427
bool call_instrumentation,
427-
_PyInterpreterFrame* frame,
428+
_PyControlRecord* frame,
428429
_Py_CODEUNIT* this_instr,
429430
PyThreadState* tstate);
430431

@@ -474,7 +475,8 @@ _Py_BuildMap_StackRefSteal(
474475

475476
PyAPI_FUNC(void)
476477
_Py_assert_within_stack_bounds(
477-
_PyInterpreterFrame *frame, _PyStackRef *stack_pointer,
478+
_PyControlRecord *frame, _PyInterpreterDataFrame *fp,
479+
_PyStackRef *stack_pointer,
478480
const char *filename, int lineno);
479481

480482
PyAPI_FUNC(_PyStackRef)

‎Include/internal/pycore_code.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ PyAPI_FUNC(void) _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr);
329329
PyAPI_FUNC(void) _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr);
330330
PyAPI_FUNC(void) _Py_GatherStats_GetIter(_PyStackRef iterable);
331331
PyAPI_FUNC(void) _Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr);
332-
PyAPI_FUNC(void) _Py_Specialize_Resume(_Py_CODEUNIT *instr, PyThreadState *tstate, _PyInterpreterFrame *frame);
332+
PyAPI_FUNC(void) _Py_Specialize_Resume(_Py_CODEUNIT *instr, PyThreadState *tstate, _PyControlRecord *frame);
333333
PyAPI_FUNC(void) _Py_Specialize_GetIter(_PyStackRef iterable, _Py_CODEUNIT *instr);
334334

335335
// Utility functions for reading/writing 32/64-bit values in the inline caches.

‎Include/internal/pycore_debug_offsets.h‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ extern "C" {
5353
# define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled)
5454
# define _Py_Debug_Free_Threaded 1
5555
# define _Py_Debug_code_object_co_tlbc offsetof(PyCodeObject, co_tlbc)
56-
# define _Py_Debug_interpreter_frame_tlbc_index offsetof(_PyInterpreterFrame, tlbc_index)
56+
# define _Py_Debug_interpreter_frame_tlbc_index offsetof(_PyControlRecord, tlbc_index)
5757
# define _Py_Debug_interpreter_state_tlbc_generation offsetof(PyInterpreterState, tlbc_indices.tlbc_generation)
5858
#else
5959
# define _Py_Debug_gilruntimestate_enabled 0
@@ -130,6 +130,8 @@ typedef struct _Py_DebugOffsets {
130130
uint64_t owner;
131131
uint64_t stackpointer;
132132
uint64_t tlbc_index;
133+
uint64_t state;
134+
uint64_t c_frame;
133135
} interpreter_frame;
134136

135137
// Code object offset;
@@ -242,6 +244,7 @@ typedef struct _Py_DebugOffsets {
242244
uint64_t gi_name;
243245
uint64_t gi_iframe;
244246
uint64_t gi_frame_state;
247+
uint64_t gi_live_frame;
245248
} gen_object;
246249

247250
struct _llist_node {
@@ -309,14 +312,16 @@ typedef struct _Py_DebugOffsets {
309312
.exc_value = offsetof(_PyErr_StackItem, exc_value), \
310313
}, \
311314
.interpreter_frame = { \
312-
.size = sizeof(_PyInterpreterFrame), \
313-
.previous = offsetof(_PyInterpreterFrame, previous), \
314-
.executable = offsetof(_PyInterpreterFrame, f_executable), \
315-
.instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \
316-
.localsplus = offsetof(_PyInterpreterFrame, localsplus), \
317-
.owner = offsetof(_PyInterpreterFrame, owner), \
318-
.stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \
315+
.size = sizeof(_PyControlRecord), \
316+
.previous = 0, \
317+
.executable = offsetof(_PyControlRecord, f_executable), \
318+
.instr_ptr = offsetof(_PyControlRecord, instr_ptr), \
319+
.localsplus = offsetof(_PyControlRecord, data), \
320+
.owner = offsetof(_PyControlRecord, owner), \
321+
.stackpointer = offsetof(_PyControlRecord, stackpointer), \
319322
.tlbc_index = _Py_Debug_interpreter_frame_tlbc_index, \
323+
.state = offsetof(_PyControlRecord, state), \
324+
.c_frame = offsetof(_PyControlRecord, c_frame), \
320325
}, \
321326
.code_object = { \
322327
.size = sizeof(PyCodeObject), \
@@ -401,6 +406,7 @@ typedef struct _Py_DebugOffsets {
401406
.gi_name = offsetof(PyGenObject, gi_name), \
402407
.gi_iframe = offsetof(PyGenObject, gi_iframe), \
403408
.gi_frame_state = offsetof(PyGenObject, gi_frame_state), \
409+
.gi_live_frame = offsetof(PyGenObject, gi_live_frame), \
404410
}, \
405411
.llist_node = { \
406412
.next = offsetof(struct llist_node, next), \

‎Include/internal/pycore_frame.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/* See InternalDocs/frames.md for an explanation of the frame stack
2-
* including explanation of the PyFrameObject and _PyInterpreterFrame
2+
* including explanation of the PyFrameObject and _PyControlRecord
33
* structs. */
44

55
#ifndef Py_INTERNAL_FRAME_H
@@ -12,13 +12,13 @@ extern "C" {
1212
# error "this header requires Py_BUILD_CORE define"
1313
#endif
1414

15-
#include "pycore_typedefs.h" // _PyInterpreterFrame
15+
#include "pycore_typedefs.h" // _PyControlRecord
1616

1717

1818
struct _frame {
1919
PyObject_HEAD
2020
PyFrameObject *f_back; /* previous frame, or NULL */
21-
_PyInterpreterFrame *f_frame; /* points to the frame data */
21+
_PyControlRecord *f_frame; /* points to the frame data */
2222
PyObject *f_trace; /* Trace function */
2323
int f_lineno; /* Current line number. Only valid if non-zero */
2424
char f_trace_lines; /* Emit per-line trace events? */

‎Include/internal/pycore_gc.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ extern "C" {
1010

1111
#include "pycore_interp_structs.h" // PyGC_Head
1212
#include "pycore_pystate.h" // _PyInterpreterState_GET()
13-
#include "pycore_typedefs.h" // _PyInterpreterFrame
13+
#include "pycore_typedefs.h" // _PyControlRecord
1414

1515

1616
/* Get an object's GC head */
@@ -352,7 +352,8 @@ union _PyStackRef;
352352

353353
// GC visit callback for tracked interpreter frames
354354
// GH-150766: exported for greenlet
355-
PyAPI_FUNC(int) _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg);
355+
PyAPI_FUNC(int) _PyGC_VisitFrameStack(_PyControlRecord *frame, _PyInterpreterDataFrame *fp,
356+
visitproc visit, void *arg);
356357
PyAPI_FUNC(int) _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg);
357358

358359
#ifdef Py_GIL_DISABLED

0 commit comments

Comments
 (0)