Starting to add support for Python 3.11. WIP #939

Added support for setting tracing to different threads
along with some general fixes.

Main things missing:
- Compile with cython extensions
- Bytecode manipulation
- Frame eval mode (requires bytecode manipulation)
This commit is contained in:
Fabio Zadrozny 2022-06-24 15:48:09 -03:00
parent 7676e298b5
commit 161aa2683f
36 changed files with 1591 additions and 1007 deletions

View file

@ -209,7 +209,7 @@ def parse_cmdline(argv=None):
elif opt in ("-c", "--config_file"):
config_file = value.strip()
if os.path.exists(config_file):
f = open(config_file, 'rU')
f = open(config_file, 'r')
try:
config_file_contents = f.read()
finally:

View file

@ -12,7 +12,7 @@ from _pydevd_frame_eval.vendored.bytecode import cfg as bytecode_cfg
import dis
import opcode as _opcode
from _pydevd_bundle.pydevd_constants import KeyifyList, DebugInfoHolder
from _pydevd_bundle.pydevd_constants import KeyifyList, DebugInfoHolder, IS_PY311_OR_GREATER
from bisect import bisect
from collections import deque
@ -125,6 +125,8 @@ class _StackInterpreter(object):
if instr.name == 'LOAD_ASSERTION_ERROR':
return 'AssertionError'
name = self._getname(instr)
if isinstance(name, CodeType):
name = name.co_qualname # Note: only available for Python 3.11
if isinstance(name, _Variable):
name = name.name
@ -279,8 +281,14 @@ class _StackInterpreter(object):
self._stack.append(instr)
def on_MAKE_FUNCTION(self, instr):
qualname = self._stack.pop()
code_obj_instr = self._stack.pop()
if not IS_PY311_OR_GREATER:
# The qualifier name is no longer put in the stack.
qualname = self._stack.pop()
code_obj_instr = self._stack.pop()
else:
# In 3.11 the code object has a co_qualname which we can use.
qualname = code_obj_instr = self._stack.pop()
arg = instr.arg
if arg & 0x08:
_func_closure = self._stack.pop()
@ -313,6 +321,9 @@ class _StackInterpreter(object):
func_name_instr = self._stack.pop()
self._handle_call_from_instr(func_name_instr, instr)
def on_PUSH_NULL(self, instr):
self._stack.append(instr)
def on_CALL_FUNCTION(self, instr):
arg = instr.arg
@ -523,6 +534,12 @@ class _StackInterpreter(object):
on_GET_AWAITABLE = _no_stack_change
on_GET_YIELD_FROM_ITER = _no_stack_change
def on_RETURN_GENERATOR(self, instr):
self._stack.append(instr)
on_RETURN_GENERATOR = _no_stack_change
on_RESUME = _no_stack_change
def on_MAP_ADD(self, instr):
self.on_POP_TOP(instr)
self.on_POP_TOP(instr)
@ -650,6 +667,9 @@ class _StackInterpreter(object):
on_UNARY_NOT = _no_stack_change
on_UNARY_INVERT = _no_stack_change
on_CACHE = _no_stack_change
on_PRECALL = _no_stack_change
def _get_smart_step_into_targets(code):
'''
@ -668,17 +688,20 @@ def _get_smart_step_into_targets(code):
try:
func_name = 'on_%s' % (instr.name,)
func = getattr(stack, func_name, None)
if DEBUG:
if instr.name != 'CACHE': # Filter the ones we don't want to see.
print('\nWill handle: ', instr, '>>', stack._getname(instr), '<<')
print('Current stack:')
for entry in stack._stack:
print(' arg:', stack._getname(entry), '(', entry, ')')
if func is None:
if STRICT_MODE:
raise AssertionError('%s not found.' % (func_name,))
else:
continue
if DEBUG:
print('\nWill handle: ', instr, '>>', stack._getname(instr), '<<')
func(instr)
if DEBUG:
for entry in stack._stack:
print(' arg:', stack._getname(entry), '(', entry, ')')
except:
if STRICT_MODE:
raise # Error in strict mode.
@ -784,7 +807,11 @@ def calculate_smart_step_into_variants(frame, start_line, end_line, base=0):
call_order_cache = {}
if DEBUG:
dis.dis(code)
print('dis.dis:')
if IS_PY311_OR_GREATER:
dis.dis(code, show_caches=False)
else:
dis.dis(code)
for target in _get_smart_step_into_targets(code):
variant = _convert_target_to_variant(target, start_line, end_line, call_order_cache, lasti, base)

View file

@ -327,7 +327,7 @@ if sys.version_info[:2] <= (3, 9):
return try_except_info_lst
if sys.version_info[:2] >= (3, 10):
elif sys.version_info[:2] == (3, 10):
class _TargetInfo(object):
@ -460,6 +460,16 @@ if sys.version_info[:2] >= (3, 10):
return try_except_info_lst
elif sys.version_info[:2] >= (3, 11):
def collect_try_except_info(co, use_func_first_line=False):
'''
Note: if the filename is available and we can get the source,
`collect_try_except_info_from_source` is preferred (this is kept as
a fallback for cases where sources aren't available).
'''
return []
import ast as ast_module

View file

@ -171,6 +171,7 @@ IS_PY37_OR_GREATER = sys.version_info >= (3, 7)
IS_PY38_OR_GREATER = sys.version_info >= (3, 8)
IS_PY39_OR_GREATER = sys.version_info >= (3, 9)
IS_PY310_OR_GREATER = sys.version_info >= (3, 10)
IS_PY311_OR_GREATER = sys.version_info >= (3, 11)
def version_str(v):

File diff suppressed because it is too large Load diff

View file

@ -90,7 +90,10 @@ def _patch_threading_to_hide_pydevd_threads():
new_threading_enumerate = None
if found_load_names == set(('_active_limbo_lock', '_limbo', '_active', 'values', 'list')):
if found_load_names in (
{'_active_limbo_lock', '_limbo', '_active', 'values', 'list'},
{'_active_limbo_lock', '_limbo', '_active', 'values', 'NULL + list'}
):
pydev_log.debug('Applying patching to hide pydevd threads (Py3 version).')
def new_threading_enumerate():

View file

@ -3,6 +3,7 @@ from _pydevd_bundle import pydevd_xml
from os.path import basename
from _pydev_bundle import pydev_log
from urllib.parse import unquote_plus
from _pydevd_bundle.pydevd_constants import IS_PY311_OR_GREATER
#===================================================================================================
@ -195,6 +196,24 @@ def get_referrer_info(searched_obj):
sys.stderr.write(' Found as %s in tuple: \n' % (found_as,))
break
elif IS_PY311_OR_GREATER:
# Up to Python 3.10, gc.get_referrers for an instance actually returned the
# object.__dict__, but on Python 3.11 it returns the actual object, so,
# handling is a bit easier (we don't need the workaround from the dict
# case to find the actual instance, we just need to find the attribute name).
if DEBUG:
sys.stderr.write('Found dict referrer: %r\n' % (r,))
dct = getattr(r, '__dict__', None)
if dct:
# Try to check if it's a value in the dict (and under which key it was found)
for key, val in dct.items():
if val is searched_obj:
found_as = key
if DEBUG:
sys.stderr.write(' Found as %r in object instance\n' % (found_as,))
break
if found_as:
if not isinstance(found_as, str):
found_as = str(found_as)

View file

@ -148,7 +148,7 @@ def notify_error(*args):
#=======================================================================================================================
def code_objects_equal(code0, code1):
for d in dir(code0):
if d.startswith('_') or 'line' in d or d == 'replace':
if d.startswith('_') or 'line' in d or d in ('replace', 'co_positions', 'co_qualname'):
continue
if getattr(code0, d) != getattr(code1, d):
return False

View file

@ -104,13 +104,13 @@ def build():
# set VS100COMNTOOLS=C:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools
env = os.environ.copy()
if sys.version_info[:2] in ((2, 6), (2, 7), (3, 5), (3, 6), (3, 7), (3, 8), (3, 9), (3, 10)):
if sys.version_info[:2] in ((2, 6), (2, 7), (3, 5), (3, 6), (3, 7), (3, 8), (3, 9), (3, 10), (3, 11)):
import setuptools # We have to import it first for the compiler to be found
from distutils import msvc9compiler
if sys.version_info[:2] in ((2, 6), (2, 7)):
vcvarsall = msvc9compiler.find_vcvarsall(9.0)
elif sys.version_info[:2] in ((3, 5), (3, 6), (3, 7), (3, 8), (3, 9), (3, 10)):
elif sys.version_info[:2] in ((3, 5), (3, 6), (3, 7), (3, 8), (3, 9), (3, 10), (3, 11)):
vcvarsall = msvc9compiler.find_vcvarsall(14.0)
if vcvarsall is None or not os.path.exists(vcvarsall):
raise RuntimeError('Error finding vcvarsall.')

View file

@ -31,6 +31,7 @@ def get_main_thread_id(unlikely_thread_id=None):
if (frame.f_code.co_name, basename) in [
('_run_module_as_main', 'runpy.py'),
('_run_module_as_main', '<frozen runpy>'),
('run_module_as_main', 'runpy.py'),
('run_module', 'runpy.py'),
('run_path', 'runpy.py'),

View file

@ -5,6 +5,7 @@
#include "py_utils.hpp"
#include "py_custom_pyeval_settrace_common.hpp"
#include "py_custom_pyeval_settrace_310.hpp"
#include "py_custom_pyeval_settrace_311.hpp"
// On Python 3.7 onwards the thread state is not kept in PyThread_set_key_value (rather
// it uses PyThread_tss_set using PyThread_tss_set(&_PyRuntime.gilstate.autoTSSkey, (void *)tstate)
@ -181,6 +182,8 @@ void InternalPySetTrace(PyThreadState* curThread, PyObjectHolder* traceFunc, boo
} else if (PyThreadState_310::IsFor(version)) {
// 3.10 has other changes on the actual algorithm (use_tracing is per-frame now), so, we have a full new version for it.
InternalPySetTrace_Template310<PyThreadState_310*>(reinterpret_cast<PyThreadState_310*>(curThread), traceFunc, isDebug);
} else if (PyThreadState_311::IsFor(version)) {
InternalPySetTrace_Template311<PyThreadState_311*>(reinterpret_cast<PyThreadState_311*>(curThread), traceFunc, isDebug);
} else {
printf("Unable to set trace to target thread with Python version: %d", version);
}

View file

@ -0,0 +1,120 @@
#ifndef _PY_CUSTOM_PYEVAL_SETTRACE_311_HPP_
#define _PY_CUSTOM_PYEVAL_SETTRACE_311_HPP_
#include "python.h"
#include "py_utils.hpp"
static PyObject *
InternalCallTrampoline311(PyObject* callback,
PyFrameObject311 *frame, int what, PyObject *arg)
{
PyObject *result;
PyObject *stack[3];
// Note: this is commented out from CPython (we shouldn't need it and it adds a reasonable overhead).
// if (PyFrame_FastToLocalsWithError(frame) < 0) {
// return NULL;
// }
//
stack[0] = (PyObject *)frame;
stack[1] = InternalWhatstrings_37[what];
stack[2] = (arg != NULL) ? arg : internalInitializeCustomPyEvalSetTrace->pyNone;
// Helper to print info.
//printf("--- start\n");
//printf("%s\n", internalInitializeCustomPyEvalSetTrace->pyUnicode_AsUTF8(internalInitializeCustomPyEvalSetTrace->pyObject_Repr((PyObject *)stack[0])));
//printf("%s\n", internalInitializeCustomPyEvalSetTrace->pyUnicode_AsUTF8(internalInitializeCustomPyEvalSetTrace->pyObject_Repr((PyObject *)stack[1])));
//printf("%s\n", internalInitializeCustomPyEvalSetTrace->pyUnicode_AsUTF8(internalInitializeCustomPyEvalSetTrace->pyObject_Repr((PyObject *)stack[2])));
//printf("--- end\n");
result = internalInitializeCustomPyEvalSetTrace->pyObject_FastCallDict(callback, stack, 3, NULL);
// Note: this is commented out from CPython (we shouldn't need it and it adds a reasonable overhead).
// PyFrame_LocalsToFast(frame, 1);
if (result == NULL) {
internalInitializeCustomPyEvalSetTrace->pyTraceBack_Here(frame);
}
return result;
}
// See: static int trace_trampoline(PyObject *self, PyFrameObject *frame, int what, PyObject *arg)
// in: https://github.com/python/cpython/blob/3.11/Python/sysmodule.c
static int
InternalTraceTrampoline311(PyObject *self, PyFrameObject *frameParam,
int what, PyObject *arg)
{
PyObject *callback;
PyObject *result;
PyFrameObject311 *frame = reinterpret_cast<PyFrameObject311*>(frameParam);
if (what == PyTrace_CALL){
callback = self;
} else {
callback = frame->f_trace;
}
if (callback == NULL){
return 0;
}
result = InternalCallTrampoline311(callback, frame, what, arg);
if (result == NULL) {
// Note: calling the original sys.settrace here.
internalInitializeCustomPyEvalSetTrace->pyEval_SetTrace(NULL, NULL);
PyObject *temp_f_trace = frame->f_trace;
frame->f_trace = NULL;
if(temp_f_trace != NULL){
DecRef(temp_f_trace, internalInitializeCustomPyEvalSetTrace->isDebug);
}
return -1;
}
if (result != internalInitializeCustomPyEvalSetTrace->pyNone) {
PyObject *tmp = frame->f_trace;
frame->f_trace = result;
DecRef(tmp, internalInitializeCustomPyEvalSetTrace->isDebug);
}
else {
DecRef(result, internalInitializeCustomPyEvalSetTrace->isDebug);
}
return 0;
}
// Based on ceval.c (PyEval_SetTrace(Py_tracefunc func, PyObject *arg))
// https://github.com/python/cpython/blob/3.11/Python/ceval.c
template<typename T>
void InternalPySetTrace_Template311(T tstate, PyObjectHolder* traceFunc, bool isDebug)
{
PyObject *traceobj = tstate->c_traceobj;
PyObject *arg = traceFunc->ToPython();
IncRef(arg);
tstate->c_tracefunc = NULL;
tstate->c_traceobj = NULL;
// This is different (previously it was just: tstate->use_tracing, now
// this flag is per-frame).
int use_tracing = (tstate->c_profilefunc != NULL);
// Note: before 3.11 this was just 1 or 0, now it needs to be 255 or 0.
tstate->cframe->use_tracing = (use_tracing ? 255 : 0);
if(traceobj != NULL){
DecRef(traceobj, isDebug);
}
tstate->c_tracefunc = InternalTraceTrampoline311;
tstate->c_traceobj = arg;
/* Flag that tracing or profiling is turned on */
use_tracing = ((InternalTraceTrampoline311 != NULL)
|| (tstate->c_profilefunc != NULL));
// Note: before 3.11 this was just 1 or 0, now it needs to be 255 or 0.
tstate->cframe->use_tracing = (use_tracing ? 255 : 0);
};
#endif //_PY_CUSTOM_PYEVAL_SETTRACE_311_HPP_

View file

@ -33,6 +33,8 @@ DWORD GetPythonThreadId(PythonVersion version, PyThreadState* curThread) {
threadId = (DWORD)((PyThreadState_39*)curThread)->thread_id;
} else if (PyThreadState_310::IsFor(version)) {
threadId = (DWORD)((PyThreadState_310*)curThread)->thread_id;
} else if (PyThreadState_311::IsFor(version)) {
threadId = (DWORD)((PyThreadState_311*)curThread)->thread_id;
}
return threadId;
}

View file

@ -35,6 +35,7 @@ typedef PyObject* PyEval_CallObjectWithKeywords(PyObject *callable, PyObject *ar
typedef void (PyEval_SetTrace)(Py_tracefunc, PyObject *);
typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *frame, int, PyObject *);
typedef int (_PyEval_SetTrace)(PyThreadState *tstate, Py_tracefunc func, PyObject *arg);
typedef PyObject* PyObject_Repr(PyObject *);
typedef const char* PyUnicode_AsUTF8(PyObject *unicode);

View file

@ -20,7 +20,8 @@ enum PythonVersion {
PythonVersion_37 = 0x0307,
PythonVersion_38 = 0x0308,
PythonVersion_39 = 0x0309,
PythonVersion_310 = 0x030A
PythonVersion_310 = 0x030A,
PythonVersion_311 = 0x030B
};
@ -66,6 +67,9 @@ static PythonVersion GetPythonVersion(void *module) {
if(version[3] == '0'){
return PythonVersion_310;
}
if(version[3] == '1'){
return PythonVersion_311;
}
}
return PythonVersion_Unknown; // we don't care about 3.1 anymore...

View file

@ -80,14 +80,18 @@ typedef struct {
} PyUnicodeObject;
class PyFrameObject : public PyVarObject {
class PyFrameObject : public PyObject {
// After 3.10 we don't really have things we want to reuse common, so,
// create an empty base (based on PyVarObject).
// create an empty base (it's not based on PyVarObject because on Python 3.11
// it's just a PyObject and no longer a PyVarObject -- the part related to
// the var object must be declared in ech subclass in this case).
};
// 2.4 - 3.7 compatible
class PyFrameObjectBaseUpTo39 : public PyFrameObject {
public:
size_t ob_size; /* Number of items in variable part -- i.e.: PyVarObject*/
PyFrameObjectBaseUpTo39 *f_back; /* previous frame, or nullptr */
PyObject *f_code; /* code segment */
PyObject *f_builtins; /* builtin symbol table (PyDictObject) */
@ -108,6 +112,8 @@ public:
// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h
class PyFrameObject310 : public PyFrameObject {
public:
size_t ob_size; /* Number of items in variable part -- i.e.: PyVarObject*/
PyFrameObject310 *f_back; /* previous frame, or NULL */
PyObject *f_code; /* code segment */
PyObject *f_builtins; /* builtin symbol table (PyDictObject) */
@ -119,6 +125,45 @@ public:
// It has more things, but we're only interested in things up to f_trace.
};
typedef uint16_t _Py_CODEUNIT;
// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h
typedef struct _PyInterpreterFrame311 {
/* "Specials" section */
PyFunctionObject *f_func; /* Strong reference */
PyObject *f_globals; /* Borrowed reference */
PyObject *f_builtins; /* Borrowed reference */
PyObject *f_locals; /* Strong reference, may be NULL */
void *f_code; /* Strong reference */
void *frame_obj; /* Strong reference, may be NULL */
/* Linkage section */
struct _PyInterpreterFrame311 *previous;
// NOTE: This is not necessarily the last instruction started in the given
// frame. Rather, it is the code unit *prior to* the *next* instruction. For
// example, it may be an inline CACHE entry, an instruction we just jumped
// over, or (in the case of a newly-created frame) a totally invalid value:
_Py_CODEUNIT *prev_instr;
int stacktop; /* Offset of TOS from localsplus */
bool is_entry; // Whether this is the "root" frame for the current _PyCFrame.
char owner;
/* Locals and stack */
PyObject *localsplus[1];
} _PyInterpreterFrame311;
// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h
// Note that in 3.11 it's no longer a "PyVarObject".
class PyFrameObject311 : public PyFrameObject {
public:
PyFrameObject311 *f_back; /* previous frame, or NULL */
struct _PyInterpreterFrame311 *f_frame; /* points to the frame data */
PyObject *f_trace; /* Trace function */
int f_lineno; /* Current line number. Only valid if non-zero */
char f_trace_lines; /* Emit per-line trace events? */
char f_trace_opcodes; /* Emit per-opcode trace events? */
char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */
// It has more things, but we're not interested on those.
};
typedef void (*destructor)(PyObject *);
@ -572,6 +617,60 @@ public:
}
};
// i.e.: https://github.com/python/cpython/blob/3.11/Include/cpython/pystate.h
class PyThreadState_311 : public PyThreadState {
public:
PyThreadState *prev;
PyThreadState *next;
PyInterpreterState *interp;
int _initialized;
int _static;
int recursion_remaining;
int recursion_limit;
int recursion_headroom; /* Allow 50 more calls to handle any errors. */
/* 'tracing' keeps track of the execution depth when tracing/profiling.
This is to prevent the actual trace/profile code from being recorded in
the trace/profile. */
int tracing;
int tracing_what;
/* Pointer to current CFrame in the C stack frame of the currently,
* or most recently, executing _PyEval_EvalFrameDefault. */
CFrame *cframe;
Py_tracefunc c_profilefunc;
Py_tracefunc c_tracefunc;
PyObject *c_profileobj;
PyObject *c_traceobj;
PyObject *curexc_type;
PyObject *curexc_value;
PyObject *curexc_traceback;
_PyErr_StackItem *exc_info;
PyObject *dict; /* Stores per-thread state */
int gilstate_counter;
PyObject *async_exc; /* Asynchronous exception to raise */
unsigned long thread_id; /* Thread id where this tstate was created */
static bool IsFor(int majorVersion, int minorVersion) {
return majorVersion == 3 && minorVersion == 11;
}
static bool IsFor(PythonVersion version) {
return version == PythonVersion_311;
}
};
class PyIntObject : public PyObject {
public:
long ob_ival;

View file

@ -240,7 +240,7 @@ def get_python_helper_lib_filename():
def _load_python_helper_lib_uncached():
if (not IS_CPYTHON or sys.version_info[:2] > (3, 10)
if (not IS_CPYTHON or sys.version_info[:2] > (3, 11)
or hasattr(sys, 'gettotalrefcount') or LOAD_NATIVE_LIB_FLAG in ENV_FALSE_LOWER_VALUES):
pydev_log.info('Helper lib to set tracing to all threads not loaded.')
return None

View file

@ -90,7 +90,9 @@ class TestCPython(unittest.TestCase):
'math.cpython-36m' in completions or
'math.cpython-37m' in completions or
'math.cpython-38' in completions or
'math.cpython-39' in completions
'math.cpython-39' in completions or
'math.cpython-310' in completions or
'math.cpython-311' in completions
):
return
self.assertTrue(completions.startswith(start) or completions.startswith(start_2), '%s DOESNT START WITH %s' % (completions, (start, start_2)))

View file

@ -160,7 +160,7 @@ class _ExcVerifier(object):
if update_try_except_infos is not None:
update_try_except_infos(try_except_infos)
if sys.version_info[:2] != (3, 10):
if sys.version_info[:2] not in ((3, 10), (3, 11)):
assert str(try_except_infos) == expected_as_str
from _pydevd_bundle.pydevd_collect_bytecode_info import collect_try_except_info_from_source

View file

@ -43,13 +43,15 @@ def collect_smart_step_into_variants(*args, **kwargs):
try:
return pydevd_bytecode_utils.calculate_smart_step_into_variants(*args, **kwargs)
except:
# In a failure, rerun with DEBUG!
debug = pydevd_bytecode_utils.DEBUG
pydevd_bytecode_utils.DEBUG = True
try:
return pydevd_bytecode_utils.calculate_smart_step_into_variants(*args, **kwargs)
finally:
pydevd_bytecode_utils.DEBUG = debug
pass
# In a failure, rerun with DEBUG!
debug = pydevd_bytecode_utils.DEBUG
pydevd_bytecode_utils.DEBUG = True
try:
return pydevd_bytecode_utils.calculate_smart_step_into_variants(*args, **kwargs)
finally:
pydevd_bytecode_utils.DEBUG = debug
def check_names_from_func_str(func_str, expected):