mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-02 05:12:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Merge branch '3.14' of https://github.com/python/cpython into 3.14

Browse source
This commit is contained in:
Hugo van Kemenade 2025-05-26 21:51:15 +03:00
commit bc3d892113
17 changed files with 1468 additions and 990 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.github/CODEOWNERS vendored
View file

@ -326,3 +326,8 @@ Modules/_xxtestfuzz/ @ammaraskar
**/*templateobject* @lysnikolaou
**/*templatelib* @lysnikolaou
**/*tstring* @lysnikolaou
# Remote debugging
Python/remote_debug.h @pablogsal
Python/remote_debugging.c @pablogsal
Modules/_remote_debugging_module.c @pablogsal @ambv @1st1

1
.github/workflows/tail-call.yml vendored
View file

@ -137,4 +137,3 @@ jobs:
CC=clang-20 ./configure --with-tail-call-interp --disable-gil
make all --jobs 4
./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3

8
.pre-commit-config.yaml
View file

@ -43,12 +43,14 @@ repos:
exclude: ^Lib/test/test_tomllib/
- id: check-yaml
- id: end-of-file-fixer
types: [python]
types_or: [python, yaml]
exclude: Lib/test/tokenizedata/coding20731.py
- id: end-of-file-fixer
files: '^\.github/CODEOWNERS$'
- id: trailing-whitespace
types_or: [c, inc, python, rst]
types_or: [c, inc, python, rst, yaml]
- id: trailing-whitespace
files: '\.(gram)$'
files: '^\.github/CODEOWNERS|\.(gram)$'
- repo: https://github.com/python-jsonschema/check-jsonschema
rev: 0.33.0

1
.readthedocs.yml
View file

@ -32,4 +32,3 @@ build:
- make -C Doc venv html
- mkdir _readthedocs
- mv Doc/build/html _readthedocs/html

1589
Doc/data/python3.14.abi
View file

File diff suppressed because it is too large Load diff

14
Doc/reference/datamodel.rst
View file

@ -1228,10 +1228,22 @@ Special attributes
:attr:`__annotations__ attributes <object.__annotations__>`.
For best practices on working with :attr:`~object.__annotations__`,
please see :mod:`annotationlib`. Where possible, use
please see :mod:`annotationlib`. Use
:func:`annotationlib.get_annotations` instead of accessing this
attribute directly.
.. warning::
Accessing the :attr:`!__annotations__` attribute directly
on a class object may return annotations for the wrong class, specifically
in certain cases where the class, its base class, or a metaclass
is defined under ``from __future__ import annotations``.
See :pep:`749 <749#pep749-metaclasses>` for details.
This attribute does not exist on certain builtin classes. On
user-defined classes without ``__annotations__``, it is an
empty dictionary.
.. versionchanged:: 3.14
Annotations are now :ref:`lazily evaluated <lazy-evaluation>`.
See :pep:`649`.

11
Doc/whatsnew/3.14.rst
View file

@ -74,7 +74,7 @@ deferred evaluation of annotations (:pep:`649`),
and a new type of interpreter that uses tail calls.
The library changes include the addition of a new :mod:`!annotationlib` module
for introspecting and wrapping annotations (:pep:`649`),
for introspecting and wrapping annotations (:pep:`749`),
a new :mod:`!compression.zstd` module for Zstandard support (:pep:`784`),
plus syntax highlighting in the REPL,
as well as the usual deprecations and removals,
@ -444,6 +444,10 @@ In particular, do not read annotations directly from the namespace dictionary
attribute of type objects. Use :func:`annotationlib.get_annotate_from_class_namespace`
during class construction and :func:`annotationlib.get_annotations` afterwards.
In previous releases, it was sometimes possible to access class annotations from
an instance of an annotated class. This behavior was undocumented and accidental,
and will no longer work in Python 3.14.
``from __future__ import annotations``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@ -2501,6 +2505,11 @@ Changes in the Python API
See :ref:`above <whatsnew314-typing-union>` for more details.
(Contributed by Jelle Zijlstra in :gh:`105499`.)
* The runtime behavior of annotations has changed in various ways; see
:ref:`above <whatsnew314-pep649>` for details. While most code that interacts
with annotations should continue to work, some undocumented details may behave
differently.
Build changes
=============

1
Include/internal/pycore_global_objects_fini_generated.h generated
View file

@ -894,6 +894,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(debug));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default));

1
Include/internal/pycore_global_strings.h
View file

@ -385,6 +385,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(data)
STRUCT_FOR_ID(database)
STRUCT_FOR_ID(day)
STRUCT_FOR_ID(debug)
STRUCT_FOR_ID(decode)
STRUCT_FOR_ID(decoder)
STRUCT_FOR_ID(default)

1
Include/internal/pycore_runtime_init_generated.h generated
View file

@ -892,6 +892,7 @@ extern "C" {
INIT_ID(data), \
INIT_ID(database), \
INIT_ID(day), \
INIT_ID(debug), \
INIT_ID(decode), \
INIT_ID(decoder), \
INIT_ID(default), \

4
Include/internal/pycore_unicodeobject_generated.h generated
View file

@ -1328,6 +1328,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(debug);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(decode);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));

6
Lib/test/test_external_inspection.py
View file

@ -34,17 +34,17 @@ skip_if_not_supported = unittest.skipIf(
def get_stack_trace(pid):
unwinder = RemoteUnwinder(pid, all_threads=True)
unwinder = RemoteUnwinder(pid, all_threads=True, debug=True)
return unwinder.get_stack_trace()
def get_async_stack_trace(pid):
unwinder = RemoteUnwinder(pid)
unwinder = RemoteUnwinder(pid, debug=True)
return unwinder.get_async_stack_trace()
def get_all_awaited_by(pid):
unwinder = RemoteUnwinder(pid)
unwinder = RemoteUnwinder(pid, debug=True)
return unwinder.get_all_awaited_by()

1
Misc/NEWS.d/next/Library/2025-05-26-17-06-40.gh-issue-134637.9-3zRL.rst Normal file
View file

@ -0,0 +1 @@
Fix performance regression in calling a :mod:`ctypes` function pointer in :term:`free threading`.

142
Modules/_ctypes/_ctypes.c
View file

@ -3591,6 +3591,45 @@ generic_pycdata_new(ctypes_state *st,
PyCFuncPtr_Type
*/
static inline void
atomic_xsetref(PyObject **field, PyObject *value)
{
#ifdef Py_GIL_DISABLED
PyObject *old = *field;
_Py_atomic_store_ptr(field, value);
Py_XDECREF(old);
#else
Py_XSETREF(*field, value);
#endif
}
/*
This function atomically loads the reference from *field, and
tries to get a new reference to it. If the incref fails,
it acquires critical section of obj and returns a new reference to the *field.
In the general case, this avoids contention on acquiring the critical section.
*/
static inline PyObject *
atomic_xgetref(PyObject *obj, PyObject **field)
{
#ifdef Py_GIL_DISABLED
PyObject *value = _Py_atomic_load_ptr(field);
if (value == NULL) {
return NULL;
}
if (_Py_TryIncrefCompare(field, value)) {
return value;
}
Py_BEGIN_CRITICAL_SECTION(obj);
value = Py_XNewRef(*field);
Py_END_CRITICAL_SECTION();
return value;
#else
return Py_XNewRef(*field);
#endif
}
/*[clinic input]
@critical_section
@setter
@ -3607,7 +3646,7 @@ _ctypes_CFuncPtr_errcheck_set_impl(PyCFuncPtrObject *self, PyObject *value)
return -1;
}
Py_XINCREF(value);
Py_XSETREF(self->errcheck, value);
atomic_xsetref(&self->errcheck, value);
return 0;
}
@ -3639,12 +3678,10 @@ static int
_ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value)
/*[clinic end generated code: output=0be0a086abbabf18 input=683c3bef4562ccc6]*/
{
PyObject *checker, *oldchecker;
PyObject *checker;
if (value == NULL) {
oldchecker = self->checker;
self->checker = NULL;
Py_CLEAR(self->restype);
Py_XDECREF(oldchecker);
atomic_xsetref(&self->restype, NULL);
atomic_xsetref(&self->checker, NULL);
return 0;
}
ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self)));
@ -3660,11 +3697,9 @@ _ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value)
if (PyObject_GetOptionalAttr(value, &_Py_ID(_check_retval_), &checker) < 0) {
return -1;
}
oldchecker = self->checker;
self->checker = checker;
Py_INCREF(value);
Py_XSETREF(self->restype, value);
Py_XDECREF(oldchecker);
atomic_xsetref(&self->checker, checker);
atomic_xsetref(&self->restype, value);
return 0;
}
@ -3709,16 +3744,16 @@ _ctypes_CFuncPtr_argtypes_set_impl(PyCFuncPtrObject *self, PyObject *value)
PyObject *converters;
if (value == NULL || value == Py_None) {
Py_CLEAR(self->converters);
Py_CLEAR(self->argtypes);
atomic_xsetref(&self->argtypes, NULL);
atomic_xsetref(&self->converters, NULL);
} else {
ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self)));
converters = converters_from_argtypes(st, value);
if (!converters)
return -1;
Py_XSETREF(self->converters, converters);
atomic_xsetref(&self->converters, converters);
Py_INCREF(value);
Py_XSETREF(self->argtypes, value);
atomic_xsetref(&self->argtypes, value);
}
return 0;
}
@ -4514,16 +4549,11 @@ _build_result(PyObject *result, PyObject *callargs,
}
static PyObject *
PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds)
PyCFuncPtr_call(PyObject *op, PyObject *inargs, PyObject *kwds)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
PyObject *restype;
PyObject *converters;
PyObject *checker;
PyObject *argtypes;
PyObject *result;
PyObject *callargs;
PyObject *errcheck;
PyObject *result = NULL;
PyObject *callargs = NULL;
PyObject *ret = NULL;
#ifdef MS_WIN32
IUnknown *piunk = NULL;
#endif
@ -4541,13 +4571,24 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds)
}
assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */
restype = self->restype ? self->restype : info->restype;
converters = self->converters ? self->converters : info->converters;
checker = self->checker ? self->checker : info->checker;
argtypes = self->argtypes ? self->argtypes : info->argtypes;
/* later, we probably want to have an errcheck field in stginfo */
errcheck = self->errcheck /* ? self->errcheck : info->errcheck */;
PyObject *restype = atomic_xgetref(op, &self->restype);
if (restype == NULL) {
restype = Py_XNewRef(info->restype);
}
PyObject *converters = atomic_xgetref(op, &self->converters);
if (converters == NULL) {
converters = Py_XNewRef(info->converters);
}
PyObject *checker = atomic_xgetref(op, &self->checker);
if (checker == NULL) {
checker = Py_XNewRef(info->checker);
}
PyObject *argtypes = atomic_xgetref(op, &self->argtypes);
if (argtypes == NULL) {
argtypes = Py_XNewRef(info->argtypes);
}
/* later, we probably want to have an errcheck field in stginfo */
PyObject *errcheck = atomic_xgetref(op, &self->errcheck);
pProc = *(void **)self->b_ptr;
#ifdef MS_WIN32
@ -4558,25 +4599,25 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds)
if (!this) {
PyErr_SetString(PyExc_ValueError,
"native com method call without 'this' parameter");
return NULL;
goto finally;
}
if (!CDataObject_Check(st, this)) {
PyErr_SetString(PyExc_TypeError,
"Expected a COM this pointer as first argument");
return NULL;
goto finally;
}
/* there should be more checks? No, in Python */
/* First arg is a pointer to an interface instance */
if (!this->b_ptr || *(void **)this->b_ptr == NULL) {
PyErr_SetString(PyExc_ValueError,
"NULL COM pointer access");
return NULL;
goto finally;
}
piunk = *(IUnknown **)this->b_ptr;
if (NULL == piunk->lpVtbl) {
PyErr_SetString(PyExc_ValueError,
"COM method call without VTable");
return NULL;
goto finally;
}
pProc = ((void **)piunk->lpVtbl)[self->index - 0x1000];
}
@ -4584,8 +4625,9 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds)
callargs = _build_callargs(st, self, argtypes,
inargs, kwds,
&outmask, &inoutmask, &numretvals);
if (callargs == NULL)
return NULL;
if (callargs == NULL) {
goto finally;
}
if (converters) {
int required = Py_SAFE_DOWNCAST(PyTuple_GET_SIZE(converters),
@ -4604,7 +4646,7 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds)
required,
required == 1 ? "" : "s",
actual);
return NULL;
goto finally;
}
} else if (required != actual) {
Py_DECREF(callargs);
@ -4613,7 +4655,7 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds)
required,
required == 1 ? "" : "s",
actual);
return NULL;
goto finally;
}
}
@ -4644,23 +4686,19 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds)
if (v == NULL || v != callargs) {
Py_DECREF(result);
Py_DECREF(callargs);
return v;
ret = v;
goto finally;
}
Py_DECREF(v);
}
return _build_result(result, callargs,
outmask, inoutmask, numretvals);
}
static PyObject *
PyCFuncPtr_call(PyObject *op, PyObject *inargs, PyObject *kwds)
{
PyObject *result;
Py_BEGIN_CRITICAL_SECTION(op);
result = PyCFuncPtr_call_lock_held(op, inargs, kwds);
Py_END_CRITICAL_SECTION();
return result;
ret = _build_result(result, callargs, outmask, inoutmask, numretvals);
finally:
Py_XDECREF(restype);
Py_XDECREF(converters);
Py_XDECREF(checker);
Py_XDECREF(argtypes);
Py_XDECREF(errcheck);
return ret;
}
static int

373
Modules/_remote_debugging_module.c
View file

File diff suppressed because it is too large Load diff

33
Modules/clinic/_remote_debugging_module.c.h generated
View file

@ -10,7 +10,7 @@ preserve
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__,
"RemoteUnwinder(pid, *, all_threads=False)\n"
"RemoteUnwinder(pid, *, all_threads=False, debug=False)\n"
"--\n"
"\n"
"Initialize a new RemoteUnwinder object for debugging a remote Python process.\n"
@ -19,6 +19,8 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__,
" pid: Process ID of the target Python process to debug\n"
" all_threads: If True, initialize state for all threads in the process.\n"
" If False, only initialize for the main thread.\n"
" debug: If True, chain exceptions to explain the sequence of events that\n"
" lead to the exception.\n"
"\n"
"The RemoteUnwinder provides functionality to inspect and debug a running Python\n"
"process, including examining thread states, stack frames and other runtime data.\n"
@ -30,7 +32,8 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__,
static int
_remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self,
int pid, int all_threads);
int pid, int all_threads,
int debug);
static int
_remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObject *kwargs)
@ -38,7 +41,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje
int return_value = -1;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS 2
#define NUM_KEYWORDS 3
static struct {
PyGC_Head _this_is_not_used;
PyObject_VAR_HEAD
@ -47,7 +50,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_hash = -1,
.ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), },
.ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(debug), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -56,19 +59,20 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
static const char * const _keywords[] = {"pid", "all_threads", NULL};
static const char * const _keywords[] = {"pid", "all_threads", "debug", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
.fname = "RemoteUnwinder",
.kwtuple = KWTUPLE,
};
#undef KWTUPLE
PyObject *argsbuf[2];
PyObject *argsbuf[3];
PyObject * const *fastargs;
Py_ssize_t nargs = PyTuple_GET_SIZE(args);
Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1;
int pid;
int all_threads = 0;
int debug = 0;
fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser,
/*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
@ -82,12 +86,21 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje
if (!noptargs) {
goto skip_optional_kwonly;
}
all_threads = PyObject_IsTrue(fastargs[1]);
if (all_threads < 0) {
if (fastargs[1]) {
all_threads = PyObject_IsTrue(fastargs[1]);
if (all_threads < 0) {
goto exit;
}
if (!--noptargs) {
goto skip_optional_kwonly;
}
}
debug = PyObject_IsTrue(fastargs[2]);
if (debug < 0) {
goto exit;
}
skip_optional_kwonly:
return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads);
return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, debug);
exit:
return return_value;
@ -240,4 +253,4 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject
return return_value;
}
/*[clinic end generated code: output=654772085f1f4bf6 input=a9049054013a1b77]*/
/*[clinic end generated code: output=774ec34aa653402d input=a9049054013a1b77]*/

267
Python/remote_debug.h
View file

@ -73,6 +73,18 @@ extern "C" {
# define HAVE_PROCESS_VM_READV 0
#endif
#define _set_debug_exception_cause(exception, format, ...) \
do { \
if (!PyErr_ExceptionMatches(PyExc_PermissionError)) { \
PyThreadState *tstate = _PyThreadState_GET(); \
if (!_PyErr_Occurred(tstate)) { \
_PyErr_Format(tstate, exception, format, ##__VA_ARGS__); \
} else { \
_PyErr_FormatFromCause(exception, format, ##__VA_ARGS__); \
} \
} \
} while (0)
static inline size_t
get_page_size(void) {
size_t page_size = 0;
@ -137,12 +149,17 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) {
handle->pid = pid;
#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX
handle->task = pid_to_task(handle->pid);
if (handle->task == 0) {
_set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize macOS process handle");
return -1;
}
#elif defined(MS_WINDOWS)
handle->hProcess = OpenProcess(
PROCESS_VM_READ | PROCESS_VM_WRITE | PROCESS_VM_OPERATION | PROCESS_QUERY_INFORMATION,
FALSE, pid);
if (handle->hProcess == NULL) {
PyErr_SetFromWindowsErr(0);
_set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize Windows process handle");
return -1;
}
#endif
@ -206,8 +223,10 @@ return_section_address64(
&object_name
);
if (ret != KERN_SUCCESS) {
PyErr_SetString(
PyExc_RuntimeError, "Cannot get any more VM maps.\n");
PyErr_Format(PyExc_RuntimeError,
"mach_vm_region failed while parsing 64-bit Mach-O binary "
"at base address 0x%lx (kern_return_t: %d)",
base, ret);
return 0;
}
}
@ -227,9 +246,6 @@ return_section_address64(
cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize);
}
// We should not be here, but if we are there, we should say about this
PyErr_SetString(
PyExc_RuntimeError, "Cannot find section address.\n");
return 0;
}
@ -270,8 +286,10 @@ return_section_address32(
&object_name
);
if (ret != KERN_SUCCESS) {
PyErr_SetString(
PyExc_RuntimeError, "Cannot get any more VM maps.\n");
PyErr_Format(PyExc_RuntimeError,
"mach_vm_region failed while parsing 32-bit Mach-O binary "
"at base address 0x%lx (kern_return_t: %d)",
base, ret);
return 0;
}
}
@ -291,9 +309,6 @@ return_section_address32(
cmd = (struct segment_command*)((void*)cmd + cmd->cmdsize);
}
// We should not be here, but if we are there, we should say about this
PyErr_SetString(
PyExc_RuntimeError, "Cannot find section address.\n");
return 0;
}
@ -311,8 +326,20 @@ return_section_address_fat(
int is_abi64;
size_t cpu_size = sizeof(cpu), abi64_size = sizeof(is_abi64);
sysctlbyname("hw.cputype", &cpu, &cpu_size, NULL, 0);
sysctlbyname("hw.cpu64bit_capable", &is_abi64, &abi64_size, NULL, 0);
if (sysctlbyname("hw.cputype", &cpu, &cpu_size, NULL, 0) != 0) {
PyErr_Format(PyExc_OSError,
"Failed to determine CPU type via sysctlbyname "
"for fat binary analysis at 0x%lx: %s",
base, strerror(errno));
return 0;
}
if (sysctlbyname("hw.cpu64bit_capable", &is_abi64, &abi64_size, NULL, 0) != 0) {
PyErr_Format(PyExc_OSError,
"Failed to determine CPU ABI capability via sysctlbyname "
"for fat binary analysis at 0x%lx: %s",
base, strerror(errno));
return 0;
}
cpu |= is_abi64 * CPU_ARCH_ABI64;
@ -343,13 +370,18 @@ return_section_address_fat(
return return_section_address64(section, proc_ref, base, (void*)hdr);
default:
PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic in fat binary.\n");
PyErr_Format(PyExc_RuntimeError,
"Unknown Mach-O magic number 0x%x in fat binary architecture %u at base 0x%lx",
hdr->magic, i, base);
return 0;
}
}
}
PyErr_SetString(PyExc_RuntimeError, "No matching architecture found in fat binary.\n");
PyErr_Format(PyExc_RuntimeError,
"No matching architecture found for CPU type 0x%x "
"in fat binary at base 0x%lx (%u architectures examined)",
cpu, base, nfat_arch);
return 0;
}
@ -358,20 +390,26 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_
{
int fd = open(path, O_RDONLY);
if (fd == -1) {
PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path);
PyErr_Format(PyExc_OSError,
"Cannot open binary file '%s' for section '%s' search: %s",
path, secname, strerror(errno));
return 0;
}
struct stat fs;
if (fstat(fd, &fs) == -1) {
PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path);
PyErr_Format(PyExc_OSError,
"Cannot get file size for binary '%s' during section '%s' search: %s",
path, secname, strerror(errno));
close(fd);
return 0;
}
void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path);
PyErr_Format(PyExc_OSError,
"Cannot memory map binary file '%s' (size: %lld bytes) for section '%s' search: %s",
path, (long long)fs.st_size, secname, strerror(errno));
close(fd);
return 0;
}
@ -393,13 +431,22 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_
result = return_section_address_fat(secname, proc_ref, base, map);
break;
default:
PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic");
PyErr_Format(PyExc_RuntimeError,
"Unrecognized Mach-O magic number 0x%x in binary file '%s' for section '%s' search",
magic, path, secname);
break;
}
munmap(map, fs.st_size);
if (munmap(map, fs.st_size) != 0) {
PyErr_Format(PyExc_OSError,
"Failed to unmap binary file '%s' (size: %lld bytes): %s",
path, (long long)fs.st_size, strerror(errno));
result = 0;
}
if (close(fd) != 0) {
PyErr_SetFromErrno(PyExc_OSError);
PyErr_Format(PyExc_OSError,
"Failed to close binary file '%s': %s",
path, strerror(errno));
result = 0;
}
return result;
@ -414,7 +461,10 @@ pid_to_task(pid_t pid)
result = task_for_pid(mach_task_self(), pid, &task);
if (result != KERN_SUCCESS) {
PyErr_Format(PyExc_PermissionError, "Cannot get task for PID %d", pid);
PyErr_Format(PyExc_PermissionError,
"Cannot get task port for PID %d (kern_return_t: %d). "
"This typically requires running as root or having the 'com.apple.system-task-ports' entitlement.",
pid, result);
return 0;
}
return task;
@ -431,13 +481,15 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s
mach_port_t proc_ref = pid_to_task(handle->pid);
if (proc_ref == 0) {
if (!PyErr_Occurred()) {
PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID");
PyErr_Format(PyExc_PermissionError,
"Cannot get task port for PID %d during section search",
handle->pid);
}
return 0;
}
int match_found = 0;
char map_filename[MAXPATHLEN + 1];
while (mach_vm_region(
proc_ref,
&address,
@ -447,6 +499,7 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s
&count,
&object_name) == KERN_SUCCESS)
{
if ((region_info.protection & VM_PROT_READ) == 0
|| (region_info.protection & VM_PROT_EXECUTE) == 0) {
address += size;
@ -467,17 +520,17 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s
filename = map_filename; // No path, use the whole string
}
if (!match_found && strncmp(filename, substr, strlen(substr)) == 0) {
match_found = 1;
return search_section_in_file(
if (strncmp(filename, substr, strlen(substr)) == 0) {
uintptr_t result = search_section_in_file(
secname, map_filename, address, size, proc_ref);
if (result != 0) {
return result;
}
}
address += size;
}
PyErr_SetString(PyExc_RuntimeError,
"mach_vm_region failed to find the section");
return 0;
}
@ -500,24 +553,38 @@ search_elf_file_for_section(
int fd = open(elf_file, O_RDONLY);
if (fd < 0) {
PyErr_SetFromErrno(PyExc_OSError);
PyErr_Format(PyExc_OSError,
"Cannot open ELF file '%s' for section '%s' search: %s",
elf_file, secname, strerror(errno));
goto exit;
}
struct stat file_stats;
if (fstat(fd, &file_stats) != 0) {
PyErr_SetFromErrno(PyExc_OSError);
PyErr_Format(PyExc_OSError,
"Cannot get file size for ELF file '%s' during section '%s' search: %s",
elf_file, secname, strerror(errno));
goto exit;
}
file_memory = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
if (file_memory == MAP_FAILED) {
PyErr_SetFromErrno(PyExc_OSError);
PyErr_Format(PyExc_OSError,
"Cannot memory map ELF file '%s' (size: %lld bytes) for section '%s' search: %s",
elf_file, (long long)file_stats.st_size, secname, strerror(errno));
goto exit;
}
Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory;
// Validate ELF header
if (elf_header->e_shstrndx >= elf_header->e_shnum) {
PyErr_Format(PyExc_RuntimeError,
"Invalid ELF file '%s': string table index %u >= section count %u",
elf_file, elf_header->e_shstrndx, elf_header->e_shnum);
goto exit;
}
Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff);
Elf_Shdr* shstrtab_section = &section_header_table[elf_header->e_shstrndx];
@ -534,6 +601,10 @@ search_elf_file_for_section(
}
}
if (section == NULL) {
goto exit;
}
Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff);
// Find the first PT_LOAD segment
Elf_Phdr* first_load_segment = NULL;
@ -544,18 +615,25 @@ search_elf_file_for_section(
}
}
if (section != NULL && first_load_segment != NULL) {
uintptr_t elf_load_addr = first_load_segment->p_vaddr
- (first_load_segment->p_vaddr % first_load_segment->p_align);
result = start_address + (uintptr_t)section->sh_addr - elf_load_addr;
if (first_load_segment == NULL) {
PyErr_Format(PyExc_RuntimeError,
"No PT_LOAD segment found in ELF file '%s' (%u program headers examined)",
elf_file, elf_header->e_phnum);
goto exit;
}
uintptr_t elf_load_addr = first_load_segment->p_vaddr
- (first_load_segment->p_vaddr % first_load_segment->p_align);
result = start_address + (uintptr_t)section->sh_addr - elf_load_addr;
exit:
if (file_memory != NULL) {
munmap(file_memory, file_stats.st_size);
}
if (fd >= 0 && close(fd) != 0) {
PyErr_SetFromErrno(PyExc_OSError);
PyErr_Format(PyExc_OSError,
"Failed to close ELF file '%s': %s",
elf_file, strerror(errno));
result = 0;
}
return result;
@ -569,7 +647,9 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c
FILE* maps_file = fopen(maps_file_path, "r");
if (maps_file == NULL) {
PyErr_SetFromErrno(PyExc_OSError);
PyErr_Format(PyExc_OSError,
"Cannot open process memory map file '%s' for PID %d section search: %s",
maps_file_path, handle->pid, strerror(errno));
return 0;
}
@ -578,11 +658,16 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c
char *line = PyMem_Malloc(linesz);
if (!line) {
fclose(maps_file);
PyErr_NoMemory();
_set_debug_exception_cause(PyExc_MemoryError,
"Cannot allocate memory for reading process map file '%s'",
maps_file_path);
return 0;
}
uintptr_t retval = 0;
int lines_processed = 0;
int matches_found = 0;
while (fgets(line + linelen, linesz - linelen, maps_file) != NULL) {
linelen = strlen(line);
if (line[linelen - 1] != '\n') {
@ -593,7 +678,9 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c
if (!biggerline) {
PyMem_Free(line);
fclose(maps_file);
PyErr_NoMemory();
_set_debug_exception_cause(PyExc_MemoryError,
"Cannot reallocate memory while reading process map file '%s' (attempted size: %zu)",
maps_file_path, linesz);
return 0;
}
line = biggerline;
@ -604,6 +691,7 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c
line[linelen - 1] = '\0';
// and prepare to read the next line into the start of the buffer.
linelen = 0;
lines_processed++;
unsigned long start = 0;
unsigned long path_pos = 0;
@ -624,6 +712,7 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c
}
if (strstr(filename, substr)) {
matches_found++;
retval = search_elf_file_for_section(handle, secname, start, path);
if (retval) {
break;
@ -633,7 +722,9 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c
PyMem_Free(line);
if (fclose(maps_file) != 0) {
PyErr_SetFromErrno(PyExc_OSError);
PyErr_Format(PyExc_OSError,
"Failed to close process map file '%s': %s",
maps_file_path, strerror(errno));
retval = 0;
}
@ -649,11 +740,20 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char*
HANDLE hFile = CreateFileW(mod_path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (hFile == INVALID_HANDLE_VALUE) {
PyErr_SetFromWindowsErr(0);
DWORD error = GetLastError();
PyErr_Format(PyExc_OSError,
"Cannot open PE file for section '%s' analysis (error %lu)",
secname, error);
return NULL;
}
HANDLE hMap = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, 0);
if (!hMap) {
PyErr_SetFromWindowsErr(0);
DWORD error = GetLastError();
PyErr_Format(PyExc_OSError,
"Cannot create file mapping for PE file section '%s' analysis (error %lu)",
secname, error);
CloseHandle(hFile);
return NULL;
}
@ -661,6 +761,10 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char*
BYTE* mapView = (BYTE*)MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 0);
if (!mapView) {
PyErr_SetFromWindowsErr(0);
DWORD error = GetLastError();
PyErr_Format(PyExc_OSError,
"Cannot map view of PE file for section '%s' analysis (error %lu)",
secname, error);
CloseHandle(hMap);
CloseHandle(hFile);
return NULL;
@ -668,7 +772,9 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char*
IMAGE_DOS_HEADER* pDOSHeader = (IMAGE_DOS_HEADER*)mapView;
if (pDOSHeader->e_magic != IMAGE_DOS_SIGNATURE) {
PyErr_SetString(PyExc_RuntimeError, "Invalid DOS signature.");
PyErr_Format(PyExc_RuntimeError,
"Invalid DOS signature (0x%x) in PE file for section '%s' analysis (expected 0x%x)",
pDOSHeader->e_magic, secname, IMAGE_DOS_SIGNATURE);
UnmapViewOfFile(mapView);
CloseHandle(hMap);
CloseHandle(hFile);
@ -677,7 +783,9 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char*
IMAGE_NT_HEADERS* pNTHeaders = (IMAGE_NT_HEADERS*)(mapView + pDOSHeader->e_lfanew);
if (pNTHeaders->Signature != IMAGE_NT_SIGNATURE) {
PyErr_SetString(PyExc_RuntimeError, "Invalid NT signature.");
PyErr_Format(PyExc_RuntimeError,
"Invalid NT signature (0x%lx) in PE file for section '%s' analysis (expected 0x%lx)",
pNTHeaders->Signature, secname, IMAGE_NT_SIGNATURE);
UnmapViewOfFile(mapView);
CloseHandle(hMap);
CloseHandle(hFile);
@ -711,17 +819,27 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const
} while (hProcSnap == INVALID_HANDLE_VALUE && GetLastError() == ERROR_BAD_LENGTH);
if (hProcSnap == INVALID_HANDLE_VALUE) {
PyErr_SetString(PyExc_PermissionError, "Unable to create module snapshot. Check permissions or PID.");
PyErr_SetFromWindowsErr(0);
DWORD error = GetLastError();
PyErr_Format(PyExc_PermissionError,
"Unable to create module snapshot for PID %d section '%s' "
"search (error %lu). Check permissions or PID validity",
handle->pid, secname, error);
return 0;
}
MODULEENTRY32W moduleEntry;
moduleEntry.dwSize = sizeof(moduleEntry);
void* runtime_addr = NULL;
int modules_examined = 0;
int matches_found = 0;
for (BOOL hasModule = Module32FirstW(hProcSnap, &moduleEntry); hasModule; hasModule = Module32NextW(hProcSnap, &moduleEntry)) {
modules_examined++;
// Look for either python executable or DLL
if (wcsstr(moduleEntry.szModule, substr)) {
matches_found++;
runtime_addr = analyze_pe(moduleEntry.szExePath, moduleEntry.modBaseAddr, secname);
if (runtime_addr != NULL) {
break;
@ -730,6 +848,7 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const
}
CloseHandle(hProcSnap);
return (uintptr_t)runtime_addr;
}
@ -747,7 +866,9 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle)
if (address == 0) {
// Error out: 'python' substring covers both executable and DLL
PyObject *exc = PyErr_GetRaisedException();
PyErr_SetString(PyExc_RuntimeError, "Failed to find the PyRuntime section in the process.");
PyErr_Format(PyExc_RuntimeError,
"Failed to find the PyRuntime section in process %d on Windows platform",
handle->pid);
_PyErr_ChainExceptions1(exc);
}
#elif defined(__linux__)
@ -756,16 +877,28 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle)
if (address == 0) {
// Error out: 'python' substring covers both executable and DLL
PyObject *exc = PyErr_GetRaisedException();
PyErr_SetString(PyExc_RuntimeError, "Failed to find the PyRuntime section in the process.");
PyErr_Format(PyExc_RuntimeError,
"Failed to find the PyRuntime section in process %d on Linux platform",
handle->pid);
_PyErr_ChainExceptions1(exc);
}
#elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX
// On macOS, try libpython first, then fall back to python
address = search_map_for_section(handle, "PyRuntime", "libpython");
if (address == 0) {
// TODO: Differentiate between not found and error
const char* candidates[] = {"libpython", "python", "Python", NULL};
for (const char** candidate = candidates; *candidate; candidate++) {
PyErr_Clear();
address = search_map_for_section(handle, "PyRuntime", "python");
address = search_map_for_section(handle, "PyRuntime", *candidate);
if (address != 0) {
break;
}
}
if (address == 0) {
PyObject *exc = PyErr_GetRaisedException();
PyErr_Format(PyExc_RuntimeError,
"Failed to find the PyRuntime section in process %d "
"on macOS platform (tried both libpython and python)",
handle->pid);
_PyErr_ChainExceptions1(exc);
}
#else
Py_UNREACHABLE();
@ -784,6 +917,11 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address
do {
if (!ReadProcessMemory(handle->hProcess, (LPCVOID)(remote_address + result), (char*)dst + result, len - result, &read_bytes)) {
PyErr_SetFromWindowsErr(0);
DWORD error = GetLastError();
_set_debug_exception_cause(PyExc_OSError,
"ReadProcessMemory failed for PID %d at address 0x%lx "
"(size %zu, partial read %zu bytes): Windows error %lu",
handle->pid, remote_address + result, len - result, result, error);
return -1;
}
result += read_bytes;
@ -804,6 +942,10 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address
read_bytes = process_vm_readv(handle->pid, local, 1, remote, 1, 0);
if (read_bytes < 0) {
PyErr_SetFromErrno(PyExc_OSError);
_set_debug_exception_cause(PyExc_OSError,
"process_vm_readv failed for PID %d at address 0x%lx "
"(size %zu, partial read %zd bytes): %s",
handle->pid, remote_address + result, len - result, result, strerror(errno));
return -1;
}
@ -822,13 +964,22 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address
if (kr != KERN_SUCCESS) {
switch (kr) {
case KERN_PROTECTION_FAILURE:
PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory");
PyErr_Format(PyExc_PermissionError,
"Memory protection failure reading from PID %d at address "
"0x%lx (size %zu): insufficient permissions",
handle->pid, remote_address, len);
break;
case KERN_INVALID_ARGUMENT:
PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite");
PyErr_Format(PyExc_ValueError,
"Invalid argument to mach_vm_read_overwrite for PID %d at "
"address 0x%lx (size %zu)",
handle->pid, remote_address, len);
break;
default:
PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory");
PyErr_Format(PyExc_RuntimeError,
"mach_vm_read_overwrite failed for PID %d at address 0x%lx "
"(size %zu): kern_return_t %d",
handle->pid, remote_address, len, kr);
}
return -1;
}
@ -868,7 +1019,10 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle,
if (entry->data == NULL) {
entry->data = PyMem_RawMalloc(page_size);
if (entry->data == NULL) {
PyErr_NoMemory();
_set_debug_exception_cause(PyExc_MemoryError,
"Cannot allocate %zu bytes for page cache entry "
"during read from PID %d at address 0x%lx",
page_size, handle->pid, addr);
return -1;
}
}
@ -900,13 +1054,16 @@ _Py_RemoteDebug_ReadDebugOffsets(
*runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(handle);
if (!*runtime_start_address) {
if (!PyErr_Occurred()) {
PyErr_SetString(
PyExc_RuntimeError, "Failed to get PyRuntime address");
PyErr_Format(PyExc_RuntimeError,
"Failed to locate PyRuntime address for PID %d",
handle->pid);
}
_set_debug_exception_cause(PyExc_RuntimeError, "PyRuntime address lookup failed during debug offsets initialization");
return -1;
}
size_t size = sizeof(struct _Py_DebugOffsets);
if (0 != _Py_RemoteDebug_ReadRemoteMemory(handle, *runtime_start_address, size, debug_offsets)) {
_set_debug_exception_cause(PyExc_RuntimeError, "Failed to read debug offsets structure from remote process");
return -1;
}
return 0;