bpo-36842: Implement PEP 578 (GH-12613)

Adds sys.audit, sys.addaudithook, io.open_code, and associated C APIs.
This commit is contained in:
Steve Dower 2019-05-23 08:45:22 -07:00 committed by GitHub
parent e788057a91
commit b82e17e626
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
70 changed files with 3565 additions and 1816 deletions

4
Python/Python-ast.c generated
View file

@ -9024,6 +9024,10 @@ mod_ty PyAST_obj2mod_ex(PyObject* ast, PyArena* arena, int mode, int feature_ver
char *req_name[] = {"Module", "Expression", "Interactive"};
int isinstance;
if (PySys_Audit("compile", "OO", ast, Py_None) < 0) {
return NULL;
}
req_type[0] = (PyObject*)Module_type;
req_type[1] = (PyObject*)Expression_type;
req_type[2] = (PyObject*)Interactive_type;

View file

@ -977,9 +977,13 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals,
}
if (PyCode_Check(source)) {
if (PySys_Audit("exec", "O", source) < 0) {
return NULL;
}
if (PyCode_GetNumFree((PyCodeObject *)source) > 0) {
PyErr_SetString(PyExc_TypeError,
"code object passed to eval() may not contain free variables");
"code object passed to eval() may not contain free variables");
return NULL;
}
return PyEval_EvalCode(source, globals, locals);
@ -1061,6 +1065,10 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals,
}
if (PyCode_Check(source)) {
if (PySys_Audit("exec", "O", source) < 0) {
return NULL;
}
if (PyCode_GetNumFree((PyCodeObject *)source) > 0) {
PyErr_SetString(PyExc_TypeError,
"code object passed to exec() may not "
@ -1207,7 +1215,14 @@ static PyObject *
builtin_id(PyModuleDef *self, PyObject *v)
/*[clinic end generated code: output=0aa640785f697f65 input=5a534136419631f4]*/
{
return PyLong_FromVoidPtr(v);
PyObject *id = PyLong_FromVoidPtr(v);
if (id && PySys_Audit("builtins.id", "O", id) < 0) {
Py_DECREF(id);
return NULL;
}
return id;
}
@ -1986,6 +2001,10 @@ builtin_input_impl(PyObject *module, PyObject *prompt)
return NULL;
}
if (PySys_Audit("builtins.input", "O", prompt ? prompt : Py_None) < 0) {
return NULL;
}
/* First of all, flush stderr */
tmp = _PyObject_CallMethodId(ferr, &PyId_flush, NULL);
if (tmp == NULL)
@ -2116,6 +2135,13 @@ builtin_input_impl(PyObject *module, PyObject *prompt)
Py_DECREF(stdin_errors);
Py_XDECREF(po);
PyMem_FREE(s);
if (result != NULL) {
if (PySys_Audit("builtins.input/result", "O", result) < 0) {
return NULL;
}
}
return result;
_readline_errors:

View file

@ -4555,6 +4555,10 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
void
PyEval_SetProfile(Py_tracefunc func, PyObject *arg)
{
if (PySys_Audit("sys.setprofile", NULL) < 0) {
return;
}
PyThreadState *tstate = _PyThreadState_GET();
PyObject *temp = tstate->c_profileobj;
Py_XINCREF(arg);
@ -4572,6 +4576,10 @@ PyEval_SetProfile(Py_tracefunc func, PyObject *arg)
void
PyEval_SetTrace(Py_tracefunc func, PyObject *arg)
{
if (PySys_Audit("sys.settrace", NULL) < 0) {
return;
}
_PyRuntimeState *runtime = &_PyRuntime;
PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
PyObject *temp = tstate->c_traceobj;
@ -4608,6 +4616,11 @@ void
_PyEval_SetCoroutineWrapper(PyObject *wrapper)
{
PyThreadState *tstate = _PyThreadState_GET();
if (PySys_Audit("sys.set_coroutine_wrapper", NULL) < 0) {
return;
}
Py_XINCREF(wrapper);
Py_XSETREF(tstate->coroutine_wrapper, wrapper);
}
@ -4623,6 +4636,11 @@ void
_PyEval_SetAsyncGenFirstiter(PyObject *firstiter)
{
PyThreadState *tstate = _PyThreadState_GET();
if (PySys_Audit("sys.set_asyncgen_hook_firstiter", NULL) < 0) {
return;
}
Py_XINCREF(firstiter);
Py_XSETREF(tstate->async_gen_firstiter, firstiter);
}
@ -4638,6 +4656,11 @@ void
_PyEval_SetAsyncGenFinalizer(PyObject *finalizer)
{
PyThreadState *tstate = _PyThreadState_GET();
if (PySys_Audit("sys.set_asyncgen_hook_finalizer", NULL) < 0) {
return;
}
Py_XINCREF(finalizer);
Py_XSETREF(tstate->async_gen_finalizer, finalizer);
}

View file

@ -2,6 +2,38 @@
preserve
[clinic start generated code]*/
PyDoc_STRVAR(sys_addaudithook__doc__,
"addaudithook($module, /, hook)\n"
"--\n"
"\n"
"Adds a new audit hook callback.");
#define SYS_ADDAUDITHOOK_METHODDEF \
{"addaudithook", (PyCFunction)(void(*)(void))sys_addaudithook, METH_FASTCALL|METH_KEYWORDS, sys_addaudithook__doc__},
static PyObject *
sys_addaudithook_impl(PyObject *module, PyObject *hook);
static PyObject *
sys_addaudithook(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
static const char * const _keywords[] = {"hook", NULL};
static _PyArg_Parser _parser = {NULL, _keywords, "addaudithook", 0};
PyObject *argsbuf[1];
PyObject *hook;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
if (!args) {
goto exit;
}
hook = args[0];
return_value = sys_addaudithook_impl(module, hook);
exit:
return return_value;
}
PyDoc_STRVAR(sys_displayhook__doc__,
"displayhook($module, object, /)\n"
"--\n"
@ -1076,4 +1108,4 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored))
#ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
#define SYS_GETANDROIDAPILEVEL_METHODDEF
#endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
/*[clinic end generated code: output=603e4d5a453dc769 input=a9049054013a1b77]*/
/*[clinic end generated code: output=3c32bc91ec659509 input=a9049054013a1b77]*/

View file

@ -1262,6 +1262,10 @@ _Py_open_impl(const char *pathname, int flags, int gil_held)
#endif
if (gil_held) {
if (PySys_Audit("open", "sOi", pathname, Py_None, flags) < 0) {
return -1;
}
do {
Py_BEGIN_ALLOW_THREADS
fd = open(pathname, flags);
@ -1331,6 +1335,9 @@ FILE *
_Py_wfopen(const wchar_t *path, const wchar_t *mode)
{
FILE *f;
if (PySys_Audit("open", "uui", path, mode, 0) < 0) {
return NULL;
}
#ifndef MS_WINDOWS
char *cpath;
char cmode[10];
@ -1366,6 +1373,10 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode)
FILE*
_Py_fopen(const char *pathname, const char *mode)
{
if (PySys_Audit("open", "ssi", pathname, mode, 0) < 0) {
return NULL;
}
FILE *f = fopen(pathname, mode);
if (f == NULL)
return NULL;
@ -1401,6 +1412,9 @@ _Py_fopen_obj(PyObject *path, const char *mode)
assert(PyGILState_Check());
if (PySys_Audit("open", "Osi", path, mode, 0) < 0) {
return NULL;
}
if (!PyUnicode_Check(path)) {
PyErr_Format(PyExc_TypeError,
"str file path expected under Windows, got %R",
@ -1434,6 +1448,10 @@ _Py_fopen_obj(PyObject *path, const char *mode)
return NULL;
path_bytes = PyBytes_AS_STRING(bytes);
if (PySys_Audit("open", "Osi", path, mode, 0) < 0) {
return NULL;
}
do {
Py_BEGIN_ALLOW_THREADS
f = fopen(path_bytes, mode);

View file

@ -1661,6 +1661,17 @@ import_find_and_load(PyObject *abs_name)
_PyTime_t t1 = 0, accumulated_copy = accumulated;
PyObject *sys_path = PySys_GetObject("path");
PyObject *sys_meta_path = PySys_GetObject("meta_path");
PyObject *sys_path_hooks = PySys_GetObject("path_hooks");
if (PySys_Audit("import", "OOOOO",
abs_name, Py_None, sys_path ? sys_path : Py_None,
sys_meta_path ? sys_meta_path : Py_None,
sys_path_hooks ? sys_path_hooks : Py_None) < 0) {
return NULL;
}
/* XOptions is initialized after first some imports.
* So we can't have negative cache before completed initialization.
* Anyway, importlib._find_and_load is much slower than

View file

@ -119,6 +119,11 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp)
if (path == NULL)
goto error;
if (PySys_Audit("import", "OOOOO", name_unicode, path,
Py_None, Py_None, Py_None) < 0) {
return NULL;
}
#ifdef MS_WINDOWS
exportfunc = _PyImport_FindSharedFuncptrWindows(hook_prefix, name_buf,
path, fp);

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -1250,6 +1250,13 @@ Py_FinalizeEx(void)
/* nothing */;
#endif
/* Clear all loghooks */
/* We want minimal exposure of this function, so define the extern
* here. The linker should discover the correct function without
* exporting a symbol. */
extern void _PySys_ClearAuditHooks(void);
_PySys_ClearAuditHooks();
/* Destroy all modules */
PyImport_Cleanup();

View file

@ -45,8 +45,19 @@ static void _PyThreadState_Delete(_PyRuntimeState *runtime, PyThreadState *tstat
static _PyInitError
_PyRuntimeState_Init_impl(_PyRuntimeState *runtime)
{
/* We preserve the hook across init, because there is
currently no public API to set it between runtime
initialization and interpreter initialization. */
void *open_code_hook = runtime->open_code_hook;
void *open_code_userdata = runtime->open_code_userdata;
_Py_AuditHookEntry *audit_hook_head = runtime->audit_hook_head;
memset(runtime, 0, sizeof(*runtime));
runtime->open_code_hook = open_code_hook;
runtime->open_code_userdata = open_code_userdata;
runtime->audit_hook_head = audit_hook_head;
_PyGC_Initialize(&runtime->gc);
_PyEval_Initialize(&runtime->ceval);
_PyPreConfig_InitPythonConfig(&runtime->preconfig);
@ -181,6 +192,10 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime)
PyInterpreterState *
PyInterpreterState_New(void)
{
if (PySys_Audit("cpython.PyInterpreterState_New", NULL) < 0) {
return NULL;
}
PyInterpreterState *interp = PyMem_RawMalloc(sizeof(PyInterpreterState));
if (interp == NULL) {
return NULL;
@ -233,6 +248,8 @@ PyInterpreterState_New(void)
interp->tstate_next_unique_id = 0;
interp->audit_hooks = NULL;
return interp;
}
@ -240,11 +257,18 @@ PyInterpreterState_New(void)
static void
_PyInterpreterState_Clear(_PyRuntimeState *runtime, PyInterpreterState *interp)
{
if (PySys_Audit("cpython.PyInterpreterState_Clear", NULL) < 0) {
PyErr_Clear();
}
HEAD_LOCK(runtime);
for (PyThreadState *p = interp->tstate_head; p != NULL; p = p->next) {
PyThreadState_Clear(p);
}
HEAD_UNLOCK(runtime);
Py_CLEAR(interp->audit_hooks);
_PyCoreConfig_Clear(&interp->core_config);
Py_CLEAR(interp->codec_search_path);
Py_CLEAR(interp->codec_search_cache);
@ -1057,6 +1081,10 @@ _PyThread_CurrentFrames(void)
PyObject *result;
PyInterpreterState *i;
if (PySys_Audit("sys._current_frames", NULL) < 0) {
return NULL;
}
result = PyDict_New();
if (result == NULL)
return NULL;

View file

@ -1091,6 +1091,12 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals,
co = PyAST_CompileObject(mod, filename, flags, -1, arena);
if (co == NULL)
return NULL;
if (PySys_Audit("exec", "O", co) < 0) {
Py_DECREF(co);
return NULL;
}
v = run_eval_code_obj(co, globals, locals);
Py_DECREF(co);
return v;

View file

@ -22,7 +22,9 @@ Data members:
#include "pycore_pymem.h"
#include "pycore_pathconfig.h"
#include "pycore_pystate.h"
#include "pycore_tupleobject.h"
#include "pythread.h"
#include "pydtrace.h"
#include "osdefs.h"
#include <locale.h>
@ -111,6 +113,308 @@ PySys_SetObject(const char *name, PyObject *v)
}
}
static int
should_audit(void)
{
PyThreadState *ts = _PyThreadState_GET();
if (!ts) {
return 0;
}
PyInterpreterState *is = ts ? ts->interp : NULL;
return _PyRuntime.audit_hook_head
|| (is && is->audit_hooks)
|| PyDTrace_AUDIT_ENABLED();
}
int
PySys_Audit(const char *event, const char *argFormat, ...)
{
PyObject *eventName = NULL;
PyObject *eventArgs = NULL;
PyObject *hooks = NULL;
PyObject *hook = NULL;
int res = -1;
/* N format is inappropriate, because you do not know
whether the reference is consumed by the call.
Assert rather than exception for perf reasons */
assert(!argFormat || !strchr(argFormat, 'N'));
/* Early exit when no hooks are registered */
if (!should_audit()) {
return 0;
}
_Py_AuditHookEntry *e = _PyRuntime.audit_hook_head;
PyThreadState *ts = _PyThreadState_GET();
PyInterpreterState *is = ts ? ts->interp : NULL;
int dtrace = PyDTrace_AUDIT_ENABLED();
PyObject *exc_type, *exc_value, *exc_tb;
if (ts) {
PyErr_Fetch(&exc_type, &exc_value, &exc_tb);
}
/* Initialize event args now */
if (argFormat && argFormat[0]) {
va_list args;
va_start(args, argFormat);
eventArgs = Py_VaBuildValue(argFormat, args);
if (eventArgs && !PyTuple_Check(eventArgs)) {
PyObject *argTuple = PyTuple_Pack(1, eventArgs);
Py_DECREF(eventArgs);
eventArgs = argTuple;
}
} else {
eventArgs = PyTuple_New(0);
}
if (!eventArgs) {
goto exit;
}
/* Call global hooks */
for (; e; e = e->next) {
if (e->hookCFunction(event, eventArgs, e->userData) < 0) {
goto exit;
}
}
/* Dtrace USDT point */
if (dtrace) {
PyDTrace_AUDIT(event, (void *)eventArgs);
}
/* Call interpreter hooks */
if (is && is->audit_hooks) {
eventName = PyUnicode_FromString(event);
if (!eventName) {
goto exit;
}
hooks = PyObject_GetIter(is->audit_hooks);
if (!hooks) {
goto exit;
}
/* Disallow tracing in hooks unless explicitly enabled */
ts->tracing++;
ts->use_tracing = 0;
while ((hook = PyIter_Next(hooks)) != NULL) {
PyObject *o;
int canTrace = -1;
o = PyObject_GetAttrString(hook, "__cantrace__");
if (o) {
canTrace = PyObject_IsTrue(o);
Py_DECREF(o);
} else if (PyErr_Occurred() &&
PyErr_ExceptionMatches(PyExc_AttributeError)) {
PyErr_Clear();
canTrace = 0;
}
if (canTrace < 0) {
break;
}
if (canTrace) {
ts->use_tracing = (ts->c_tracefunc || ts->c_profilefunc);
ts->tracing--;
}
o = PyObject_CallFunctionObjArgs(hook, eventName,
eventArgs, NULL);
if (canTrace) {
ts->tracing++;
ts->use_tracing = 0;
}
if (!o) {
break;
}
Py_DECREF(o);
Py_CLEAR(hook);
}
ts->use_tracing = (ts->c_tracefunc || ts->c_profilefunc);
ts->tracing--;
if (PyErr_Occurred()) {
goto exit;
}
}
res = 0;
exit:
Py_XDECREF(hook);
Py_XDECREF(hooks);
Py_XDECREF(eventName);
Py_XDECREF(eventArgs);
if (ts) {
if (!res) {
PyErr_Restore(exc_type, exc_value, exc_tb);
} else {
assert(PyErr_Occurred());
Py_XDECREF(exc_type);
Py_XDECREF(exc_value);
Py_XDECREF(exc_tb);
}
}
return res;
}
/* We expose this function primarily for our own cleanup during
* finalization. In general, it should not need to be called,
* and as such it is not defined in any header files.
*/
void _PySys_ClearAuditHooks(void) {
/* Must be finalizing to clear hooks */
_PyRuntimeState *runtime = &_PyRuntime;
PyThreadState *ts = _PyRuntimeState_GetThreadState(runtime);
assert(!ts || _Py_CURRENTLY_FINALIZING(runtime, ts));
if (!ts || !_Py_CURRENTLY_FINALIZING(runtime, ts))
return;
if (Py_VerboseFlag) {
PySys_WriteStderr("# clear sys.audit hooks\n");
}
/* Hooks can abort later hooks for this event, but cannot
abort the clear operation itself. */
PySys_Audit("cpython._PySys_ClearAuditHooks", NULL);
PyErr_Clear();
_Py_AuditHookEntry *e = _PyRuntime.audit_hook_head, *n;
_PyRuntime.audit_hook_head = NULL;
while (e) {
n = e->next;
PyMem_RawFree(e);
e = n;
}
}
int
PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData)
{
/* Invoke existing audit hooks to allow them an opportunity to abort. */
/* Cannot invoke hooks until we are initialized */
if (Py_IsInitialized()) {
if (PySys_Audit("sys.addaudithook", NULL) < 0) {
if (PyErr_ExceptionMatches(PyExc_Exception)) {
/* We do not report errors derived from Exception */
PyErr_Clear();
return 0;
}
return -1;
}
}
_Py_AuditHookEntry *e = _PyRuntime.audit_hook_head;
if (!e) {
e = (_Py_AuditHookEntry*)PyMem_RawMalloc(sizeof(_Py_AuditHookEntry));
_PyRuntime.audit_hook_head = e;
} else {
while (e->next)
e = e->next;
e = e->next = (_Py_AuditHookEntry*)PyMem_RawMalloc(
sizeof(_Py_AuditHookEntry));
}
if (!e) {
if (Py_IsInitialized())
PyErr_NoMemory();
return -1;
}
e->next = NULL;
e->hookCFunction = (Py_AuditHookFunction)hook;
e->userData = userData;
return 0;
}
/*[clinic input]
sys.addaudithook
hook: object
Adds a new audit hook callback.
[clinic start generated code]*/
static PyObject *
sys_addaudithook_impl(PyObject *module, PyObject *hook)
/*[clinic end generated code: output=4f9c17aaeb02f44e input=0f3e191217a45e34]*/
{
/* Invoke existing audit hooks to allow them an opportunity to abort. */
if (PySys_Audit("sys.addaudithook", NULL) < 0) {
if (PyErr_ExceptionMatches(PyExc_Exception)) {
/* We do not report errors derived from Exception */
PyErr_Clear();
Py_RETURN_NONE;
}
return NULL;
}
PyInterpreterState *is = _PyInterpreterState_Get();
if (is->audit_hooks == NULL) {
is->audit_hooks = PyList_New(0);
if (is->audit_hooks == NULL) {
return NULL;
}
}
if (PyList_Append(is->audit_hooks, hook) < 0) {
return NULL;
}
Py_RETURN_NONE;
}
PyDoc_STRVAR(audit_doc,
"audit(event, *args)\n\
\n\
Passes the event to any audit hooks that are attached.");
static PyObject *
sys_audit(PyObject *self, PyObject *const *args, Py_ssize_t argc)
{
if (argc == 0) {
PyErr_SetString(PyExc_TypeError, "audit() missing 1 required positional argument: 'event'");
return NULL;
}
if (!should_audit()) {
Py_RETURN_NONE;
}
PyObject *auditEvent = args[0];
if (!auditEvent) {
PyErr_SetString(PyExc_TypeError, "expected str for argument 'event'");
return NULL;
}
if (!PyUnicode_Check(auditEvent)) {
PyErr_Format(PyExc_TypeError, "expected str for argument 'event', not %.200s",
Py_TYPE(auditEvent)->tp_name);
return NULL;
}
const char *event = PyUnicode_AsUTF8(auditEvent);
if (!event) {
return NULL;
}
PyObject *auditArgs = _PyTuple_FromArray(args + 1, argc - 1);
if (!auditArgs) {
return NULL;
}
int res = PySys_Audit(event, "O", auditArgs);
Py_DECREF(auditArgs);
if (res < 0) {
return NULL;
}
Py_RETURN_NONE;
}
static PyObject *
sys_breakpointhook(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords)
{
@ -1469,6 +1773,10 @@ sys__getframe_impl(PyObject *module, int depth)
{
PyFrameObject *f = _PyThreadState_GET()->frame;
if (PySys_Audit("sys._getframe", "O", f) < 0) {
return NULL;
}
while (depth > 0 && f != NULL) {
f = f->f_back;
--depth;
@ -1642,8 +1950,11 @@ sys_getandroidapilevel_impl(PyObject *module)
#endif /* ANDROID_API_LEVEL */
static PyMethodDef sys_methods[] = {
/* Might as well keep this in alphabetic order */
SYS_ADDAUDITHOOK_METHODDEF
{"audit", (PyCFunction)(void(*)(void))sys_audit, METH_FASTCALL, audit_doc },
{"breakpointhook", (PyCFunction)(void(*)(void))sys_breakpointhook,
METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc},
SYS_CALLSTATS_METHODDEF