closes bpo-31650: PEP 552 (Deterministic pycs) implementation (#4575)

Python now supports checking bytecode cache up-to-dateness with a hash of the
source contents rather than volatile source metadata. See the PEP for details.

While a fairly straightforward idea, quite a lot of code had to be modified due
to the pervasiveness of pyc implementation details in the codebase. Changes in
this commit include:

- The core changes to importlib to understand how to read, validate, and
  regenerate hash-based pycs.

- Support for generating hash-based pycs in py_compile and compileall.

- Modifications to our siphash implementation to support passing a custom
  key. We then expose it to importlib through _imp.

- Updates to all places in the interpreter, standard library, and tests that
  manually generate or parse pyc files to grok the new format.

- Support in the interpreter command line code for long options like
  --check-hash-based-pycs.

- Tests and documentation for all of the above.
This commit is contained in:
Benjamin Peterson 2017-12-09 10:26:52 -08:00 committed by GitHub
parent 28d8d14013
commit 42aa93b8ff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
33 changed files with 3364 additions and 2505 deletions

View file

@ -51,7 +51,8 @@ void _PyOS_ResetGetOpt(void)
opt_ptr = L"";
}
int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring)
int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring,
const _PyOS_LongOption *longopts, int *longindex)
{
wchar_t *ptr;
wchar_t option;
@ -86,13 +87,41 @@ int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring)
return 'V';
}
opt_ptr = &argv[_PyOS_optind++][1];
}
if ((option = *opt_ptr++) == L'\0')
return -1;
if (option == L'-') {
// Parse long option.
if (*opt_ptr == L'\0') {
fprintf(stderr, "expected long option\n");
return -1;
}
*longindex = 0;
const _PyOS_LongOption *opt;
for (opt = &longopts[*longindex]; opt->name; opt = &longopts[++(*longindex)]) {
if (!wcscmp(opt->name, opt_ptr))
break;
}
if (!opt->name) {
fprintf(stderr, "unknown option %ls\n", argv[_PyOS_optind - 1]);
return '_';
}
opt_ptr = L"";
if (!opt->has_arg) {
return opt->val;
}
if (_PyOS_optind >= argc) {
fprintf(stderr, "Argument expected for the %ls options\n",
argv[_PyOS_optind - 1]);
return '_';
}
_PyOS_optarg = argv[_PyOS_optind++];
return opt->val;
}
if (option == 'J') {
if (_PyOS_opterr)
fprintf(stderr, "-J is reserved for Jython\n");