gh-111956: Add thread-safe one-time initialization. (gh-111960)

Python/getargs.c

@@ -1877,8 +1877,9 @@ new_kwtuple(const char * const *keywords, int total, int pos)
 }
 
 static int
-_parser_init(struct _PyArg_Parser *parser)
+_parser_init(void *arg)
 {
+    struct _PyArg_Parser *parser = (struct _PyArg_Parser *)arg;
     const char * const *keywords = parser->keywords;
     assert(keywords != NULL);
     assert(parser->pos == 0 &&
@@ -1889,7 +1890,7 @@ _parser_init(struct _PyArg_Parser *parser)
 
     int len, pos;
     if (scan_keywords(keywords, &len, &pos) < 0) {
-        return 0;
+        return -1;
     }
 
     const char *fname, *custommsg = NULL;
@@ -1898,7 +1899,7 @@ _parser_init(struct _PyArg_Parser *parser)
         assert(parser->fname == NULL);
         if (parse_format(parser->format, len, pos,
                          &fname, &custommsg, &min, &max) < 0) {
-            return 0;
+            return -1;
         }
     }
     else {
@@ -1911,7 +1912,7 @@ _parser_init(struct _PyArg_Parser *parser)
     if (kwtuple == NULL) {
         kwtuple = new_kwtuple(keywords, len, pos);
         if (kwtuple == NULL) {
-            return 0;
+            return -1;
         }
         owned = 1;
     }
@@ -1925,40 +1926,27 @@ _parser_init(struct _PyArg_Parser *parser)
     parser->min = min;
     parser->max = max;
     parser->kwtuple = kwtuple;
-    parser->initialized = owned ? 1 : -1;
+    parser->is_kwtuple_owned = owned;
 
     assert(parser->next == NULL);
-    parser->next = _PyRuntime.getargs.static_parsers;
-    _PyRuntime.getargs.static_parsers = parser;
-    return 1;
+    parser->next = _Py_atomic_load_ptr(&_PyRuntime.getargs.static_parsers);
+    do {
+        // compare-exchange updates parser->next on failure
+    } while (_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers,
+                                             &parser->next, parser));
+    return 0;
 }
 
 static int
 parser_init(struct _PyArg_Parser *parser)
 {
-    // volatile as it can be modified by other threads
-    // and should not be optimized or reordered by compiler
-    if (*((volatile int *)&parser->initialized)) {
-        assert(parser->kwtuple != NULL);
-        return 1;
-    }
-    PyThread_acquire_lock(_PyRuntime.getargs.mutex, WAIT_LOCK);
-    // Check again if another thread initialized the parser
-    // while we were waiting for the lock.
-    if (*((volatile int *)&parser->initialized)) {
-        assert(parser->kwtuple != NULL);
-        PyThread_release_lock(_PyRuntime.getargs.mutex);
-        return 1;
-    }
-    int ret = _parser_init(parser);
-    PyThread_release_lock(_PyRuntime.getargs.mutex);
-    return ret;
+    return _PyOnceFlag_CallOnce(&parser->once, &_parser_init, parser);
 }
 
 static void
 parser_clear(struct _PyArg_Parser *parser)
 {
-    if (parser->initialized == 1) {
+    if (parser->is_kwtuple_owned) {
         Py_CLEAR(parser->kwtuple);
     }
 }
@@ -2025,7 +2013,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs,
         return 0;
     }
 
-    if (!parser_init(parser)) {
+    if (parser_init(parser) < 0) {
         return 0;
     }
 
@@ -2258,7 +2246,7 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs,
         args = buf;
     }
 
-    if (!parser_init(parser)) {
+    if (parser_init(parser) < 0) {
         return NULL;
     }
 
@@ -2435,7 +2423,7 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs,
         args = buf;
     }
 
-    if (!parser_init(parser)) {
+    if (parser_init(parser) < 0) {
         return NULL;
     }
 

Python/lock.c

@@ -295,3 +295,61 @@ PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns)
         return _Py_atomic_load_uint8(&evt->v) == _Py_LOCKED;
     }
 }
+
+static int
+unlock_once(_PyOnceFlag *o, int res)
+{
+    // On success (res=0), we set the state to _Py_ONCE_INITIALIZED.
+    // On failure (res=-1), we reset the state to _Py_UNLOCKED.
+    uint8_t new_value;
+    switch (res) {
+        case -1: new_value = _Py_UNLOCKED; break;
+        case  0: new_value = _Py_ONCE_INITIALIZED; break;
+        default: {
+            Py_FatalError("invalid result from _PyOnceFlag_CallOnce");
+            Py_UNREACHABLE();
+            break;
+        }
+    }
+
+    uint8_t old_value = _Py_atomic_exchange_uint8(&o->v, new_value);
+    if ((old_value & _Py_HAS_PARKED) != 0) {
+        // wake up anyone waiting on the once flag
+        _PyParkingLot_UnparkAll(&o->v);
+    }
+    return res;
+}
+
+int
+_PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg)
+{
+    uint8_t v = _Py_atomic_load_uint8(&flag->v);
+    for (;;) {
+        if (v == _Py_UNLOCKED) {
+            if (!_Py_atomic_compare_exchange_uint8(&flag->v, &v, _Py_LOCKED)) {
+                continue;
+            }
+            int res = fn(arg);
+            return unlock_once(flag, res);
+        }
+
+        if (v == _Py_ONCE_INITIALIZED) {
+            return 0;
+        }
+
+        // The once flag is initializing (locked).
+        assert((v & _Py_LOCKED));
+        if (!(v & _Py_HAS_PARKED)) {
+            // We are the first waiter. Set the _Py_HAS_PARKED flag.
+            uint8_t new_value = v | _Py_HAS_PARKED;
+            if (!_Py_atomic_compare_exchange_uint8(&flag->v, &v, new_value)) {
+                continue;
+            }
+            v = new_value;
+        }
+
+        // Wait for initialization to finish.
+        _PyParkingLot_Park(&flag->v, &v, sizeof(v), -1, NULL, 1);
+        v = _Py_atomic_load_uint8(&flag->v);
+    }
+}

Python/pystate.c

@@ -379,12 +379,11 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS
 static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime);
 _Py_COMP_DIAG_POP
 
-#define NUMLOCKS 9
+#define NUMLOCKS 8
 #define LOCKS_INIT(runtime) \
     { \
         &(runtime)->interpreters.mutex, \
         &(runtime)->xi.registry.mutex, \
-        &(runtime)->getargs.mutex, \
         &(runtime)->unicode_state.ids.lock, \
         &(runtime)->imports.extensions.mutex, \
         &(runtime)->ceval.pending_mainthread.lock, \