gh-106149: move CFG and basicblock definitions into flowgraph.c, use them as opaque types in compile.c (#107639)

Include/internal/pycore_compile.h

@@ -54,6 +54,11 @@ typedef struct {
     int s_next_free_label; /* next free label id */
 } _PyCompile_InstructionSequence;
 
+int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int lbl);
+int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq,
+                                         int opcode, int oparg,
+                                         _PyCompilerSrcLocation loc);
+
 typedef struct {
     PyObject *u_name;
     PyObject *u_qualname;  /* dot-separated qualified name (lazy) */

Include/internal/pycore_flowgraph.h

@@ -11,88 +11,26 @@ extern "C" {
 #include "pycore_opcode_utils.h"
 #include "pycore_compile.h"
 
-
-typedef struct {
-    int i_opcode;
-    int i_oparg;
-    _PyCompilerSrcLocation i_loc;
-    struct _PyCfgBasicblock_ *i_target; /* target block (if jump instruction) */
-    struct _PyCfgBasicblock_ *i_except; /* target block when exception is raised */
-} _PyCfgInstruction;
-
 typedef struct {
     int id;
 } _PyCfgJumpTargetLabel;
 
+struct _PyCfgBuilder;
 
-typedef struct {
+int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl);
-    struct _PyCfgBasicblock_ *handlers[CO_MAXBLOCKS+1];
+int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc);
-    int depth;
-} _PyCfgExceptStack;
 
-typedef struct _PyCfgBasicblock_ {
+struct _PyCfgBuilder* _PyCfgBuilder_New(void);
-    /* Each basicblock in a compilation unit is linked via b_list in the
+void _PyCfgBuilder_Free(struct _PyCfgBuilder *g);
-       reverse order that the block are allocated.  b_list points to the next
+int _PyCfgBuilder_CheckSize(struct _PyCfgBuilder* g);
-       block in this list, not to be confused with b_next, which is next by
-       control flow. */
-    struct _PyCfgBasicblock_ *b_list;
-    /* The label of this block if it is a jump target, -1 otherwise */
-    _PyCfgJumpTargetLabel b_label;
-    /* Exception stack at start of block, used by assembler to create the exception handling table */
-    _PyCfgExceptStack *b_exceptstack;
-    /* pointer to an array of instructions, initially NULL */
-    _PyCfgInstruction *b_instr;
-    /* If b_next is non-NULL, it is a pointer to the next
-       block reached by normal control flow. */
-    struct _PyCfgBasicblock_ *b_next;
-    /* number of instructions used */
-    int b_iused;
-    /* length of instruction array (b_instr) */
-    int b_ialloc;
-    /* Used by add_checks_for_loads_of_unknown_variables */
-    uint64_t b_unsafe_locals_mask;
-    /* Number of predecessors that a block has. */
-    int b_predecessors;
-    /* depth of stack upon entry of block, computed by stackdepth() */
-    int b_startdepth;
-    /* Basic block is an exception handler that preserves lasti */
-    unsigned b_preserve_lasti : 1;
-    /* Used by compiler passes to mark whether they have visited a basic block. */
-    unsigned b_visited : 1;
-    /* b_except_handler is used by the cold-detection algorithm to mark exception targets */
-    unsigned b_except_handler : 1;
-    /* b_cold is true if this block is not perf critical (like an exception handler) */
-    unsigned b_cold : 1;
-    /* b_warm is used by the cold-detection algorithm to mark blocks which are definitely not cold */
-    unsigned b_warm : 1;
-} _PyCfgBasicblock;
 
-int _PyBasicblock_InsertInstruction(_PyCfgBasicblock *block, int pos, _PyCfgInstruction *instr);
+int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject *const_cache,
-
-typedef struct cfg_builder_ {
-    /* The entryblock, at which control flow begins. All blocks of the
-       CFG are reachable through the b_next links */
-    _PyCfgBasicblock *g_entryblock;
-    /* Pointer to the most recently allocated block.  By following
-       b_list links, you can reach all allocated blocks. */
-    _PyCfgBasicblock *g_block_list;
-    /* pointer to the block currently being constructed */
-    _PyCfgBasicblock *g_curblock;
-    /* label for the next instruction to be placed */
-    _PyCfgJumpTargetLabel g_current_label;
-} _PyCfgBuilder;
-
-int _PyCfgBuilder_UseLabel(_PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl);
-int _PyCfgBuilder_Addop(_PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc);
-
-int _PyCfgBuilder_Init(_PyCfgBuilder *g);
-void _PyCfgBuilder_Fini(_PyCfgBuilder *g);
-
-int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_cache,
                             int nlocals, int nparams, int firstlineno);
-int _PyCfg_Stackdepth(_PyCfgBuilder *g);
+
-void _PyCfg_ConvertPseudoOps(_PyCfgBasicblock *entryblock);
+int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_InstructionSequence *seq);
-int _PyCfg_ResolveJumps(_PyCfgBuilder *g);
+int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd,
+                                             int code_flags, int *stackdepth, int *nlocalsplus,
+                                             _PyCompile_InstructionSequence *seq);
 
 PyCodeObject *
 _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache,

Python/compile.c

@@ -70,9 +70,7 @@
         && ((C)->u->u_ste->ste_type == ModuleBlock))
 
 typedef _PyCompilerSrcLocation location;
-typedef _PyCfgInstruction cfg_instr;
+typedef struct _PyCfgBuilder cfg_builder;
-typedef _PyCfgBasicblock basicblock;
-typedef _PyCfgBuilder cfg_builder;
 
 #define LOCATION(LNO, END_LNO, COL, END_COL) \
     ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)})
@@ -101,7 +99,7 @@ static jump_target_label NO_LABEL = {-1};
     }
 
 #define USE_LABEL(C, LBL) \
-    RETURN_IF_ERROR(instr_sequence_use_label(INSTR_SEQUENCE(C), (LBL).id))
+    RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id))
 
 
 /* fblockinfo tracks the current frame block.
@@ -218,8 +216,9 @@ instr_sequence_new_label(instr_sequence *seq)
     return lbl;
 }
 
-static int
+int
-instr_sequence_use_label(instr_sequence *seq, int lbl) {
+_PyCompile_InstructionSequence_UseLabel(instr_sequence *seq, int lbl)
+{
     int old_size = seq->s_labelmap_size;
     RETURN_IF_ERROR(
         _PyCompile_EnsureArrayLargeEnough(lbl,
@@ -238,8 +237,9 @@ instr_sequence_use_label(instr_sequence *seq, int lbl) {
 
 #define MAX_OPCODE 511
 
-static int
+int
-instr_sequence_addop(instr_sequence *seq, int opcode, int oparg, location loc)
+_PyCompile_InstructionSequence_Addop(instr_sequence *seq, int opcode, int oparg,
+                                     location loc)
 {
     assert(0 <= opcode && opcode <= MAX_OPCODE);
     assert(IS_WITHIN_OPCODE_RANGE(opcode));
@@ -288,10 +288,12 @@ instr_sequence_fini(instr_sequence *seq) {
     seq->s_instrs = NULL;
 }
 
-static int
+static cfg_builder*
-instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) {
+instr_sequence_to_cfg(instr_sequence *seq) {
-    memset(g, 0, sizeof(cfg_builder));
+    cfg_builder *g = _PyCfgBuilder_New();
-    RETURN_IF_ERROR(_PyCfgBuilder_Init(g));
+    if (g == NULL) {
+        return NULL;
+    }
 
     /* There can be more than one label for the same offset. The
      * offset2lbl maping selects one of them which we use consistently.
@@ -300,7 +302,7 @@ instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) {
     int *offset2lbl = PyMem_Malloc(seq->s_used * sizeof(int));
     if (offset2lbl == NULL) {
         PyErr_NoMemory();
-        return ERROR;
+        goto error;
     }
     for (int i = 0; i < seq->s_used; i++) {
         offset2lbl[i] = -1;
@@ -336,23 +338,17 @@ instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) {
             goto error;
         }
     }
+    if (_PyCfgBuilder_CheckSize(g) < 0) {
+        goto error;
+    }
     PyMem_Free(offset2lbl);
-
+    return g;
-    int nblocks = 0;
-    for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) {
-        nblocks++;
-    }
-    if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) {
-        PyErr_NoMemory();
-        return ERROR;
-    }
-    return SUCCESS;
 error:
+    _PyCfgBuilder_Free(g);
     PyMem_Free(offset2lbl);
-    return ERROR;
+    return NULL;
 }
 
-
 /* The following items change on entry and exit of code blocks.
    They must be saved and restored when returning to a block.
 */
@@ -920,7 +916,7 @@ codegen_addop_noarg(instr_sequence *seq, int opcode, location loc)
 {
     assert(!OPCODE_HAS_ARG(opcode));
     assert(!IS_ASSEMBLER_OPCODE(opcode));
-    return instr_sequence_addop(seq, opcode, 0, loc);
+    return _PyCompile_InstructionSequence_Addop(seq, opcode, 0, loc);
 }
 
 static Py_ssize_t
@@ -1153,7 +1149,7 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc)
 
     int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int);
     assert(!IS_ASSEMBLER_OPCODE(opcode));
-    return instr_sequence_addop(seq, opcode, oparg_, loc);
+    return _PyCompile_InstructionSequence_Addop(seq, opcode, oparg_, loc);
 }
 
 static int
@@ -1163,7 +1159,7 @@ codegen_addop_j(instr_sequence *seq, location loc,
     assert(IS_LABEL(target));
     assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode));
     assert(!IS_ASSEMBLER_OPCODE(opcode));
-    return instr_sequence_addop(seq, opcode, target.id, loc);
+    return _PyCompile_InstructionSequence_Addop(seq, opcode, target.id, loc);
 }
 
 #define RETURN_IF_ERROR_IN_SCOPE(C, CALL) { \
@@ -7492,201 +7488,6 @@ _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj)
     return SUCCESS;
 }
 
-
-static int *
-build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd)
-{
-    int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames);
-    int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars);
-    int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars);
-
-    int noffsets = ncellvars + nfreevars;
-    int *fixed = PyMem_New(int, noffsets);
-    if (fixed == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-    for (int i = 0; i < noffsets; i++) {
-        fixed[i] = nlocals + i;
-    }
-
-    PyObject *varname, *cellindex;
-    Py_ssize_t pos = 0;
-    while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) {
-        PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname);
-        if (varindex != NULL) {
-            assert(PyLong_AS_LONG(cellindex) < INT_MAX);
-            assert(PyLong_AS_LONG(varindex) < INT_MAX);
-            int oldindex = (int)PyLong_AS_LONG(cellindex);
-            int argoffset = (int)PyLong_AS_LONG(varindex);
-            fixed[oldindex] = argoffset;
-        }
-    }
-
-    return fixed;
-}
-
-#define IS_GENERATOR(CF) \
-    ((CF) & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR))
-
-static int
-insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock,
-                           int *fixed, int nfreevars, int code_flags)
-{
-    assert(umd->u_firstlineno > 0);
-
-    /* Add the generator prefix instructions. */
-    if (IS_GENERATOR(code_flags)) {
-        /* Note that RETURN_GENERATOR + POP_TOP have a net stack effect
-         * of 0. This is because RETURN_GENERATOR pushes an element
-         * with _PyFrame_StackPush before switching stacks.
-         */
-        cfg_instr make_gen = {
-            .i_opcode = RETURN_GENERATOR,
-            .i_oparg = 0,
-            .i_loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1),
-            .i_target = NULL,
-        };
-        RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 0, &make_gen));
-        cfg_instr pop_top = {
-            .i_opcode = POP_TOP,
-            .i_oparg = 0,
-            .i_loc = NO_LOCATION,
-            .i_target = NULL,
-        };
-        RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 1, &pop_top));
-    }
-
-    /* Set up cells for any variable that escapes, to be put in a closure. */
-    const int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars);
-    if (ncellvars) {
-        // umd->u_cellvars has the cells out of order so we sort them
-        // before adding the MAKE_CELL instructions.  Note that we
-        // adjust for arg cells, which come first.
-        const int nvars = ncellvars + (int)PyDict_GET_SIZE(umd->u_varnames);
-        int *sorted = PyMem_RawCalloc(nvars, sizeof(int));
-        if (sorted == NULL) {
-            PyErr_NoMemory();
-            return ERROR;
-        }
-        for (int i = 0; i < ncellvars; i++) {
-            sorted[fixed[i]] = i + 1;
-        }
-        for (int i = 0, ncellsused = 0; ncellsused < ncellvars; i++) {
-            int oldindex = sorted[i] - 1;
-            if (oldindex == -1) {
-                continue;
-            }
-            cfg_instr make_cell = {
-                .i_opcode = MAKE_CELL,
-                // This will get fixed in offset_derefs().
-                .i_oparg = oldindex,
-                .i_loc = NO_LOCATION,
-                .i_target = NULL,
-            };
-            if (_PyBasicblock_InsertInstruction(entryblock, ncellsused, &make_cell) < 0) {
-                PyMem_RawFree(sorted);
-                return ERROR;
-            }
-            ncellsused += 1;
-        }
-        PyMem_RawFree(sorted);
-    }
-
-    if (nfreevars) {
-        cfg_instr copy_frees = {
-            .i_opcode = COPY_FREE_VARS,
-            .i_oparg = nfreevars,
-            .i_loc = NO_LOCATION,
-            .i_target = NULL,
-        };
-        RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 0, &copy_frees));
-    }
-
-    return SUCCESS;
-}
-
-static int
-fix_cell_offsets(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixedmap)
-{
-    int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames);
-    int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars);
-    int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars);
-    int noffsets = ncellvars + nfreevars;
-
-    // First deal with duplicates (arg cells).
-    int numdropped = 0;
-    for (int i = 0; i < noffsets ; i++) {
-        if (fixedmap[i] == i + nlocals) {
-            fixedmap[i] -= numdropped;
-        }
-        else {
-            // It was a duplicate (cell/arg).
-            numdropped += 1;
-        }
-    }
-
-    // Then update offsets, either relative to locals or by cell2arg.
-    for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
-        for (int i = 0; i < b->b_iused; i++) {
-            cfg_instr *inst = &b->b_instr[i];
-            // This is called before extended args are generated.
-            assert(inst->i_opcode != EXTENDED_ARG);
-            int oldoffset = inst->i_oparg;
-            switch(inst->i_opcode) {
-                case MAKE_CELL:
-                case LOAD_CLOSURE:
-                case LOAD_DEREF:
-                case STORE_DEREF:
-                case DELETE_DEREF:
-                case LOAD_FROM_DICT_OR_DEREF:
-                    assert(oldoffset >= 0);
-                    assert(oldoffset < noffsets);
-                    assert(fixedmap[oldoffset] >= 0);
-                    inst->i_oparg = fixedmap[oldoffset];
-            }
-        }
-    }
-
-    return numdropped;
-}
-
-
-static int
-prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_flags)
-{
-    assert(PyDict_GET_SIZE(umd->u_varnames) < INT_MAX);
-    assert(PyDict_GET_SIZE(umd->u_cellvars) < INT_MAX);
-    assert(PyDict_GET_SIZE(umd->u_freevars) < INT_MAX);
-    int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames);
-    int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars);
-    int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars);
-    assert(INT_MAX - nlocals - ncellvars > 0);
-    assert(INT_MAX - nlocals - ncellvars - nfreevars > 0);
-    int nlocalsplus = nlocals + ncellvars + nfreevars;
-    int* cellfixedoffsets = build_cellfixedoffsets(umd);
-    if (cellfixedoffsets == NULL) {
-        return ERROR;
-    }
-
-
-    // This must be called before fix_cell_offsets().
-    if (insert_prefix_instructions(umd, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) {
-        PyMem_Free(cellfixedoffsets);
-        return ERROR;
-    }
-
-    int numdropped = fix_cell_offsets(umd, g->g_entryblock, cellfixedoffsets);
-    PyMem_Free(cellfixedoffsets);  // At this point we're done with it.
-    cellfixedoffsets = NULL;
-    if (numdropped < 0) {
-        return ERROR;
-    }
-
-    nlocalsplus -= numdropped;
-    return nlocalsplus;
-}
-
 static int
 add_return_at_end(struct compiler *c, int addNone)
 {
@@ -7700,12 +7501,11 @@ add_return_at_end(struct compiler *c, int addNone)
     return SUCCESS;
 }
 
-static int cfg_to_instr_sequence(cfg_builder *g, instr_sequence *seq);
-
 static PyCodeObject *
 optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache,
                    int code_flags, PyObject *filename)
 {
+    cfg_builder *g = NULL;
     instr_sequence optimized_instrs;
     memset(&optimized_instrs, 0, sizeof(instr_sequence));
 
@@ -7714,51 +7514,28 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache,
     if (consts == NULL) {
         goto error;
     }
-    cfg_builder g;
+    g = instr_sequence_to_cfg(&u->u_instr_sequence);
-    if (instr_sequence_to_cfg(&u->u_instr_sequence, &g) < 0) {
+    if (g == NULL) {
         goto error;
     }
-    int nparams = (int)PyList_GET_SIZE(u->u_ste->ste_varnames);
     int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames);
+    int nparams = (int)PyList_GET_SIZE(u->u_ste->ste_varnames);
     assert(u->u_metadata.u_firstlineno);
-    if (_PyCfg_OptimizeCodeUnit(&g, consts, const_cache, nlocals,
+
+    if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals,
                                 nparams, u->u_metadata.u_firstlineno) < 0) {
         goto error;
     }
 
-    int stackdepth = _PyCfg_Stackdepth(&g);
+    int stackdepth;
-    if (stackdepth < 0) {
+    int nlocalsplus;
+    if (_PyCfg_OptimizedCfgToInstructionSequence(g, &u->u_metadata, code_flags,
+                                                 &stackdepth, &nlocalsplus,
+                                                 &optimized_instrs) < 0) {
         goto error;
     }
 
-    /* prepare_localsplus adds instructions for generators that push
-     * and pop an item on the stack. This assertion makes sure there
-     * is space on the stack for that.
-     * It should always be true, because a generator must have at
-     * least one expression or call to INTRINSIC_STOPITERATION_ERROR,
-     * which requires stackspace.
-     */
-    assert(!(IS_GENERATOR(code_flags) && stackdepth == 0));
-
     /** Assembly **/
-    int nlocalsplus = prepare_localsplus(&u->u_metadata, &g, code_flags);
-    if (nlocalsplus < 0) {
-        goto error;
-    }
-
-    _PyCfg_ConvertPseudoOps(g.g_entryblock);
-
-    /* Order of basic blocks must have been determined by now */
-
-    if (_PyCfg_ResolveJumps(&g) < 0) {
-        goto error;
-    }
-
-    /* Can't modify the bytecode after computing jump offsets. */
-
-    if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) {
-        goto error;
-    }
 
     co = _PyAssemble_MakeCodeObject(&u->u_metadata, const_cache, consts,
                                     stackdepth, &optimized_instrs, nlocalsplus,
@@ -7767,7 +7544,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache,
 error:
     Py_XDECREF(consts);
     instr_sequence_fini(&optimized_instrs);
-    _PyCfgBuilder_Fini(&g);
+    _PyCfgBuilder_Free(g);
     return co;
 }
 
@@ -7790,39 +7567,6 @@ optimize_and_assemble(struct compiler *c, int addNone)
     return optimize_and_assemble_code_unit(u, const_cache, code_flags, filename);
 }
 
-static int
-cfg_to_instr_sequence(cfg_builder *g, instr_sequence *seq)
-{
-    int lbl = 0;
-    for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) {
-        b->b_label = (jump_target_label){lbl};
-        lbl += b->b_iused;
-    }
-    for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) {
-        RETURN_IF_ERROR(instr_sequence_use_label(seq, b->b_label.id));
-        for (int i = 0; i < b->b_iused; i++) {
-            cfg_instr *instr = &b->b_instr[i];
-            if (OPCODE_HAS_JUMP(instr->i_opcode)) {
-                instr->i_oparg = instr->i_target->b_label.id;
-            }
-            RETURN_IF_ERROR(
-                instr_sequence_addop(seq, instr->i_opcode, instr->i_oparg, instr->i_loc));
-
-            _PyCompile_ExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info;
-            if (instr->i_except != NULL) {
-                hi->h_label = instr->i_except->b_label.id;
-                hi->h_startdepth = instr->i_except->b_startdepth;
-                hi->h_preserve_lasti = instr->i_except->b_preserve_lasti;
-            }
-            else {
-                hi->h_label = -1;
-            }
-        }
-    }
-    return SUCCESS;
-}
-
-
 /* Access to compiler optimizations for unit tests.
  *
  * _PyCompile_CodeGen takes and AST, applies code-gen and
@@ -7872,7 +7616,7 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq)
 
     for (int i = 0; i < num_insts; i++) {
         if (is_target[i]) {
-            if (instr_sequence_use_label(seq, i) < 0) {
+            if (_PyCompile_InstructionSequence_UseLabel(seq, i) < 0) {
                 goto error;
             }
         }
@@ -7912,7 +7656,7 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq)
         if (PyErr_Occurred()) {
             goto error;
         }
-        if (instr_sequence_addop(seq, opcode, oparg, loc) < 0) {
+        if (_PyCompile_InstructionSequence_Addop(seq, opcode, oparg, loc) < 0) {
             goto error;
         }
     }
@@ -7923,23 +7667,26 @@ error:
     return ERROR;
 }
 
-static int
+static cfg_builder*
-instructions_to_cfg(PyObject *instructions, cfg_builder *g)
+instructions_to_cfg(PyObject *instructions)
 {
+    cfg_builder *g = NULL;
     instr_sequence seq;
     memset(&seq, 0, sizeof(instr_sequence));
 
     if (instructions_to_instr_sequence(instructions, &seq) < 0) {
         goto error;
     }
-    if (instr_sequence_to_cfg(&seq, g) < 0) {
+    g = instr_sequence_to_cfg(&seq);
+    if (g == NULL) {
         goto error;
     }
     instr_sequence_fini(&seq);
-    return SUCCESS;
+    return g;
 error:
+    _PyCfgBuilder_Free(g);
     instr_sequence_fini(&seq);
-    return ERROR;
+    return NULL;
 }
 
 static PyObject *
@@ -7978,42 +7725,14 @@ error:
 static PyObject *
 cfg_to_instructions(cfg_builder *g)
 {
-    PyObject *instructions = PyList_New(0);
+    instr_sequence seq;
-    if (instructions == NULL) {
+    memset(&seq, 0, sizeof(seq));
+    if (_PyCfg_ToInstructionSequence(g, &seq) < 0) {
         return NULL;
     }
-    int lbl = 0;
+    PyObject *res = instr_sequence_to_instructions(&seq);
-    for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) {
+    instr_sequence_fini(&seq);
-        b->b_label = (jump_target_label){lbl};
+    return res;
-        lbl += b->b_iused;
-    }
-    for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) {
-        for (int i = 0; i < b->b_iused; i++) {
-            cfg_instr *instr = &b->b_instr[i];
-            location loc = instr->i_loc;
-            int arg = HAS_TARGET(instr->i_opcode) ?
-                      instr->i_target->b_label.id : instr->i_oparg;
-
-            PyObject *inst_tuple = Py_BuildValue(
-                "(iiiiii)", instr->i_opcode, arg,
-                loc.lineno, loc.end_lineno,
-                loc.col_offset, loc.end_col_offset);
-            if (inst_tuple == NULL) {
-                goto error;
-            }
-
-            if (PyList_Append(instructions, inst_tuple) != 0) {
-                Py_DECREF(inst_tuple);
-                goto error;
-            }
-            Py_DECREF(inst_tuple);
-        }
-    }
-
-    return instructions;
-error:
-    Py_DECREF(instructions);
-    return NULL;
 }
 
 // C implementation of inspect.cleandoc()
@@ -8195,34 +7914,36 @@ finally:
 PyObject *
 _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts, int nlocals)
 {
+    cfg_builder *g = NULL;
     PyObject *res = NULL;
     PyObject *const_cache = PyDict_New();
     if (const_cache == NULL) {
         return NULL;
     }
 
-    cfg_builder g;
+    g = instructions_to_cfg(instructions);
-    if (instructions_to_cfg(instructions, &g) < 0) {
+    if (g == NULL) {
         goto error;
     }
     int nparams = 0, firstlineno = 1;
-    if (_PyCfg_OptimizeCodeUnit(&g, consts, const_cache, nlocals,
+    if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals,
                                 nparams, firstlineno) < 0) {
         goto error;
     }
-    res = cfg_to_instructions(&g);
+    res = cfg_to_instructions(g);
 error:
     Py_DECREF(const_cache);
-    _PyCfgBuilder_Fini(&g);
+    _PyCfgBuilder_Free(g);
     return res;
 }
 
-int _PyCfg_JumpLabelsToTargets(basicblock *entryblock);
+int _PyCfg_JumpLabelsToTargets(cfg_builder *g);
 
 PyCodeObject *
 _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename,
                     PyObject *instructions)
 {
+    cfg_builder *g = NULL;
     PyCodeObject *co = NULL;
     instr_sequence optimized_instrs;
     memset(&optimized_instrs, 0, sizeof(instr_sequence));
@@ -8232,37 +7953,20 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename,
         return NULL;
     }
 
-    cfg_builder g;
+    g = instructions_to_cfg(instructions);
-    if (instructions_to_cfg(instructions, &g) < 0) {
+    if (g == NULL) {
         goto error;
     }
 
-    if (_PyCfg_JumpLabelsToTargets(g.g_entryblock) < 0) {
+    if (_PyCfg_JumpLabelsToTargets(g) < 0) {
-        goto error;
-    }
-
-    int stackdepth = _PyCfg_Stackdepth(&g);
-    if (stackdepth < 0) {
         goto error;
     }
 
     int code_flags = 0;
-    int nlocalsplus = prepare_localsplus(umd, &g, code_flags);
+    int stackdepth, nlocalsplus;
-    if (nlocalsplus < 0) {
+    if (_PyCfg_OptimizedCfgToInstructionSequence(g, umd, code_flags,
-        goto error;
+                                                 &stackdepth, &nlocalsplus,
-    }
+                                                 &optimized_instrs) < 0) {
-
-    _PyCfg_ConvertPseudoOps(g.g_entryblock);
-
-    /* Order of basic blocks must have been determined by now */
-
-    if (_PyCfg_ResolveJumps(&g) < 0) {
-        goto error;
-    }
-
-    /* Can't modify the bytecode after computing jump offsets. */
-
-    if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) {
         goto error;
     }
 
@@ -8277,7 +7981,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename,
 
 error:
     Py_DECREF(const_cache);
-    _PyCfgBuilder_Fini(&g);
+    _PyCfgBuilder_Free(g);
     instr_sequence_fini(&optimized_instrs);
     return co;
 }

Python/flowgraph.c

@@ -24,15 +24,75 @@
 
 typedef _PyCompilerSrcLocation location;
 typedef _PyCfgJumpTargetLabel jump_target_label;
-typedef _PyCfgBasicblock basicblock;
+
-typedef _PyCfgBuilder cfg_builder;
+typedef struct _PyCfgInstruction {
-typedef _PyCfgInstruction cfg_instr;
+    int i_opcode;
+    int i_oparg;
+    _PyCompilerSrcLocation i_loc;
+    struct _PyCfgBasicblock *i_target; /* target block (if jump instruction) */
+    struct _PyCfgBasicblock *i_except; /* target block when exception is raised */
+} cfg_instr;
+
+typedef struct _PyCfgBasicblock {
+    /* Each basicblock in a compilation unit is linked via b_list in the
+       reverse order that the block are allocated.  b_list points to the next
+       block in this list, not to be confused with b_next, which is next by
+       control flow. */
+    struct _PyCfgBasicblock *b_list;
+    /* The label of this block if it is a jump target, -1 otherwise */
+    _PyCfgJumpTargetLabel b_label;
+    /* Exception stack at start of block, used by assembler to create the exception handling table */
+    struct _PyCfgExceptStack *b_exceptstack;
+    /* pointer to an array of instructions, initially NULL */
+    cfg_instr *b_instr;
+    /* If b_next is non-NULL, it is a pointer to the next
+       block reached by normal control flow. */
+    struct _PyCfgBasicblock *b_next;
+    /* number of instructions used */
+    int b_iused;
+    /* length of instruction array (b_instr) */
+    int b_ialloc;
+    /* Used by add_checks_for_loads_of_unknown_variables */
+    uint64_t b_unsafe_locals_mask;
+    /* Number of predecessors that a block has. */
+    int b_predecessors;
+    /* depth of stack upon entry of block, computed by stackdepth() */
+    int b_startdepth;
+    /* Basic block is an exception handler that preserves lasti */
+    unsigned b_preserve_lasti : 1;
+    /* Used by compiler passes to mark whether they have visited a basic block. */
+    unsigned b_visited : 1;
+    /* b_except_handler is used by the cold-detection algorithm to mark exception targets */
+    unsigned b_except_handler : 1;
+    /* b_cold is true if this block is not perf critical (like an exception handler) */
+    unsigned b_cold : 1;
+    /* b_warm is used by the cold-detection algorithm to mark blocks which are definitely not cold */
+    unsigned b_warm : 1;
+} basicblock;
+
+
+struct _PyCfgBuilder {
+    /* The entryblock, at which control flow begins. All blocks of the
+       CFG are reachable through the b_next links */
+    struct _PyCfgBasicblock *g_entryblock;
+    /* Pointer to the most recently allocated block.  By following
+       b_list links, you can reach all allocated blocks. */
+    struct _PyCfgBasicblock *g_block_list;
+    /* pointer to the block currently being constructed */
+    struct _PyCfgBasicblock *g_curblock;
+    /* label for the next instruction to be placed */
+    _PyCfgJumpTargetLabel g_current_label;
+};
+
+typedef struct _PyCfgBuilder cfg_builder;
 
 static const jump_target_label NO_LABEL = {-1};
 
 #define SAME_LABEL(L1, L2) ((L1).id == (L2).id)
 #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL)))
 
+#define LOCATION(LNO, END_LNO, COL, END_COL) \
+    ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)})
 
 static inline int
 is_block_push(cfg_instr *i)
@@ -50,7 +110,7 @@ is_jump(cfg_instr *i)
 #define INSTR_SET_OP1(I, OP, ARG) \
     do { \
         assert(OPCODE_HAS_ARG(OP)); \
-        _PyCfgInstruction *_instr__ptr_ = (I); \
+        cfg_instr *_instr__ptr_ = (I); \
         _instr__ptr_->i_opcode = (OP); \
         _instr__ptr_->i_oparg = (ARG); \
     } while (0);
@@ -59,7 +119,7 @@ is_jump(cfg_instr *i)
 #define INSTR_SET_OP0(I, OP) \
     do { \
         assert(!OPCODE_HAS_ARG(OP)); \
-        _PyCfgInstruction *_instr__ptr_ = (I); \
+        cfg_instr *_instr__ptr_ = (I); \
         _instr__ptr_->i_opcode = (OP); \
         _instr__ptr_->i_oparg = 0; \
     } while (0);
@@ -148,8 +208,8 @@ basicblock_last_instr(const basicblock *b) {
 }
 
 static inline int
-basicblock_nofallthrough(const _PyCfgBasicblock *b) {
+basicblock_nofallthrough(const basicblock *b) {
-    _PyCfgInstruction *last = basicblock_last_instr(b);
+    cfg_instr *last = basicblock_last_instr(b);
     return (last &&
             (IS_SCOPE_EXIT_OPCODE(last->i_opcode) ||
              IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)));
@@ -175,8 +235,8 @@ copy_basicblock(cfg_builder *g, basicblock *block)
     return result;
 }
 
-int
+static int
-_PyBasicblock_InsertInstruction(basicblock *block, int pos, cfg_instr *instr) {
+basicblock_insert_instruction(basicblock *block, int pos, cfg_instr *instr) {
     RETURN_IF_ERROR(basicblock_next_instr(block));
     for (int i = block->b_iused - 1; i > pos; i--) {
         block->b_instr[i] = block->b_instr[i-1];
@@ -311,8 +371,8 @@ cfg_builder_check(cfg_builder *g)
 }
 #endif
 
-int
+static int
-_PyCfgBuilder_Init(cfg_builder *g)
+init_cfg_builder(cfg_builder *g)
 {
     g->g_block_list = NULL;
     basicblock *block = cfg_builder_new_block(g);
@@ -324,9 +384,28 @@ _PyCfgBuilder_Init(cfg_builder *g)
     return SUCCESS;
 }
 
-void
+cfg_builder *
-_PyCfgBuilder_Fini(cfg_builder* g)
+_PyCfgBuilder_New(void)
 {
+    cfg_builder *g = PyMem_Malloc(sizeof(cfg_builder));
+    if (g == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    memset(g, 0, sizeof(cfg_builder));
+    if (init_cfg_builder(g) < 0) {
+        PyMem_Free(g);
+        return NULL;
+    }
+    return g;
+}
+
+void
+_PyCfgBuilder_Free(cfg_builder *g)
+{
+    if (g == NULL) {
+        return;
+    }
     assert(cfg_builder_check(g));
     basicblock *b = g->g_block_list;
     while (b != NULL) {
@@ -337,6 +416,21 @@ _PyCfgBuilder_Fini(cfg_builder* g)
         PyObject_Free((void *)b);
         b = next;
     }
+    PyMem_Free(g);
+}
+
+int
+_PyCfgBuilder_CheckSize(cfg_builder *g)
+{
+    int nblocks = 0;
+    for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) {
+        nblocks++;
+    }
+    if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) {
+        PyErr_NoMemory();
+        return ERROR;
+    }
+    return SUCCESS;
 }
 
 int
@@ -450,7 +544,7 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) {
 
 
 static int
-normalize_jumps(_PyCfgBuilder *g)
+normalize_jumps(cfg_builder *g)
 {
     basicblock *entryblock = g->g_entryblock;
     for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
@@ -463,14 +557,6 @@ normalize_jumps(_PyCfgBuilder *g)
     return SUCCESS;
 }
 
-int
-_PyCfg_ResolveJumps(_PyCfgBuilder *g)
-{
-    RETURN_IF_ERROR(normalize_jumps(g));
-    assert(no_redundant_jumps(g));
-    return SUCCESS;
-}
-
 static int
 check_cfg(cfg_builder *g) {
     for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) {
@@ -529,9 +615,9 @@ translate_jump_labels_to_targets(basicblock *entryblock)
 }
 
 int
-_PyCfg_JumpLabelsToTargets(basicblock *entryblock)
+_PyCfg_JumpLabelsToTargets(cfg_builder *g)
 {
-    return translate_jump_labels_to_targets(entryblock);
+    return translate_jump_labels_to_targets(g->g_entryblock);
 }
 
 static int
@@ -553,10 +639,14 @@ mark_except_handlers(basicblock *entryblock) {
 }
 
 
-typedef _PyCfgExceptStack ExceptStack;
+struct _PyCfgExceptStack {
+    basicblock *handlers[CO_MAXBLOCKS+1];
+    int depth;
+};
+
 
 static basicblock *
-push_except_block(ExceptStack *stack, cfg_instr *setup) {
+push_except_block(struct _PyCfgExceptStack *stack, cfg_instr *setup) {
     assert(is_block_push(setup));
     int opcode = setup->i_opcode;
     basicblock * target = setup->i_target;
@@ -568,19 +658,19 @@ push_except_block(ExceptStack *stack, cfg_instr *setup) {
 }
 
 static basicblock *
-pop_except_block(ExceptStack *stack) {
+pop_except_block(struct _PyCfgExceptStack *stack) {
     assert(stack->depth > 0);
     return stack->handlers[--stack->depth];
 }
 
 static basicblock *
-except_stack_top(ExceptStack *stack) {
+except_stack_top(struct _PyCfgExceptStack *stack) {
     return stack->handlers[stack->depth];
 }
 
-static ExceptStack *
+static struct _PyCfgExceptStack *
 make_except_stack(void) {
-    ExceptStack *new = PyMem_Malloc(sizeof(ExceptStack));
+    struct _PyCfgExceptStack *new = PyMem_Malloc(sizeof(struct _PyCfgExceptStack));
     if (new == NULL) {
         PyErr_NoMemory();
         return NULL;
@@ -590,14 +680,14 @@ make_except_stack(void) {
     return new;
 }
 
-static ExceptStack *
+static struct _PyCfgExceptStack *
-copy_except_stack(ExceptStack *stack) {
+copy_except_stack(struct _PyCfgExceptStack *stack) {
-    ExceptStack *copy = PyMem_Malloc(sizeof(ExceptStack));
+    struct _PyCfgExceptStack *copy = PyMem_Malloc(sizeof(struct _PyCfgExceptStack));
     if (copy == NULL) {
         PyErr_NoMemory();
         return NULL;
     }
-    memcpy(copy, stack, sizeof(ExceptStack));
+    memcpy(copy, stack, sizeof(struct _PyCfgExceptStack));
     return copy;
 }
 
@@ -633,8 +723,8 @@ stackdepth_push(basicblock ***sp, basicblock *b, int depth)
 /* Find the flow path that needs the largest stack.  We assume that
  * cycles in the flow graph have no net effect on the stack depth.
  */
-int
+static int
-_PyCfg_Stackdepth(cfg_builder *g)
+calculate_stackdepth(cfg_builder *g)
 {
     basicblock *entryblock = g->g_entryblock;
     for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
@@ -723,7 +813,7 @@ label_exception_targets(basicblock *entryblock) {
     if (todo_stack == NULL) {
         return ERROR;
     }
-    ExceptStack *except_stack = make_except_stack();
+    struct _PyCfgExceptStack *except_stack = make_except_stack();
     if (except_stack == NULL) {
         PyMem_Free(todo_stack);
         PyErr_NoMemory();
@@ -747,7 +837,7 @@ label_exception_targets(basicblock *entryblock) {
             cfg_instr *instr = &b->b_instr[i];
             if (is_block_push(instr)) {
                 if (!instr->i_target->b_visited) {
-                    ExceptStack *copy = copy_except_stack(except_stack);
+                    struct _PyCfgExceptStack *copy = copy_except_stack(except_stack);
                     if (copy == NULL) {
                         goto error;
                     }
@@ -766,7 +856,7 @@ label_exception_targets(basicblock *entryblock) {
                 assert(i == b->b_iused -1);
                 if (!instr->i_target->b_visited) {
                     if (BB_HAS_FALLTHROUGH(b)) {
-                        ExceptStack *copy = copy_except_stack(except_stack);
+                        struct _PyCfgExceptStack *copy = copy_except_stack(except_stack);
                         if (copy == NULL) {
                             goto error;
                         }
@@ -2103,8 +2193,8 @@ push_cold_blocks_to_end(cfg_builder *g) {
     return SUCCESS;
 }
 
-void
+static void
-_PyCfg_ConvertPseudoOps(basicblock *entryblock)
+convert_pseudo_ops(basicblock *entryblock)
 {
     for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
         for (int i = 0; i < b->b_iused; i++) {
@@ -2293,3 +2383,270 @@ _PyCfg_OptimizeCodeUnit(cfg_builder *g, PyObject *consts, PyObject *const_cache,
     RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno));
     return SUCCESS;
 }
+
+static int *
+build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd)
+{
+    int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames);
+    int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars);
+    int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars);
+
+    int noffsets = ncellvars + nfreevars;
+    int *fixed = PyMem_New(int, noffsets);
+    if (fixed == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    for (int i = 0; i < noffsets; i++) {
+        fixed[i] = nlocals + i;
+    }
+
+    PyObject *varname, *cellindex;
+    Py_ssize_t pos = 0;
+    while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) {
+        PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname);
+        if (varindex != NULL) {
+            assert(PyLong_AS_LONG(cellindex) < INT_MAX);
+            assert(PyLong_AS_LONG(varindex) < INT_MAX);
+            int oldindex = (int)PyLong_AS_LONG(cellindex);
+            int argoffset = (int)PyLong_AS_LONG(varindex);
+            fixed[oldindex] = argoffset;
+        }
+    }
+
+    return fixed;
+}
+
+#define IS_GENERATOR(CF) \
+    ((CF) & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR))
+
+static int
+insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock,
+                           int *fixed, int nfreevars, int code_flags)
+{
+    assert(umd->u_firstlineno > 0);
+
+    /* Add the generator prefix instructions. */
+    if (IS_GENERATOR(code_flags)) {
+        /* Note that RETURN_GENERATOR + POP_TOP have a net stack effect
+         * of 0. This is because RETURN_GENERATOR pushes an element
+         * with _PyFrame_StackPush before switching stacks.
+         */
+        cfg_instr make_gen = {
+            .i_opcode = RETURN_GENERATOR,
+            .i_oparg = 0,
+            .i_loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1),
+            .i_target = NULL,
+        };
+        RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &make_gen));
+        cfg_instr pop_top = {
+            .i_opcode = POP_TOP,
+            .i_oparg = 0,
+            .i_loc = NO_LOCATION,
+            .i_target = NULL,
+        };
+        RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 1, &pop_top));
+    }
+
+    /* Set up cells for any variable that escapes, to be put in a closure. */
+    const int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars);
+    if (ncellvars) {
+        // umd->u_cellvars has the cells out of order so we sort them
+        // before adding the MAKE_CELL instructions.  Note that we
+        // adjust for arg cells, which come first.
+        const int nvars = ncellvars + (int)PyDict_GET_SIZE(umd->u_varnames);
+        int *sorted = PyMem_RawCalloc(nvars, sizeof(int));
+        if (sorted == NULL) {
+            PyErr_NoMemory();
+            return ERROR;
+        }
+        for (int i = 0; i < ncellvars; i++) {
+            sorted[fixed[i]] = i + 1;
+        }
+        for (int i = 0, ncellsused = 0; ncellsused < ncellvars; i++) {
+            int oldindex = sorted[i] - 1;
+            if (oldindex == -1) {
+                continue;
+            }
+            cfg_instr make_cell = {
+                .i_opcode = MAKE_CELL,
+                // This will get fixed in offset_derefs().
+                .i_oparg = oldindex,
+                .i_loc = NO_LOCATION,
+                .i_target = NULL,
+            };
+            if (basicblock_insert_instruction(entryblock, ncellsused, &make_cell) < 0) {
+                PyMem_RawFree(sorted);
+                return ERROR;
+            }
+            ncellsused += 1;
+        }
+        PyMem_RawFree(sorted);
+    }
+
+    if (nfreevars) {
+        cfg_instr copy_frees = {
+            .i_opcode = COPY_FREE_VARS,
+            .i_oparg = nfreevars,
+            .i_loc = NO_LOCATION,
+            .i_target = NULL,
+        };
+        RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &copy_frees));
+    }
+
+    return SUCCESS;
+}
+
+static int
+fix_cell_offsets(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixedmap)
+{
+    int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames);
+    int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars);
+    int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars);
+    int noffsets = ncellvars + nfreevars;
+
+    // First deal with duplicates (arg cells).
+    int numdropped = 0;
+    for (int i = 0; i < noffsets ; i++) {
+        if (fixedmap[i] == i + nlocals) {
+            fixedmap[i] -= numdropped;
+        }
+        else {
+            // It was a duplicate (cell/arg).
+            numdropped += 1;
+        }
+    }
+
+    // Then update offsets, either relative to locals or by cell2arg.
+    for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
+        for (int i = 0; i < b->b_iused; i++) {
+            cfg_instr *inst = &b->b_instr[i];
+            // This is called before extended args are generated.
+            assert(inst->i_opcode != EXTENDED_ARG);
+            int oldoffset = inst->i_oparg;
+            switch(inst->i_opcode) {
+                case MAKE_CELL:
+                case LOAD_CLOSURE:
+                case LOAD_DEREF:
+                case STORE_DEREF:
+                case DELETE_DEREF:
+                case LOAD_FROM_DICT_OR_DEREF:
+                    assert(oldoffset >= 0);
+                    assert(oldoffset < noffsets);
+                    assert(fixedmap[oldoffset] >= 0);
+                    inst->i_oparg = fixedmap[oldoffset];
+            }
+        }
+    }
+
+    return numdropped;
+}
+
+static int
+prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_flags)
+{
+    assert(PyDict_GET_SIZE(umd->u_varnames) < INT_MAX);
+    assert(PyDict_GET_SIZE(umd->u_cellvars) < INT_MAX);
+    assert(PyDict_GET_SIZE(umd->u_freevars) < INT_MAX);
+    int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames);
+    int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars);
+    int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars);
+    assert(INT_MAX - nlocals - ncellvars > 0);
+    assert(INT_MAX - nlocals - ncellvars - nfreevars > 0);
+    int nlocalsplus = nlocals + ncellvars + nfreevars;
+    int* cellfixedoffsets = build_cellfixedoffsets(umd);
+    if (cellfixedoffsets == NULL) {
+        return ERROR;
+    }
+
+    // This must be called before fix_cell_offsets().
+    if (insert_prefix_instructions(umd, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) {
+        PyMem_Free(cellfixedoffsets);
+        return ERROR;
+    }
+
+    int numdropped = fix_cell_offsets(umd, g->g_entryblock, cellfixedoffsets);
+    PyMem_Free(cellfixedoffsets);  // At this point we're done with it.
+    cellfixedoffsets = NULL;
+    if (numdropped < 0) {
+        return ERROR;
+    }
+
+    nlocalsplus -= numdropped;
+    return nlocalsplus;
+}
+
+int
+_PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq)
+{
+    int lbl = 0;
+    for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) {
+        b->b_label = (jump_target_label){lbl};
+        lbl += b->b_iused;
+    }
+    for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) {
+        RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(seq, b->b_label.id));
+        for (int i = 0; i < b->b_iused; i++) {
+            cfg_instr *instr = &b->b_instr[i];
+            if (OPCODE_HAS_JUMP(instr->i_opcode)) {
+                instr->i_oparg = instr->i_target->b_label.id;
+            }
+            RETURN_IF_ERROR(
+                _PyCompile_InstructionSequence_Addop(
+                    seq, instr->i_opcode, instr->i_oparg, instr->i_loc));
+
+            _PyCompile_ExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info;
+            if (instr->i_except != NULL) {
+                hi->h_label = instr->i_except->b_label.id;
+                hi->h_startdepth = instr->i_except->b_startdepth;
+                hi->h_preserve_lasti = instr->i_except->b_preserve_lasti;
+            }
+            else {
+                hi->h_label = -1;
+            }
+        }
+    }
+    return SUCCESS;
+}
+
+
+int
+_PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g,
+                                     _PyCompile_CodeUnitMetadata *umd, int code_flags,
+                                     int *stackdepth, int *nlocalsplus,
+                                     _PyCompile_InstructionSequence *seq)
+{
+    *stackdepth = calculate_stackdepth(g);
+    if (*stackdepth < 0) {
+        return ERROR;
+    }
+
+    /* prepare_localsplus adds instructions for generators that push
+     * and pop an item on the stack. This assertion makes sure there
+     * is space on the stack for that.
+     * It should always be true, because a generator must have at
+     * least one expression or call to INTRINSIC_STOPITERATION_ERROR,
+     * which requires stackspace.
+     */
+    assert(!(IS_GENERATOR(code_flags) && *stackdepth == 0));
+
+    *nlocalsplus = prepare_localsplus(umd, g, code_flags);
+    if (*nlocalsplus < 0) {
+        return ERROR;
+    }
+
+    convert_pseudo_ops(g->g_entryblock);
+
+    /* Order of basic blocks must have been determined by now */
+
+    RETURN_IF_ERROR(normalize_jumps(g));
+    assert(no_redundant_jumps(g));
+
+    /* Can't modify the bytecode after computing jump offsets. */
+    if (_PyCfg_ToInstructionSequence(g, seq) < 0) {
+        return ERROR;
+    }
+
+    return SUCCESS;
+}
+