gh-104169: Refactor tokenizer into lexer and wrappers (#110684)

* The lexer, which include the actual lexeme producing logic, goes into
  the `lexer` directory.
* The wrappers, one wrapper per input mode (file, string, utf-8, and
  readline), go into the `tokenizer` directory and include logic for
  creating a lexer instance and managing the buffer for different modes.
---------

Co-authored-by: Pablo Galindo <pablogsal@gmail.com>
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>

Parser/tokenizer/file_tokenizer.c

@@ -0,0 +1,470 @@
+#ifdef HAVE_UNISTD_H
+#  include <unistd.h>             // read()
+#endif
+
+#include "Python.h"
+#include "pycore_call.h"
+#include "pycore_import.h"
+#include "pycore_fileutils.h"
+#include "errcode.h"
+
+#include "helpers.h"
+#include "../lexer/state.h"
+#include "../lexer/lexer.h"
+#include "../lexer/buffer.h"
+
+static int
+tok_concatenate_interactive_new_line(struct tok_state *tok, const char *line) {
+    assert(tok->fp_interactive);
+
+    if (!line) {
+        return 0;
+    }
+
+    Py_ssize_t current_size = tok->interactive_src_end - tok->interactive_src_start;
+    Py_ssize_t line_size = strlen(line);
+    char last_char = line[line_size > 0 ? line_size - 1 : line_size];
+    if (last_char != '\n') {
+        line_size += 1;
+    }
+    char* new_str = tok->interactive_src_start;
+
+    new_str = PyMem_Realloc(new_str, current_size + line_size + 1);
+    if (!new_str) {
+        if (tok->interactive_src_start) {
+            PyMem_Free(tok->interactive_src_start);
+        }
+        tok->interactive_src_start = NULL;
+        tok->interactive_src_end = NULL;
+        tok->done = E_NOMEM;
+        return -1;
+    }
+    strcpy(new_str + current_size, line);
+    tok->implicit_newline = 0;
+    if (last_char != '\n') {
+        /* Last line does not end in \n, fake one */
+        new_str[current_size + line_size - 1] = '\n';
+        new_str[current_size + line_size] = '\0';
+        tok->implicit_newline = 1;
+    }
+    tok->interactive_src_start = new_str;
+    tok->interactive_src_end = new_str + current_size + line_size;
+    return 0;
+}
+
+static int
+tok_readline_raw(struct tok_state *tok)
+{
+    do {
+        if (!_PyLexer_tok_reserve_buf(tok, BUFSIZ)) {
+            return 0;
+        }
+        int n_chars = (int)(tok->end - tok->inp);
+        size_t line_size = 0;
+        char *line = _Py_UniversalNewlineFgetsWithSize(tok->inp, n_chars, tok->fp, NULL, &line_size);
+        if (line == NULL) {
+            return 1;
+        }
+        if (tok->fp_interactive &&
+            tok_concatenate_interactive_new_line(tok, line) == -1) {
+            return 0;
+        }
+        tok->inp += line_size;
+        if (tok->inp == tok->buf) {
+            return 0;
+        }
+    } while (tok->inp[-1] != '\n');
+    return 1;
+}
+
+static int
+tok_readline_recode(struct tok_state *tok) {
+    PyObject *line;
+    const  char *buf;
+    Py_ssize_t buflen;
+    line = tok->decoding_buffer;
+    if (line == NULL) {
+        line = PyObject_CallNoArgs(tok->decoding_readline);
+        if (line == NULL) {
+            _PyTokenizer_error_ret(tok);
+            goto error;
+        }
+    }
+    else {
+        tok->decoding_buffer = NULL;
+    }
+    buf = PyUnicode_AsUTF8AndSize(line, &buflen);
+    if (buf == NULL) {
+        _PyTokenizer_error_ret(tok);
+        goto error;
+    }
+    // Make room for the null terminator *and* potentially
+    // an extra newline character that we may need to artificially
+    // add.
+    size_t buffer_size = buflen + 2;
+    if (!_PyLexer_tok_reserve_buf(tok, buffer_size)) {
+        goto error;
+    }
+    memcpy(tok->inp, buf, buflen);
+    tok->inp += buflen;
+    *tok->inp = '\0';
+    if (tok->fp_interactive &&
+        tok_concatenate_interactive_new_line(tok, buf) == -1) {
+        goto error;
+    }
+    Py_DECREF(line);
+    return 1;
+error:
+    Py_XDECREF(line);
+    return 0;
+}
+
+/* Fetch the next byte from TOK. */
+static int fp_getc(struct tok_state *tok) {
+    return getc(tok->fp);
+}
+
+/* Unfetch the last byte back into TOK.  */
+static void fp_ungetc(int c, struct tok_state *tok) {
+    ungetc(c, tok->fp);
+}
+
+/* Set the readline function for TOK to a StreamReader's
+   readline function. The StreamReader is named ENC.
+
+   This function is called from _PyTokenizer_check_bom and _PyTokenizer_check_coding_spec.
+
+   ENC is usually identical to the future value of tok->encoding,
+   except for the (currently unsupported) case of UTF-16.
+
+   Return 1 on success, 0 on failure. */
+static int
+fp_setreadl(struct tok_state *tok, const char* enc)
+{
+    PyObject *readline, *open, *stream;
+    int fd;
+    long pos;
+
+    fd = fileno(tok->fp);
+    /* Due to buffering the file offset for fd can be different from the file
+     * position of tok->fp.  If tok->fp was opened in text mode on Windows,
+     * its file position counts CRLF as one char and can't be directly mapped
+     * to the file offset for fd.  Instead we step back one byte and read to
+     * the end of line.*/
+    pos = ftell(tok->fp);
+    if (pos == -1 ||
+        lseek(fd, (off_t)(pos > 0 ? pos - 1 : pos), SEEK_SET) == (off_t)-1) {
+        PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
+        return 0;
+    }
+
+    open = _PyImport_GetModuleAttrString("io", "open");
+    if (open == NULL) {
+        return 0;
+    }
+    stream = PyObject_CallFunction(open, "isisOOO",
+                    fd, "r", -1, enc, Py_None, Py_None, Py_False);
+    Py_DECREF(open);
+    if (stream == NULL) {
+        return 0;
+    }
+
+    readline = PyObject_GetAttr(stream, &_Py_ID(readline));
+    Py_DECREF(stream);
+    if (readline == NULL) {
+        return 0;
+    }
+    Py_XSETREF(tok->decoding_readline, readline);
+
+    if (pos > 0) {
+        PyObject *bufobj = _PyObject_CallNoArgs(readline);
+        if (bufobj == NULL) {
+            return 0;
+        }
+        Py_DECREF(bufobj);
+    }
+
+    return 1;
+}
+
+static int
+tok_underflow_interactive(struct tok_state *tok) {
+    if (tok->interactive_underflow == IUNDERFLOW_STOP) {
+        tok->done = E_INTERACT_STOP;
+        return 1;
+    }
+    char *newtok = PyOS_Readline(tok->fp ? tok->fp : stdin, stdout, tok->prompt);
+    if (newtok != NULL) {
+        char *translated = _PyTokenizer_translate_newlines(newtok, 0, 0, tok);
+        PyMem_Free(newtok);
+        if (translated == NULL) {
+            return 0;
+        }
+        newtok = translated;
+    }
+    if (tok->encoding && newtok && *newtok) {
+        /* Recode to UTF-8 */
+        Py_ssize_t buflen;
+        const char* buf;
+        PyObject *u = _PyTokenizer_translate_into_utf8(newtok, tok->encoding);
+        PyMem_Free(newtok);
+        if (u == NULL) {
+            tok->done = E_DECODE;
+            return 0;
+        }
+        buflen = PyBytes_GET_SIZE(u);
+        buf = PyBytes_AS_STRING(u);
+        newtok = PyMem_Malloc(buflen+1);
+        if (newtok == NULL) {
+            Py_DECREF(u);
+            tok->done = E_NOMEM;
+            return 0;
+        }
+        strcpy(newtok, buf);
+        Py_DECREF(u);
+    }
+    if (tok->fp_interactive &&
+        tok_concatenate_interactive_new_line(tok, newtok) == -1) {
+        PyMem_Free(newtok);
+        return 0;
+    }
+    if (tok->nextprompt != NULL) {
+        tok->prompt = tok->nextprompt;
+    }
+    if (newtok == NULL) {
+        tok->done = E_INTR;
+    }
+    else if (*newtok == '\0') {
+        PyMem_Free(newtok);
+        tok->done = E_EOF;
+    }
+    else if (tok->start != NULL) {
+        Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;
+        _PyLexer_remember_fstring_buffers(tok);
+        size_t size = strlen(newtok);
+        ADVANCE_LINENO();
+        if (!_PyLexer_tok_reserve_buf(tok, size + 1)) {
+            PyMem_Free(tok->buf);
+            tok->buf = NULL;
+            PyMem_Free(newtok);
+            return 0;
+        }
+        memcpy(tok->cur, newtok, size + 1);
+        PyMem_Free(newtok);
+        tok->inp += size;
+        tok->multi_line_start = tok->buf + cur_multi_line_start;
+        _PyLexer_restore_fstring_buffers(tok);
+    }
+    else {
+        _PyLexer_remember_fstring_buffers(tok);
+        ADVANCE_LINENO();
+        PyMem_Free(tok->buf);
+        tok->buf = newtok;
+        tok->cur = tok->buf;
+        tok->line_start = tok->buf;
+        tok->inp = strchr(tok->buf, '\0');
+        tok->end = tok->inp + 1;
+        _PyLexer_restore_fstring_buffers(tok);
+    }
+    if (tok->done != E_OK) {
+        if (tok->prompt != NULL) {
+            PySys_WriteStderr("\n");
+        }
+        return 0;
+    }
+
+    if (tok->tok_mode_stack_index && !_PyLexer_update_fstring_expr(tok, 0)) {
+        return 0;
+    }
+    return 1;
+}
+
+static int
+tok_underflow_file(struct tok_state *tok) {
+    if (tok->start == NULL && !INSIDE_FSTRING(tok)) {
+        tok->cur = tok->inp = tok->buf;
+    }
+    if (tok->decoding_state == STATE_INIT) {
+        /* We have not yet determined the encoding.
+           If an encoding is found, use the file-pointer
+           reader functions from now on. */
+        if (!_PyTokenizer_check_bom(fp_getc, fp_ungetc, fp_setreadl, tok)) {
+            _PyTokenizer_error_ret(tok);
+            return 0;
+        }
+        assert(tok->decoding_state != STATE_INIT);
+    }
+    /* Read until '\n' or EOF */
+    if (tok->decoding_readline != NULL) {
+        /* We already have a codec associated with this input. */
+        if (!tok_readline_recode(tok)) {
+            return 0;
+        }
+    }
+    else {
+        /* We want a 'raw' read. */
+        if (!tok_readline_raw(tok)) {
+            return 0;
+        }
+    }
+    if (tok->inp == tok->cur) {
+        tok->done = E_EOF;
+        return 0;
+    }
+    tok->implicit_newline = 0;
+    if (tok->inp[-1] != '\n') {
+        assert(tok->inp + 1 < tok->end);
+        /* Last line does not end in \n, fake one */
+        *tok->inp++ = '\n';
+        *tok->inp = '\0';
+        tok->implicit_newline = 1;
+    }
+
+    if (tok->tok_mode_stack_index && !_PyLexer_update_fstring_expr(tok, 0)) {
+        return 0;
+    }
+
+    ADVANCE_LINENO();
+    if (tok->decoding_state != STATE_NORMAL) {
+        if (tok->lineno > 2) {
+            tok->decoding_state = STATE_NORMAL;
+        }
+        else if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur),
+                                    tok, fp_setreadl))
+        {
+            return 0;
+        }
+    }
+    /* The default encoding is UTF-8, so make sure we don't have any
+       non-UTF-8 sequences in it. */
+    if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) {
+        _PyTokenizer_error_ret(tok);
+        return 0;
+    }
+    assert(tok->done == E_OK);
+    return tok->done == E_OK;
+}
+
+/* Set up tokenizer for file */
+struct tok_state *
+_PyTokenizer_FromFile(FILE *fp, const char* enc,
+                      const char *ps1, const char *ps2)
+{
+    struct tok_state *tok = _PyTokenizer_tok_new();
+    if (tok == NULL)
+        return NULL;
+    if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
+        _PyTokenizer_Free(tok);
+        return NULL;
+    }
+    tok->cur = tok->inp = tok->buf;
+    tok->end = tok->buf + BUFSIZ;
+    tok->fp = fp;
+    tok->prompt = ps1;
+    tok->nextprompt = ps2;
+    if (ps1 || ps2) {
+        tok->underflow = &tok_underflow_interactive;
+    } else {
+        tok->underflow = &tok_underflow_file;
+    }
+    if (enc != NULL) {
+        /* Must copy encoding declaration since it
+           gets copied into the parse tree. */
+        tok->encoding = _PyTokenizer_new_string(enc, strlen(enc), tok);
+        if (!tok->encoding) {
+            _PyTokenizer_Free(tok);
+            return NULL;
+        }
+        tok->decoding_state = STATE_NORMAL;
+    }
+    return tok;
+}
+
+#if defined(__wasi__) || (defined(__EMSCRIPTEN__) && (__EMSCRIPTEN_major__ >= 3))
+// fdopen() with borrowed fd. WASI does not provide dup() and Emscripten's
+// dup() emulation with open() is slow.
+typedef union {
+    void *cookie;
+    int fd;
+} borrowed;
+
+static ssize_t
+borrow_read(void *cookie, char *buf, size_t size)
+{
+    borrowed b = {.cookie = cookie};
+    return read(b.fd, (void *)buf, size);
+}
+
+static FILE *
+fdopen_borrow(int fd) {
+    // supports only reading. seek fails. close and write are no-ops.
+    cookie_io_functions_t io_cb = {borrow_read, NULL, NULL, NULL};
+    borrowed b = {.fd = fd};
+    return fopencookie(b.cookie, "r", io_cb);
+}
+#else
+static FILE *
+fdopen_borrow(int fd) {
+    fd = _Py_dup(fd);
+    if (fd < 0) {
+        return NULL;
+    }
+    return fdopen(fd, "r");
+}
+#endif
+
+/* Get the encoding of a Python file. Check for the coding cookie and check if
+   the file starts with a BOM.
+
+   _PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
+   encoding in the first or second line of the file (in which case the encoding
+   should be assumed to be UTF-8).
+
+   The char* returned is malloc'ed via PyMem_Malloc() and thus must be freed
+   by the caller. */
+char *
+_PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
+{
+    struct tok_state *tok;
+    FILE *fp;
+    char *encoding = NULL;
+
+    fp = fdopen_borrow(fd);
+    if (fp == NULL) {
+        return NULL;
+    }
+    tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL);
+    if (tok == NULL) {
+        fclose(fp);
+        return NULL;
+    }
+    if (filename != NULL) {
+        tok->filename = Py_NewRef(filename);
+    }
+    else {
+        tok->filename = PyUnicode_FromString("<string>");
+        if (tok->filename == NULL) {
+            fclose(fp);
+            _PyTokenizer_Free(tok);
+            return encoding;
+        }
+    }
+    struct token token;
+    // We don't want to report warnings here because it could cause infinite recursion
+    // if fetching the encoding shows a warning.
+    tok->report_warnings = 0;
+    while (tok->lineno < 2 && tok->done == E_OK) {
+        _PyToken_Init(&token);
+        _PyTokenizer_Get(tok, &token);
+        _PyToken_Free(&token);
+    }
+    fclose(fp);
+    if (tok->encoding) {
+        encoding = (char *)PyMem_Malloc(strlen(tok->encoding) + 1);
+        if (encoding) {
+            strcpy(encoding, tok->encoding);
+        }
+    }
+    _PyTokenizer_Free(tok);
+    return encoding;
+}