[3.12] gh-105718: Fix buffer allocation in tokenizer with readline (GH-105728) (#105729)

Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>

Lib/test/test_tokenize.py

@@ -2239,6 +2239,16 @@ def"', """\
     FSTRING_START \'f"\'          (1, 0) (1, 2)
     FSTRING_MIDDLE 'abc\\\\\\ndef'  (1, 2) (2, 3)
     FSTRING_END '"'           (2, 3) (2, 4)
+    """)
+
+        self.check_tokenize('''\
+f"{
+a}"''', """\
+    FSTRING_START 'f"'          (1, 0) (1, 2)
+    LBRACE     '{'           (1, 2) (1, 3)
+    NAME       'a'           (2, 0) (2, 1)
+    RBRACE     '}'           (2, 1) (2, 2)
+    FSTRING_END '"'           (2, 2) (2, 3)
     """)
 
         self.check_tokenize(r'Rf"abc\

Parser/tokenizer.c

@@ -1106,11 +1106,7 @@ tok_readline_string(struct tok_state* tok) {
     tok->inp += buflen;
     *tok->inp = '\0';
 
-    if (tok->start == NULL) {
-        tok->buf = tok->cur;
-    }
     tok->line_start = tok->cur;
-
     Py_DECREF(line);
     return 1;
 error:

Parser/tokenizer.h

@@ -68,7 +68,7 @@ typedef struct _tokenizer_mode {
 struct tok_state {
     /* Input state; buf <= cur <= inp <= end */
     /* NB an entire line is held in the buffer */
-    char *buf;          /* Input buffer, or NULL; malloc'ed if fp != NULL */
+    char *buf;          /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */
     char *cur;          /* Next character in buffer */
     char *inp;          /* End of data in buffer */
     int fp_interactive; /* If the file descriptor is interactive */