mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 17:08:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

gh-132449: Improve syntax error messages for keywords with typos (#132450)

Browse source 
Signed-off-by: Pablo Galindo <pablogsal@gmail.com>
Co-authored-by: Łukasz Langa <lukasz@langa.pl>
This commit is contained in:
Pablo Galindo Salgado 2025-04-22 10:01:55 +01:00 • committed by GitHub
parent 3cfab449ab
commit bf3a0a1c0f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 818 additions and 450 deletions
Download patch file Download diff file Expand all files Collapse all files

17
Parser/action_helpers.c
View file

@ -1711,3 +1711,20 @@ _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * na
}
return _PyAST_ImportFrom(module, names, level, lineno, col_offset, end_lineno, end_col_offset, arena);
}
asdl_stmt_seq*
_PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts) {
if (!p->call_invalid_rules) {
return stmts;
}
Py_ssize_t len = asdl_seq_LEN(stmts);
if (len == 0) {
return stmts;
}
stmt_ty last_stmt = asdl_seq_GET(stmts, len - 1);
p->last_stmt_location.lineno = last_stmt->lineno;
p->last_stmt_location.col_offset = last_stmt->col_offset;
p->last_stmt_location.end_lineno = last_stmt->end_lineno;
p->last_stmt_location.end_col_offset = last_stmt->end_col_offset;
return stmts;
}

907
Parser/parser.c generated
View file

@ -90,428 +90,429 @@ static char *soft_keywords[] = {
#define func_type_type 1003
#define statements_type 1004
#define statement_type 1005
#define statement_newline_type 1006
#define simple_stmts_type 1007
#define simple_stmt_type 1008
#define compound_stmt_type 1009
#define assignment_type 1010
#define annotated_rhs_type 1011
#define augassign_type 1012
#define return_stmt_type 1013
#define raise_stmt_type 1014
#define pass_stmt_type 1015
#define break_stmt_type 1016
#define continue_stmt_type 1017
#define global_stmt_type 1018
#define nonlocal_stmt_type 1019
#define del_stmt_type 1020
#define yield_stmt_type 1021
#define assert_stmt_type 1022
#define import_stmt_type 1023
#define import_name_type 1024
#define import_from_type 1025
#define import_from_targets_type 1026
#define import_from_as_names_type 1027
#define import_from_as_name_type 1028
#define dotted_as_names_type 1029
#define dotted_as_name_type 1030
#define dotted_name_type 1031 // Left-recursive
#define block_type 1032
#define decorators_type 1033
#define class_def_type 1034
#define class_def_raw_type 1035
#define function_def_type 1036
#define function_def_raw_type 1037
#define params_type 1038
#define parameters_type 1039
#define slash_no_default_type 1040
#define slash_with_default_type 1041
#define star_etc_type 1042
#define kwds_type 1043
#define param_no_default_type 1044
#define param_no_default_star_annotation_type 1045
#define param_with_default_type 1046
#define param_maybe_default_type 1047
#define param_type 1048
#define param_star_annotation_type 1049
#define annotation_type 1050
#define star_annotation_type 1051
#define default_type 1052
#define if_stmt_type 1053
#define elif_stmt_type 1054
#define else_block_type 1055
#define while_stmt_type 1056
#define for_stmt_type 1057
#define with_stmt_type 1058
#define with_item_type 1059
#define try_stmt_type 1060
#define except_block_type 1061
#define except_star_block_type 1062
#define finally_block_type 1063
#define match_stmt_type 1064
#define subject_expr_type 1065
#define case_block_type 1066
#define guard_type 1067
#define patterns_type 1068
#define pattern_type 1069
#define as_pattern_type 1070
#define or_pattern_type 1071
#define closed_pattern_type 1072
#define literal_pattern_type 1073
#define literal_expr_type 1074
#define complex_number_type 1075
#define signed_number_type 1076
#define signed_real_number_type 1077
#define real_number_type 1078
#define imaginary_number_type 1079
#define capture_pattern_type 1080
#define pattern_capture_target_type 1081
#define wildcard_pattern_type 1082
#define value_pattern_type 1083
#define attr_type 1084 // Left-recursive
#define name_or_attr_type 1085 // Left-recursive
#define group_pattern_type 1086
#define sequence_pattern_type 1087
#define open_sequence_pattern_type 1088
#define maybe_sequence_pattern_type 1089
#define maybe_star_pattern_type 1090
#define star_pattern_type 1091
#define mapping_pattern_type 1092
#define items_pattern_type 1093
#define key_value_pattern_type 1094
#define double_star_pattern_type 1095
#define class_pattern_type 1096
#define positional_patterns_type 1097
#define keyword_patterns_type 1098
#define keyword_pattern_type 1099
#define type_alias_type 1100
#define type_params_type 1101
#define type_param_seq_type 1102
#define type_param_type 1103
#define type_param_bound_type 1104
#define type_param_default_type 1105
#define type_param_starred_default_type 1106
#define expressions_type 1107
#define expression_type 1108
#define yield_expr_type 1109
#define star_expressions_type 1110
#define star_expression_type 1111
#define star_named_expressions_type 1112
#define star_named_expression_type 1113
#define assignment_expression_type 1114
#define named_expression_type 1115
#define disjunction_type 1116
#define conjunction_type 1117
#define inversion_type 1118
#define comparison_type 1119
#define compare_op_bitwise_or_pair_type 1120
#define eq_bitwise_or_type 1121
#define noteq_bitwise_or_type 1122
#define lte_bitwise_or_type 1123
#define lt_bitwise_or_type 1124
#define gte_bitwise_or_type 1125
#define gt_bitwise_or_type 1126
#define notin_bitwise_or_type 1127
#define in_bitwise_or_type 1128
#define isnot_bitwise_or_type 1129
#define is_bitwise_or_type 1130
#define bitwise_or_type 1131 // Left-recursive
#define bitwise_xor_type 1132 // Left-recursive
#define bitwise_and_type 1133 // Left-recursive
#define shift_expr_type 1134 // Left-recursive
#define sum_type 1135 // Left-recursive
#define term_type 1136 // Left-recursive
#define factor_type 1137
#define power_type 1138
#define await_primary_type 1139
#define primary_type 1140 // Left-recursive
#define slices_type 1141
#define slice_type 1142
#define atom_type 1143
#define group_type 1144
#define lambdef_type 1145
#define lambda_params_type 1146
#define lambda_parameters_type 1147
#define lambda_slash_no_default_type 1148
#define lambda_slash_with_default_type 1149
#define lambda_star_etc_type 1150
#define lambda_kwds_type 1151
#define lambda_param_no_default_type 1152
#define lambda_param_with_default_type 1153
#define lambda_param_maybe_default_type 1154
#define lambda_param_type 1155
#define fstring_middle_type 1156
#define fstring_replacement_field_type 1157
#define fstring_conversion_type 1158
#define fstring_full_format_spec_type 1159
#define fstring_format_spec_type 1160
#define fstring_type 1161
#define string_type 1162
#define strings_type 1163
#define list_type 1164
#define tuple_type 1165
#define set_type 1166
#define dict_type 1167
#define double_starred_kvpairs_type 1168
#define double_starred_kvpair_type 1169
#define kvpair_type 1170
#define for_if_clauses_type 1171
#define for_if_clause_type 1172
#define listcomp_type 1173
#define setcomp_type 1174
#define genexp_type 1175
#define dictcomp_type 1176
#define arguments_type 1177
#define args_type 1178
#define kwargs_type 1179
#define starred_expression_type 1180
#define kwarg_or_starred_type 1181
#define kwarg_or_double_starred_type 1182
#define star_targets_type 1183
#define star_targets_list_seq_type 1184
#define star_targets_tuple_seq_type 1185
#define star_target_type 1186
#define target_with_star_atom_type 1187
#define star_atom_type 1188
#define single_target_type 1189
#define single_subscript_attribute_target_type 1190
#define t_primary_type 1191 // Left-recursive
#define t_lookahead_type 1192
#define del_targets_type 1193
#define del_target_type 1194
#define del_t_atom_type 1195
#define type_expressions_type 1196
#define func_type_comment_type 1197
#define invalid_arguments_type 1198
#define invalid_kwarg_type 1199
#define expression_without_invalid_type 1200
#define invalid_legacy_expression_type 1201
#define invalid_type_param_type 1202
#define invalid_expression_type 1203
#define invalid_named_expression_type 1204
#define invalid_assignment_type 1205
#define invalid_ann_assign_target_type 1206
#define invalid_del_stmt_type 1207
#define invalid_block_type 1208
#define invalid_comprehension_type 1209
#define invalid_dict_comprehension_type 1210
#define invalid_parameters_type 1211
#define invalid_default_type 1212
#define invalid_star_etc_type 1213
#define invalid_kwds_type 1214
#define invalid_parameters_helper_type 1215
#define invalid_lambda_parameters_type 1216
#define invalid_lambda_parameters_helper_type 1217
#define invalid_lambda_star_etc_type 1218
#define invalid_lambda_kwds_type 1219
#define invalid_double_type_comments_type 1220
#define invalid_with_item_type 1221
#define invalid_for_if_clause_type 1222
#define invalid_for_target_type 1223
#define invalid_group_type 1224
#define invalid_import_type 1225
#define invalid_import_from_targets_type 1226
#define invalid_with_stmt_type 1227
#define invalid_with_stmt_indent_type 1228
#define invalid_try_stmt_type 1229
#define invalid_except_stmt_type 1230
#define invalid_except_star_stmt_type 1231
#define invalid_finally_stmt_type 1232
#define invalid_except_stmt_indent_type 1233
#define invalid_except_star_stmt_indent_type 1234
#define invalid_match_stmt_type 1235
#define invalid_case_block_type 1236
#define invalid_as_pattern_type 1237
#define invalid_class_pattern_type 1238
#define invalid_class_argument_pattern_type 1239
#define invalid_if_stmt_type 1240
#define invalid_elif_stmt_type 1241
#define invalid_else_stmt_type 1242
#define invalid_while_stmt_type 1243
#define invalid_for_stmt_type 1244
#define invalid_def_raw_type 1245
#define invalid_class_def_raw_type 1246
#define invalid_double_starred_kvpairs_type 1247
#define invalid_kvpair_type 1248
#define invalid_starred_expression_unpacking_type 1249
#define invalid_starred_expression_type 1250
#define invalid_replacement_field_type 1251
#define invalid_conversion_character_type 1252
#define invalid_arithmetic_type 1253
#define invalid_factor_type 1254
#define invalid_type_params_type 1255
#define _loop0_1_type 1256
#define _loop1_2_type 1257
#define _loop0_3_type 1258
#define _gather_4_type 1259
#define _tmp_5_type 1260
#define _tmp_6_type 1261
#define _tmp_7_type 1262
#define _tmp_8_type 1263
#define _tmp_9_type 1264
#define _tmp_10_type 1265
#define _tmp_11_type 1266
#define _loop1_12_type 1267
#define _tmp_13_type 1268
#define _loop0_14_type 1269
#define _gather_15_type 1270
#define _tmp_16_type 1271
#define _tmp_17_type 1272
#define _loop0_18_type 1273
#define _loop1_19_type 1274
#define _loop0_20_type 1275
#define _gather_21_type 1276
#define _tmp_22_type 1277
#define _loop0_23_type 1278
#define _gather_24_type 1279
#define _loop1_25_type 1280
#define _tmp_26_type 1281
#define _tmp_27_type 1282
#define _loop0_28_type 1283
#define _loop0_29_type 1284
#define _loop1_30_type 1285
#define _loop1_31_type 1286
#define _loop0_32_type 1287
#define _loop1_33_type 1288
#define _loop0_34_type 1289
#define _gather_35_type 1290
#define _tmp_36_type 1291
#define _loop1_37_type 1292
#define _loop1_38_type 1293
#define _loop1_39_type 1294
#define _loop0_40_type 1295
#define _gather_41_type 1296
#define _tmp_42_type 1297
#define _tmp_43_type 1298
#define _loop0_44_type 1299
#define _gather_45_type 1300
#define _loop0_46_type 1301
#define _gather_47_type 1302
#define _tmp_48_type 1303
#define _loop0_49_type 1304
#define _gather_50_type 1305
#define _loop0_51_type 1306
#define _gather_52_type 1307
#define _loop0_53_type 1308
#define _gather_54_type 1309
#define _loop1_55_type 1310
#define _loop1_56_type 1311
#define _loop0_57_type 1312
#define _gather_58_type 1313
#define _loop1_59_type 1314
#define _loop1_60_type 1315
#define _loop1_61_type 1316
#define _tmp_62_type 1317
#define _loop0_63_type 1318
#define _gather_64_type 1319
#define _tmp_65_type 1320
#define _tmp_66_type 1321
#define _tmp_67_type 1322
#define _tmp_68_type 1323
#define _tmp_69_type 1324
#define _tmp_70_type 1325
#define _loop0_71_type 1326
#define _loop0_72_type 1327
#define _loop1_73_type 1328
#define _loop1_74_type 1329
#define _loop0_75_type 1330
#define _loop1_76_type 1331
#define _loop0_77_type 1332
#define _loop0_78_type 1333
#define _loop1_79_type 1334
#define _tmp_80_type 1335
#define _loop0_81_type 1336
#define _gather_82_type 1337
#define _loop1_83_type 1338
#define _loop0_84_type 1339
#define _tmp_85_type 1340
#define _loop0_86_type 1341
#define _gather_87_type 1342
#define _tmp_88_type 1343
#define _loop0_89_type 1344
#define _gather_90_type 1345
#define _loop0_91_type 1346
#define _gather_92_type 1347
#define _loop0_93_type 1348
#define _loop0_94_type 1349
#define _gather_95_type 1350
#define _loop1_96_type 1351
#define _tmp_97_type 1352
#define _loop0_98_type 1353
#define _gather_99_type 1354
#define _loop0_100_type 1355
#define _gather_101_type 1356
#define _tmp_102_type 1357
#define _tmp_103_type 1358
#define _loop0_104_type 1359
#define _gather_105_type 1360
#define _tmp_106_type 1361
#define _tmp_107_type 1362
#define _tmp_108_type 1363
#define _tmp_109_type 1364
#define _tmp_110_type 1365
#define _loop1_111_type 1366
#define _tmp_112_type 1367
#define _tmp_113_type 1368
#define _tmp_114_type 1369
#define _tmp_115_type 1370
#define _tmp_116_type 1371
#define _loop0_117_type 1372
#define _loop0_118_type 1373
#define _tmp_119_type 1374
#define _tmp_120_type 1375
#define _tmp_121_type 1376
#define _tmp_122_type 1377
#define _tmp_123_type 1378
#define _tmp_124_type 1379
#define _tmp_125_type 1380
#define _tmp_126_type 1381
#define _tmp_127_type 1382
#define _loop0_128_type 1383
#define _gather_129_type 1384
#define _tmp_130_type 1385
#define _tmp_131_type 1386
#define _tmp_132_type 1387
#define _tmp_133_type 1388
#define _loop0_134_type 1389
#define _gather_135_type 1390
#define _loop0_136_type 1391
#define _gather_137_type 1392
#define _loop0_138_type 1393
#define _gather_139_type 1394
#define _tmp_140_type 1395
#define _loop0_141_type 1396
#define _tmp_142_type 1397
#define _tmp_143_type 1398
#define _tmp_144_type 1399
#define _tmp_145_type 1400
#define _tmp_146_type 1401
#define _tmp_147_type 1402
#define _tmp_148_type 1403
#define _tmp_149_type 1404
#define _tmp_150_type 1405
#define _tmp_151_type 1406
#define _tmp_152_type 1407
#define _tmp_153_type 1408
#define _tmp_154_type 1409
#define _tmp_155_type 1410
#define _tmp_156_type 1411
#define _tmp_157_type 1412
#define _tmp_158_type 1413
#define _tmp_159_type 1414
#define _tmp_160_type 1415
#define _tmp_161_type 1416
#define _tmp_162_type 1417
#define _tmp_163_type 1418
#define _tmp_164_type 1419
#define _tmp_165_type 1420
#define _tmp_166_type 1421
#define _tmp_167_type 1422
#define _loop0_168_type 1423
#define _tmp_169_type 1424
#define _tmp_170_type 1425
#define _tmp_171_type 1426
#define _tmp_172_type 1427
#define single_compound_stmt_type 1006
#define statement_newline_type 1007
#define simple_stmts_type 1008
#define simple_stmt_type 1009
#define compound_stmt_type 1010
#define assignment_type 1011
#define annotated_rhs_type 1012
#define augassign_type 1013
#define return_stmt_type 1014
#define raise_stmt_type 1015
#define pass_stmt_type 1016
#define break_stmt_type 1017
#define continue_stmt_type 1018
#define global_stmt_type 1019
#define nonlocal_stmt_type 1020
#define del_stmt_type 1021
#define yield_stmt_type 1022
#define assert_stmt_type 1023
#define import_stmt_type 1024
#define import_name_type 1025
#define import_from_type 1026
#define import_from_targets_type 1027
#define import_from_as_names_type 1028
#define import_from_as_name_type 1029
#define dotted_as_names_type 1030
#define dotted_as_name_type 1031
#define dotted_name_type 1032 // Left-recursive
#define block_type 1033
#define decorators_type 1034
#define class_def_type 1035
#define class_def_raw_type 1036
#define function_def_type 1037
#define function_def_raw_type 1038
#define params_type 1039
#define parameters_type 1040
#define slash_no_default_type 1041
#define slash_with_default_type 1042
#define star_etc_type 1043
#define kwds_type 1044
#define param_no_default_type 1045
#define param_no_default_star_annotation_type 1046
#define param_with_default_type 1047
#define param_maybe_default_type 1048
#define param_type 1049
#define param_star_annotation_type 1050
#define annotation_type 1051
#define star_annotation_type 1052
#define default_type 1053
#define if_stmt_type 1054
#define elif_stmt_type 1055
#define else_block_type 1056
#define while_stmt_type 1057
#define for_stmt_type 1058
#define with_stmt_type 1059
#define with_item_type 1060
#define try_stmt_type 1061
#define except_block_type 1062
#define except_star_block_type 1063
#define finally_block_type 1064
#define match_stmt_type 1065
#define subject_expr_type 1066
#define case_block_type 1067
#define guard_type 1068
#define patterns_type 1069
#define pattern_type 1070
#define as_pattern_type 1071
#define or_pattern_type 1072
#define closed_pattern_type 1073
#define literal_pattern_type 1074
#define literal_expr_type 1075
#define complex_number_type 1076
#define signed_number_type 1077
#define signed_real_number_type 1078
#define real_number_type 1079
#define imaginary_number_type 1080
#define capture_pattern_type 1081
#define pattern_capture_target_type 1082
#define wildcard_pattern_type 1083
#define value_pattern_type 1084
#define attr_type 1085 // Left-recursive
#define name_or_attr_type 1086 // Left-recursive
#define group_pattern_type 1087
#define sequence_pattern_type 1088
#define open_sequence_pattern_type 1089
#define maybe_sequence_pattern_type 1090
#define maybe_star_pattern_type 1091
#define star_pattern_type 1092
#define mapping_pattern_type 1093
#define items_pattern_type 1094
#define key_value_pattern_type 1095
#define double_star_pattern_type 1096
#define class_pattern_type 1097
#define positional_patterns_type 1098
#define keyword_patterns_type 1099
#define keyword_pattern_type 1100
#define type_alias_type 1101
#define type_params_type 1102
#define type_param_seq_type 1103
#define type_param_type 1104
#define type_param_bound_type 1105
#define type_param_default_type 1106
#define type_param_starred_default_type 1107
#define expressions_type 1108
#define expression_type 1109
#define yield_expr_type 1110
#define star_expressions_type 1111
#define star_expression_type 1112
#define star_named_expressions_type 1113
#define star_named_expression_type 1114
#define assignment_expression_type 1115
#define named_expression_type 1116
#define disjunction_type 1117
#define conjunction_type 1118
#define inversion_type 1119
#define comparison_type 1120
#define compare_op_bitwise_or_pair_type 1121
#define eq_bitwise_or_type 1122
#define noteq_bitwise_or_type 1123
#define lte_bitwise_or_type 1124
#define lt_bitwise_or_type 1125
#define gte_bitwise_or_type 1126
#define gt_bitwise_or_type 1127
#define notin_bitwise_or_type 1128
#define in_bitwise_or_type 1129
#define isnot_bitwise_or_type 1130
#define is_bitwise_or_type 1131
#define bitwise_or_type 1132 // Left-recursive
#define bitwise_xor_type 1133 // Left-recursive
#define bitwise_and_type 1134 // Left-recursive
#define shift_expr_type 1135 // Left-recursive
#define sum_type 1136 // Left-recursive
#define term_type 1137 // Left-recursive
#define factor_type 1138
#define power_type 1139
#define await_primary_type 1140
#define primary_type 1141 // Left-recursive
#define slices_type 1142
#define slice_type 1143
#define atom_type 1144
#define group_type 1145
#define lambdef_type 1146
#define lambda_params_type 1147
#define lambda_parameters_type 1148
#define lambda_slash_no_default_type 1149
#define lambda_slash_with_default_type 1150
#define lambda_star_etc_type 1151
#define lambda_kwds_type 1152
#define lambda_param_no_default_type 1153
#define lambda_param_with_default_type 1154
#define lambda_param_maybe_default_type 1155
#define lambda_param_type 1156
#define fstring_middle_type 1157
#define fstring_replacement_field_type 1158
#define fstring_conversion_type 1159
#define fstring_full_format_spec_type 1160
#define fstring_format_spec_type 1161
#define fstring_type 1162
#define string_type 1163
#define strings_type 1164
#define list_type 1165
#define tuple_type 1166
#define set_type 1167
#define dict_type 1168
#define double_starred_kvpairs_type 1169
#define double_starred_kvpair_type 1170
#define kvpair_type 1171
#define for_if_clauses_type 1172
#define for_if_clause_type 1173
#define listcomp_type 1174
#define setcomp_type 1175
#define genexp_type 1176
#define dictcomp_type 1177
#define arguments_type 1178
#define args_type 1179
#define kwargs_type 1180
#define starred_expression_type 1181
#define kwarg_or_starred_type 1182
#define kwarg_or_double_starred_type 1183
#define star_targets_type 1184
#define star_targets_list_seq_type 1185
#define star_targets_tuple_seq_type 1186
#define star_target_type 1187
#define target_with_star_atom_type 1188
#define star_atom_type 1189
#define single_target_type 1190
#define single_subscript_attribute_target_type 1191
#define t_primary_type 1192 // Left-recursive
#define t_lookahead_type 1193
#define del_targets_type 1194
#define del_target_type 1195
#define del_t_atom_type 1196
#define type_expressions_type 1197
#define func_type_comment_type 1198
#define invalid_arguments_type 1199
#define invalid_kwarg_type 1200
#define expression_without_invalid_type 1201
#define invalid_legacy_expression_type 1202
#define invalid_type_param_type 1203
#define invalid_expression_type 1204
#define invalid_named_expression_type 1205
#define invalid_assignment_type 1206
#define invalid_ann_assign_target_type 1207
#define invalid_del_stmt_type 1208
#define invalid_block_type 1209
#define invalid_comprehension_type 1210
#define invalid_dict_comprehension_type 1211
#define invalid_parameters_type 1212
#define invalid_default_type 1213
#define invalid_star_etc_type 1214
#define invalid_kwds_type 1215
#define invalid_parameters_helper_type 1216
#define invalid_lambda_parameters_type 1217
#define invalid_lambda_parameters_helper_type 1218
#define invalid_lambda_star_etc_type 1219
#define invalid_lambda_kwds_type 1220
#define invalid_double_type_comments_type 1221
#define invalid_with_item_type 1222
#define invalid_for_if_clause_type 1223
#define invalid_for_target_type 1224
#define invalid_group_type 1225
#define invalid_import_type 1226
#define invalid_import_from_targets_type 1227
#define invalid_with_stmt_type 1228
#define invalid_with_stmt_indent_type 1229
#define invalid_try_stmt_type 1230
#define invalid_except_stmt_type 1231
#define invalid_except_star_stmt_type 1232
#define invalid_finally_stmt_type 1233
#define invalid_except_stmt_indent_type 1234
#define invalid_except_star_stmt_indent_type 1235
#define invalid_match_stmt_type 1236
#define invalid_case_block_type 1237
#define invalid_as_pattern_type 1238
#define invalid_class_pattern_type 1239
#define invalid_class_argument_pattern_type 1240
#define invalid_if_stmt_type 1241
#define invalid_elif_stmt_type 1242
#define invalid_else_stmt_type 1243
#define invalid_while_stmt_type 1244
#define invalid_for_stmt_type 1245
#define invalid_def_raw_type 1246
#define invalid_class_def_raw_type 1247
#define invalid_double_starred_kvpairs_type 1248
#define invalid_kvpair_type 1249
#define invalid_starred_expression_unpacking_type 1250
#define invalid_starred_expression_type 1251
#define invalid_replacement_field_type 1252
#define invalid_conversion_character_type 1253
#define invalid_arithmetic_type 1254
#define invalid_factor_type 1255
#define invalid_type_params_type 1256
#define _loop0_1_type 1257
#define _loop1_2_type 1258
#define _loop0_3_type 1259
#define _gather_4_type 1260
#define _tmp_5_type 1261
#define _tmp_6_type 1262
#define _tmp_7_type 1263
#define _tmp_8_type 1264
#define _tmp_9_type 1265
#define _tmp_10_type 1266
#define _tmp_11_type 1267
#define _loop1_12_type 1268
#define _tmp_13_type 1269
#define _loop0_14_type 1270
#define _gather_15_type 1271
#define _tmp_16_type 1272
#define _tmp_17_type 1273
#define _loop0_18_type 1274
#define _loop1_19_type 1275
#define _loop0_20_type 1276
#define _gather_21_type 1277
#define _tmp_22_type 1278
#define _loop0_23_type 1279
#define _gather_24_type 1280
#define _loop1_25_type 1281
#define _tmp_26_type 1282
#define _tmp_27_type 1283
#define _loop0_28_type 1284
#define _loop0_29_type 1285
#define _loop1_30_type 1286
#define _loop1_31_type 1287
#define _loop0_32_type 1288
#define _loop1_33_type 1289
#define _loop0_34_type 1290
#define _gather_35_type 1291
#define _tmp_36_type 1292
#define _loop1_37_type 1293
#define _loop1_38_type 1294
#define _loop1_39_type 1295
#define _loop0_40_type 1296
#define _gather_41_type 1297
#define _tmp_42_type 1298
#define _tmp_43_type 1299
#define _loop0_44_type 1300
#define _gather_45_type 1301
#define _loop0_46_type 1302
#define _gather_47_type 1303
#define _tmp_48_type 1304
#define _loop0_49_type 1305
#define _gather_50_type 1306
#define _loop0_51_type 1307
#define _gather_52_type 1308
#define _loop0_53_type 1309
#define _gather_54_type 1310
#define _loop1_55_type 1311
#define _loop1_56_type 1312
#define _loop0_57_type 1313
#define _gather_58_type 1314
#define _loop1_59_type 1315
#define _loop1_60_type 1316
#define _loop1_61_type 1317
#define _tmp_62_type 1318
#define _loop0_63_type 1319
#define _gather_64_type 1320
#define _tmp_65_type 1321
#define _tmp_66_type 1322
#define _tmp_67_type 1323
#define _tmp_68_type 1324
#define _tmp_69_type 1325
#define _tmp_70_type 1326
#define _loop0_71_type 1327
#define _loop0_72_type 1328
#define _loop1_73_type 1329
#define _loop1_74_type 1330
#define _loop0_75_type 1331
#define _loop1_76_type 1332
#define _loop0_77_type 1333
#define _loop0_78_type 1334
#define _loop1_79_type 1335
#define _tmp_80_type 1336
#define _loop0_81_type 1337
#define _gather_82_type 1338
#define _loop1_83_type 1339
#define _loop0_84_type 1340
#define _tmp_85_type 1341
#define _loop0_86_type 1342
#define _gather_87_type 1343
#define _tmp_88_type 1344
#define _loop0_89_type 1345
#define _gather_90_type 1346
#define _loop0_91_type 1347
#define _gather_92_type 1348
#define _loop0_93_type 1349
#define _loop0_94_type 1350
#define _gather_95_type 1351
#define _loop1_96_type 1352
#define _tmp_97_type 1353
#define _loop0_98_type 1354
#define _gather_99_type 1355
#define _loop0_100_type 1356
#define _gather_101_type 1357
#define _tmp_102_type 1358
#define _tmp_103_type 1359
#define _loop0_104_type 1360
#define _gather_105_type 1361
#define _tmp_106_type 1362
#define _tmp_107_type 1363
#define _tmp_108_type 1364
#define _tmp_109_type 1365
#define _tmp_110_type 1366
#define _loop1_111_type 1367
#define _tmp_112_type 1368
#define _tmp_113_type 1369
#define _tmp_114_type 1370
#define _tmp_115_type 1371
#define _tmp_116_type 1372
#define _loop0_117_type 1373
#define _loop0_118_type 1374
#define _tmp_119_type 1375
#define _tmp_120_type 1376
#define _tmp_121_type 1377
#define _tmp_122_type 1378
#define _tmp_123_type 1379
#define _tmp_124_type 1380
#define _tmp_125_type 1381
#define _tmp_126_type 1382
#define _tmp_127_type 1383
#define _loop0_128_type 1384
#define _gather_129_type 1385
#define _tmp_130_type 1386
#define _tmp_131_type 1387
#define _tmp_132_type 1388
#define _tmp_133_type 1389
#define _loop0_134_type 1390
#define _gather_135_type 1391
#define _loop0_136_type 1392
#define _gather_137_type 1393
#define _loop0_138_type 1394
#define _gather_139_type 1395
#define _tmp_140_type 1396
#define _loop0_141_type 1397
#define _tmp_142_type 1398
#define _tmp_143_type 1399
#define _tmp_144_type 1400
#define _tmp_145_type 1401
#define _tmp_146_type 1402
#define _tmp_147_type 1403
#define _tmp_148_type 1404
#define _tmp_149_type 1405
#define _tmp_150_type 1406
#define _tmp_151_type 1407
#define _tmp_152_type 1408
#define _tmp_153_type 1409
#define _tmp_154_type 1410
#define _tmp_155_type 1411
#define _tmp_156_type 1412
#define _tmp_157_type 1413
#define _tmp_158_type 1414
#define _tmp_159_type 1415
#define _tmp_160_type 1416
#define _tmp_161_type 1417
#define _tmp_162_type 1418
#define _tmp_163_type 1419
#define _tmp_164_type 1420
#define _tmp_165_type 1421
#define _tmp_166_type 1422
#define _tmp_167_type 1423
#define _loop0_168_type 1424
#define _tmp_169_type 1425
#define _tmp_170_type 1426
#define _tmp_171_type 1427
#define _tmp_172_type 1428
static mod_ty file_rule(Parser *p);
static mod_ty interactive_rule(Parser *p);
@ -519,6 +520,7 @@ static mod_ty eval_rule(Parser *p);
static mod_ty func_type_rule(Parser *p);
static asdl_stmt_seq* statements_rule(Parser *p);
static asdl_stmt_seq* statement_rule(Parser *p);
static asdl_stmt_seq* single_compound_stmt_rule(Parser *p);
static asdl_stmt_seq* statement_newline_rule(Parser *p);
static asdl_stmt_seq* simple_stmts_rule(Parser *p);
static stmt_ty simple_stmt_rule(Parser *p);
@ -1167,7 +1169,7 @@ statements_rule(Parser *p)
)
{
D(fprintf(stderr, "%*c+ statements[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "statement+"));
_res = ( asdl_stmt_seq* ) _PyPegen_seq_flatten ( p , a );
_res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_seq_flatten ( p , a ) );
if (_res == NULL && PyErr_Occurred()) {
p->error_indicator = 1;
p->level--;
@ -1252,7 +1254,50 @@ statement_rule(Parser *p)
return _res;
}
// statement_newline: compound_stmt NEWLINE | simple_stmts | NEWLINE | $
// single_compound_stmt: compound_stmt
static asdl_stmt_seq*
single_compound_stmt_rule(Parser *p)
{
if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) {
_Pypegen_stack_overflow(p);
}
if (p->error_indicator) {
p->level--;
return NULL;
}
asdl_stmt_seq* _res = NULL;
int _mark = p->mark;
{ // compound_stmt
if (p->error_indicator) {
p->level--;
return NULL;
}
D(fprintf(stderr, "%*c> single_compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compound_stmt"));
stmt_ty a;
if (
(a = compound_stmt_rule(p)) // compound_stmt
)
{
D(fprintf(stderr, "%*c+ single_compound_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt"));
_res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ) );
if (_res == NULL && PyErr_Occurred()) {
p->error_indicator = 1;
p->level--;
return NULL;
}
goto done;
}
p->mark = _mark;
D(fprintf(stderr, "%*c%s single_compound_stmt[%d-%d]: %s failed!\n", p->level, ' ',
p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "compound_stmt"));
}
_res = NULL;
done:
p->level--;
return _res;
}
// statement_newline: single_compound_stmt NEWLINE | simple_stmts | NEWLINE | $
static asdl_stmt_seq*
statement_newline_rule(Parser *p)
{
@ -1274,22 +1319,22 @@ statement_newline_rule(Parser *p)
UNUSED(_start_lineno); // Only used by EXTRA macro
int _start_col_offset = p->tokens[_mark]->col_offset;
UNUSED(_start_col_offset); // Only used by EXTRA macro
{ // compound_stmt NEWLINE
{ // single_compound_stmt NEWLINE
if (p->error_indicator) {
p->level--;
return NULL;
}
D(fprintf(stderr, "%*c> statement_newline[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compound_stmt NEWLINE"));
stmt_ty a;
D(fprintf(stderr, "%*c> statement_newline[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "single_compound_stmt NEWLINE"));
asdl_stmt_seq* a;
Token * newline_var;
if (
(a = compound_stmt_rule(p)) // compound_stmt
(a = single_compound_stmt_rule(p)) // single_compound_stmt
&&
(newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE'
)
{
D(fprintf(stderr, "%*c+ statement_newline[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt NEWLINE"));
_res = ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a );
D(fprintf(stderr, "%*c+ statement_newline[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "single_compound_stmt NEWLINE"));
_res = a;
if (_res == NULL && PyErr_Occurred()) {
p->error_indicator = 1;
p->level--;
@ -1299,7 +1344,7 @@ statement_newline_rule(Parser *p)
}
p->mark = _mark;
D(fprintf(stderr, "%*c%s statement_newline[%d-%d]: %s failed!\n", p->level, ' ',
p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "compound_stmt NEWLINE"));
p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "single_compound_stmt NEWLINE"));
}
{ // simple_stmts
if (p->error_indicator) {

64
Parser/pegen.c
View file

@ -799,7 +799,7 @@ compute_parser_flags(PyCompilerFlags *flags)
Parser *
_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
int feature_version, int *errcode, PyArena *arena)
int feature_version, int *errcode, const char* source, PyArena *arena)
{
Parser *p = PyMem_Malloc(sizeof(Parser));
if (p == NULL) {
@ -847,6 +847,10 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
p->known_err_token = NULL;
p->level = 0;
p->call_invalid_rules = 0;
p->last_stmt_location.lineno = 0;
p->last_stmt_location.col_offset = 0;
p->last_stmt_location.end_lineno = 0;
p->last_stmt_location.end_col_offset = 0;
#ifdef Py_DEBUG
p->debug = _Py_GetConfig()->parser_debug;
#endif
@ -868,6 +872,10 @@ _PyPegen_Parser_Free(Parser *p)
static void
reset_parser_state_for_error_pass(Parser *p)
{
p->last_stmt_location.lineno = 0;
p->last_stmt_location.col_offset = 0;
p->last_stmt_location.end_lineno = 0;
p->last_stmt_location.end_col_offset = 0;
for (int i = 0; i < p->fill; i++) {
p->tokens[i]->memo = NULL;
}
@ -884,6 +892,51 @@ _is_end_of_source(Parser *p) {
return err == E_EOF || err == E_EOFS || err == E_EOLS;
}
static void
_PyPegen_set_syntax_error_metadata(Parser *p) {
PyObject *exc = PyErr_GetRaisedException();
if (!exc || !PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_SyntaxError)) {
PyErr_SetRaisedException(exc);
return;
}
const char *source = NULL;
if (p->tok->str != NULL) {
source = p->tok->str;
}
if (!source && p->tok->fp_interactive && p->tok->interactive_src_start) {
source = p->tok->interactive_src_start;
}
PyObject* the_source = NULL;
if (source) {
if (p->tok->encoding == NULL) {
the_source = PyUnicode_FromString(source);
} else {
the_source = PyUnicode_Decode(source, strlen(source), p->tok->encoding, NULL);
}
}
if (!the_source) {
PyErr_Clear();
the_source = Py_None;
Py_INCREF(the_source);
}
PyObject* metadata = Py_BuildValue(
"(iiN)",
p->last_stmt_location.lineno,
p->last_stmt_location.col_offset,
the_source // N gives ownership to metadata
);
if (!metadata) {
Py_DECREF(the_source);
PyErr_Clear();
return;
}
PySyntaxErrorObject *syntax_error = (PySyntaxErrorObject *)exc;
Py_XDECREF(syntax_error->metadata);
syntax_error->metadata = metadata;
PyErr_SetRaisedException(exc);
}
void *
_PyPegen_run_parser(Parser *p)
{
@ -907,6 +960,11 @@ _PyPegen_run_parser(Parser *p)
// Set SyntaxErrors accordingly depending on the parser/tokenizer status at the failure
// point.
_Pypegen_set_syntax_error(p, last_token);
// Set the metadata in the exception from p->last_stmt_location
if (PyErr_ExceptionMatches(PyExc_SyntaxError)) {
_PyPegen_set_syntax_error_metadata(p);
}
return NULL;
}
@ -955,7 +1013,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
int parser_flags = compute_parser_flags(flags);
Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
errcode, arena);
errcode, NULL, arena);
if (p == NULL) {
goto error;
}
@ -1005,7 +1063,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ?
flags->cf_feature_version : PY_MINOR_VERSION;
Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
NULL, arena);
NULL, str, arena);
if (p == NULL) {
goto error;
}

13
Parser/pegen.h
View file

@ -55,6 +55,13 @@ typedef struct {
size_t num_items;
} growable_comment_array;
typedef struct {
int lineno;
int col_offset;
int end_lineno;
int end_col_offset;
} location;
typedef struct {
struct tok_state *tok;
Token **tokens;
@ -78,6 +85,7 @@ typedef struct {
int level;
int call_invalid_rules;
int debug;
location last_stmt_location;
} Parser;
typedef struct {
@ -148,6 +156,7 @@ int _PyPegen_fill_token(Parser *p);
expr_ty _PyPegen_name_token(Parser *p);
expr_ty _PyPegen_number_token(Parser *p);
void *_PyPegen_string_token(Parser *p);
PyObject *_PyPegen_set_source_in_metadata(Parser *p, Token *t);
Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset);
Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset);
Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char*, Py_ssize_t col_offset);
@ -348,10 +357,12 @@ expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension);
void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions);
stmt_ty _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq *,
int , int, int , int , int , PyArena *);
asdl_stmt_seq* _PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts);
stmt_ty _PyPegen_register_stmt(Parser *p, stmt_ty s);
// Parser API
Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, const char*, PyArena *);
void _PyPegen_Parser_Free(Parser *);
mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
const char *, const char *, PyCompilerFlags *, int *, PyObject **,