[red-knot] handle all syntax without panic (#12499)

Extend red-knot type inference to cover all syntax, so that inferring
types for a scope gives all expressions a type. This means we can run
the red-knot semantic lint on all Python code without panics. It also
means we can infer types for `builtins.pyi` without panics.

To keep things simple, this PR intentionally doesn't add any new type
inference capabilities: the expanded coverage is all achieved with
`Type::Unknown`. But this puts the skeleton in place for adding better
inference of all these language features.

I also had to add basic Salsa cycle recovery (with just `Type::Unknown`
for now), because some `builtins.pyi` definitions are cyclic.

To test this, I added a comprehensive corpus of test snippets sourced
from Cinder under [MIT
license](https://github.com/facebookincubator/cinder/blob/cinder/3.10/cinderx/LICENSE),
which matches Ruff's license. I also added to this corpus some
additional snippets for newer language features: all the
`27_func_generic_*` and `73_class_generic_*` files, as well as
`20_lambda_default_arg.py`, and added a test which runs semantic-lint
over all these files. (The test doesn't assert the test-corpus files are
lint-free; just that they are able to lint without a panic.)

.pre-commit-config.yaml

@@ -9,7 +9,8 @@ exclude: |
     crates/ruff_python_formatter/resources/.*|
     crates/ruff_python_formatter/tests/snapshots/.*|
     crates/ruff_python_resolver/resources/.*|
-    crates/ruff_python_resolver/tests/snapshots/.*
+    crates/ruff_python_resolver/tests/snapshots/.*|
+    crates/red_knot/resources/.*
   )$
 
 repos:
@@ -63,11 +64,6 @@ repos:
         args: [--fix, --exit-non-zero-on-fix]
         types_or: [python, pyi]
         require_serial: true
-        exclude: |
-          (?x)^(
-            crates/ruff_linter/resources/.*|
-            crates/ruff_python_formatter/resources/.*
-          )$
 
   # Prettier
   - repo: https://github.com/pre-commit/mirrors-prettier

crates/red_knot/resources/test/corpus/00_const.py

@@ -0,0 +1,5 @@
+None
+False
+True
+Ellipsis
+...

crates/red_knot/resources/test/corpus/00_expr_discard.py

@@ -0,0 +1,4 @@
+"str"
+1
+1.1
+b"bin"

crates/red_knot/resources/test/corpus/00_expr_var1.py

@@ -0,0 +1 @@
+a

crates/red_knot/resources/test/corpus/01_expr_unary.py

@@ -0,0 +1,4 @@
+-a
+~a
++a
+not a

crates/red_knot/resources/test/corpus/02_expr_attr.py

@@ -0,0 +1,2 @@
+a.b
+a.b.c.d

crates/red_knot/resources/test/corpus/02_expr_attr_multiline.py

@@ -0,0 +1,3 @@
+foo(
+    bar=1
+).attr

crates/red_knot/resources/test/corpus/02_expr_attr_multiline_assign.py

@@ -0,0 +1,3 @@
+foo(
+    bar=1
+).attr = 1

crates/red_knot/resources/test/corpus/02_expr_bin_bool.py

@@ -0,0 +1,6 @@
+a or b or c
+a and b and c
+
+a or b and c
+
+#a and b or c

crates/red_knot/resources/test/corpus/02_expr_binary.py

@@ -0,0 +1,14 @@
+a + b
+a - b
+a * b
+a @ b
+a / b
+a // b
+a % b
+a ** b
+
+a << b
+a >> b
+a | b
+a ^ b
+a & b

crates/red_knot/resources/test/corpus/02_expr_bool_op_multiline.py

@@ -0,0 +1,6 @@
+x == y or (
+    x is not None and x == z
+)
+
+x == y or \
+    x <= 65 or x >= 102

crates/red_knot/resources/test/corpus/02_expr_bool_op_multiline2.py

@@ -0,0 +1,5 @@
+(
+    (a and aa) or
+    (b and bb) or
+    (c and cc)
+)

crates/red_knot/resources/test/corpus/02_expr_rel.py

@@ -0,0 +1,10 @@
+a == b
+a != b
+a < b
+a <= b
+a > b
+a >= b
+a is b
+a is not b
+a in b
+a not in b

crates/red_knot/resources/test/corpus/02_expr_rel_multiple.py

@@ -0,0 +1,2 @@
+a == b == 0
+0 < a < 10

crates/red_knot/resources/test/corpus/02_expr_subscr.py

@@ -0,0 +1,2 @@
+a[b]
+a[b][c]

crates/red_knot/resources/test/corpus/03_dict.py

@@ -0,0 +1 @@
+{a: 1, b:2}

crates/red_knot/resources/test/corpus/03_dict_ex.py

@@ -0,0 +1 @@
+{1: 2, **a, 3: 4, 5: 6}

crates/red_knot/resources/test/corpus/03_dict_literal_large.py

@@ -0,0 +1,20 @@
+DATA = {
+    'a': 1,
+    'b': 1,
+    'c': 1,
+    'd': 1,
+    'e': 1,
+    'f': 1,
+    'g': 1,
+    'h': 1,
+    'i': 1,
+    'j': 1,
+    'k': 1,
+    'l': 1,
+    'm': 1,
+    'n': 1,
+    'o': 1,
+    'p': 1,
+    'q': 1,
+    'r': 1,
+}

crates/red_knot/resources/test/corpus/03_dict_unpack_huge.py

@@ -0,0 +1,2 @@
+{**{0:1},**{1:2},**{2:3},**{3:4},**{4:5},**{5:6},**{6:7},**{7:8},**{8:9},**{9:10},**{10:11},**{11:12},**{12:13},**{13:14},**{14:15},**{15:16},**{16:17},**{17:18},**{18:19},**{19:20},**{20:21},**{21:22},**{22:23},**{23:24},**{24:25},**{25:26},**{26:27},**{27:28},**{28:29},**{29:30},**{30:31},**{31:32},**{32:33},**{33:34},**{34:35},**{35:36},**{36:37},**{37:38},**{38:39},**{39:40},**{40:41},**{41:42},**{42:43},**{43:44},**{44:45},**{45:46},**{46:47},**{47:48},**{48:49},**{49:50},**{50:51},**{51:52},**{52:53},**{53:54},**{54:55},**{55:56},**{56:57},**{57:58},**{58:59},**{59:60},**{60:61},**{61:62},**{62:63},**{63:64},**{64:65},**{65:66},**{66:67},**{67:68},**{68:69},**{69:70},**{70:71},**{71:72},**{72:73},**{73:74},**{74:75},**{75:76},**{76:77},**{77:78},**{78:79},**{79:80},**{80:81},**{81:82},**{82:83},**{83:84},**{84:85},**{85:86},**{86:87},**{87:88},**{88:89},**{89:90},**{90:91},**{91:92},**{92:93},**{93:94},**{94:95},**{95:96},**{96:97},**{97:98},**{98:99},**{99:100},**{100:101},**{101:102},**{102:103},**{103:104},**{104:105},**{105:106},**{106:107},**{107:108},**{108:109},**{109:110},**{110:111},**{111:112},**{112:113},**{113:114},**{114:115},**{115:116},**{116:117},**{117:118},**{118:119},**{119:120},**{120:121},**{121:122},**{122:123},**{123:124},**{124:125},**{125:126},**{126:127},**{127:128},**{128:129},**{129:130},**{130:131},**{131:132},**{132:133},**{133:134},**{134:135},**{135:136},**{136:137},**{137:138},**{138:139},**{139:140},**{140:141},**{141:142},**{142:143},**{143:144},**{144:145},**{145:146},**{146:147},**{147:148},**{148:149},**{149:150},**{150:151},**{151:152},**{152:153},**{153:154},**{154:155},**{155:156},**{156:157},**{157:158},**{158:159},**{159:160},**{160:161},**{161:162},**{162:163},**{163:164},**{164:165},**{165:166},**{166:167},**{167:168},**{168:169},**{169:170},**{170:171},**{171:172},**{172:173},**{173:174},**{174:175},**{175:176},**{176:177},**{177:178},**{178:179},**{179:180},**{180:181},**{181:182},**{182:183},**{183:184},**{184:185},**{185:186},**{186:187},**{187:188},**{188:189},**{189:190},**{190:191},**{191:192},**{192:193},**{193:194},**{194:195},**{195:196},**{196:197},**{197:198},**{198:199},**{199:200},**{200:201},**{201:202},**{202:203},**{203:204},**{204:205},**{205:206},**{206:207},**{207:208},**{208:209},**{209:210},**{210:211},**{211:212},**{212:213},**{213:214},**{214:215},**{215:216},**{216:217},**{217:218},**{218:219},**{219:220},**{220:221},**{221:222},**{222:223},**{223:224},**{224:225},**{225:226},**{226:227},**{227:228},**{228:229},**{229:230},**{230:231},**{231:232},**{232:233},**{233:234},**{234:235},**{235:236},**{236:237},**{237:238},**{238:239},**{239:240},**{240:241},**{241:242},**{242:243},**{243:244},**{244:245},**{245:246},**{246:247},**{247:248},**{248:249},**{249:250},**{250:251},**{251:252},**{252:253},**{253:254},**{254:255},**{255:256},**{256:257},**{257:258},**{258:259},**{259:260}}
+

crates/red_knot/resources/test/corpus/03_list.py

@@ -0,0 +1 @@
+[a, b]

crates/red_knot/resources/test/corpus/03_list_ex.py

@@ -0,0 +1 @@
+[a, *b, *d, a, c]

crates/red_knot/resources/test/corpus/03_list_large.py

@@ -0,0 +1 @@
+l = [a, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]

crates/red_knot/resources/test/corpus/03_set.py

@@ -0,0 +1 @@
+{a, b}

crates/red_knot/resources/test/corpus/03_set_multi.py

@@ -0,0 +1,38 @@
+d = {
+    0: {
+        "en",
+        "es",
+        "zh",
+        "ja",
+        "de",
+        "fr",
+        "ru",
+        "ar",
+        "pt",
+        "fa",
+        "tr",
+        "ko",
+        "id",
+        None,
+        (1, "2"),
+        (1, 2),
+    },
+    1: {
+        "en",
+        "de",
+        "fr",
+        "ar",
+        "pt",
+        "ja",
+        "zh",
+        "ru",
+        None,
+        "es",
+        "fa",
+        "tr",
+        "ko",
+        "id",
+        (1, "2"),
+        (1, 2),
+    },
+}

crates/red_knot/resources/test/corpus/03_slice.py

@@ -0,0 +1,6 @@
+arr[a:b]
+arr[a:b:c]
+arr[a:]
+arr[:b]
+arr[:]
+arr[::2]

crates/red_knot/resources/test/corpus/03_slice_ext.py

@@ -0,0 +1 @@
+arr[a:b, c:d]

crates/red_knot/resources/test/corpus/03_tuple.py

@@ -0,0 +1,2 @@
+(a,)
+(b, c)

crates/red_knot/resources/test/corpus/03_tuple_ex.py

@@ -0,0 +1,2 @@
+(*b,)
+(*b, c)

crates/red_knot/resources/test/corpus/04_assign.py

@@ -0,0 +1,4 @@
+a = 1
+b = "foo"
+c = (d, e)
+di = {f: 1, g: 2}

crates/red_knot/resources/test/corpus/04_assign_attr.py

@@ -0,0 +1,2 @@
+a.b = 1
+a.b.c.d = 2

crates/red_knot/resources/test/corpus/04_assign_attr_func.py

@@ -0,0 +1,2 @@
+def foo():
+    a.b = 1

crates/red_knot/resources/test/corpus/04_assign_subscr.py

@@ -0,0 +1 @@
+a[b] = 1

crates/red_knot/resources/test/corpus/04_assign_unpack.py

@@ -0,0 +1,2 @@
+a, b = c
+[a, b] = c

crates/red_knot/resources/test/corpus/04_assign_unpack_ex.py

@@ -0,0 +1,4 @@
+a, *b = c
+*a, b = c
+a, *b, c, d = e
+[a, *b, c] = d

crates/red_knot/resources/test/corpus/04_assign_unpack_tuple.py

@@ -0,0 +1,5 @@
+a, = x,
+a, b = x, y
+a, b, c = x, y, z
+a, b, c, d = w, x, y, z
+a, b = 1, 2

crates/red_knot/resources/test/corpus/04_aug_assign.py

@@ -0,0 +1,14 @@
+a += 1
+a -= 2
+a *= 3
+a @= 4
+a /= 5
+a //= 6
+a %= 7
+a **= 8
+
+a <<= b
+a >>= b
+a |= b
+a ^= b
+a &= b

crates/red_knot/resources/test/corpus/04_aug_assign_attr_multiline.py

@@ -0,0 +1,6 @@
+(
+    o.
+    a
+) += (
+    v
+)

crates/red_knot/resources/test/corpus/04_aug_assign_attr_sub.py

@@ -0,0 +1,3 @@
+a.b -= 2
+a[0] += 1
+a[0:2] += 1

crates/red_knot/resources/test/corpus/05_funcall.py

@@ -0,0 +1 @@
+fun()

crates/red_knot/resources/test/corpus/05_funcall_1.py

@@ -0,0 +1 @@
+fun(a)

crates/red_knot/resources/test/corpus/05_funcall_2.py

@@ -0,0 +1 @@
+fun(a, b)

crates/red_knot/resources/test/corpus/05_funcall_in_multiline_tuple.py

@@ -0,0 +1,2 @@
+a = (x,
+    foo(y))

crates/red_knot/resources/test/corpus/05_funcall_kw.py

@@ -0,0 +1 @@
+fun(a=a, kw=2)

crates/red_knot/resources/test/corpus/05_funcall_kw_many.py

@@ -0,0 +1,18 @@
+foo(
+    a=1,
+    b=1,
+    c=1,
+    d=1,
+    e=1,
+    f=1,
+    g=1,
+    h=1,
+    i=1,
+    j=1,
+    k=1,
+    l=1,
+    m=1,
+    n=1,
+    o=1,
+    p=1,
+)

crates/red_knot/resources/test/corpus/05_funcall_kw_pos.py

@@ -0,0 +1 @@
+fun(var, 10, a=a, kw=2)

crates/red_knot/resources/test/corpus/05_funcall_method_multiline.py

@@ -0,0 +1,2 @@
+("foo"
+.format())

crates/red_knot/resources/test/corpus/06_funcall_kwargs.py

@@ -0,0 +1,2 @@
+c = {a: 1, b: 2}
+fun(a, b, **c)

crates/red_knot/resources/test/corpus/06_funcall_many_args.py

@@ -0,0 +1,32 @@
+C.meth(
+    a,
+    b,
+    c,
+    d,
+    e,
+    f,
+    g,
+    h,
+    i,
+    j,
+    k,
+    l,
+    m,
+    n,
+    o,
+    p,
+    q,
+    r,
+    s,
+    t,
+    u,
+    v,
+    w,
+    x,
+    y,
+    z,
+    aa,
+    bb,
+    cc,
+    dd,
+)

crates/red_knot/resources/test/corpus/06_funcall_starargs_ex.py

@@ -0,0 +1,3 @@
+fun(*b, c)
+fun(a, *b, c)
+fun(a, *b, c, *d)

crates/red_knot/resources/test/corpus/06_funcall_varargs.py

@@ -0,0 +1,2 @@
+c = (a, b)
+fun(a, b, *c)

crates/red_knot/resources/test/corpus/06_funcall_varargs_kwargs.py

@@ -0,0 +1,3 @@
+c = (a, b)
+d = {e: 1, f: 2}
+fun(a, b, *c, **d)

crates/red_knot/resources/test/corpus/06_funcall_varargs_kwargs_mixed.py

@@ -0,0 +1,10 @@
+fun(a, kw=1, *c, **d)
+fun(a, *c, kw=1, **d)
+
+fun(a, kw=1, *c)
+fun(a, *c, kw=1)
+
+fun(a, *c)
+
+# Introduced in Python3.5, not supported yet
+#fun(*c, a)

crates/red_knot/resources/test/corpus/07_ifexpr.py

@@ -0,0 +1,2 @@
+a if b else c
+

crates/red_knot/resources/test/corpus/07_ifexpr_multiline.py

@@ -0,0 +1,4 @@
+(
+    x,
+    a if b else c
+)

crates/red_knot/resources/test/corpus/07_ifexpr_multiline2.py

@@ -0,0 +1,7 @@
+a = (
+    (
+        b()
+    )
+    if c
+    else d
+)

crates/red_knot/resources/test/corpus/08_del.py

@@ -0,0 +1,3 @@
+del a
+del a[0]
+del a.b

crates/red_knot/resources/test/corpus/08_del_multi.py

@@ -0,0 +1,2 @@
+del (a, b, c)
+del [a, b, c]

crates/red_knot/resources/test/corpus/09_pass.py

@@ -0,0 +1 @@
+pass

crates/red_knot/resources/test/corpus/10_if.py

@@ -0,0 +1,2 @@
+if a:
+    b

crates/red_knot/resources/test/corpus/10_if_chained_compare.py

@@ -0,0 +1,2 @@
+if 0 < x < 10:
+    pass

crates/red_knot/resources/test/corpus/10_if_false.py

@@ -0,0 +1,28 @@
+if 0:
+    a
+
+if False:
+    b
+
+if None:
+    c
+
+if "":
+    d
+
+if 0:
+    e.f
+    g.h()
+    i.j = 1
+    del k.l
+    import m
+    from n import o
+    p = 1
+
+def f():
+    if 0:
+        q = 1
+        r.s = 1
+        t
+        import u
+        v = u.w()

crates/red_knot/resources/test/corpus/10_if_true.py

@@ -0,0 +1,8 @@
+if 1:
+    a
+
+if True:
+    b
+
+if "foo":
+    c

crates/red_knot/resources/test/corpus/11_if_else.py

@@ -0,0 +1,4 @@
+if a:
+    b
+else:
+    c

crates/red_knot/resources/test/corpus/11_if_else_deeply_nested_for.py

@@ -0,0 +1,12 @@
+if a:
+    if b:
+        if c:
+            for x in y:
+                pass
+        else:
+            pass
+    else:
+        pass
+else:
+    pass
+foo()

crates/red_knot/resources/test/corpus/11_if_else_false.py

@@ -0,0 +1,4 @@
+if False:
+    b
+else:
+    c

crates/red_knot/resources/test/corpus/11_if_else_true.py

@@ -0,0 +1,4 @@
+if True:
+    b
+else:
+    c

crates/red_knot/resources/test/corpus/12_if_elif.py

@@ -0,0 +1,4 @@
+if a:
+    b
+elif c:
+    d

crates/red_knot/resources/test/corpus/12_if_elif_else.py

@@ -0,0 +1,6 @@
+if a:
+    b
+elif c:
+    d
+else:
+    e

crates/red_knot/resources/test/corpus/13_ifelse_complex1.py

@@ -0,0 +1,12 @@
+if py2:
+    a
+else:
+    b
+
+if var:
+    c
+else:
+    if py3:
+        d
+    else:
+        e

crates/red_knot/resources/test/corpus/13_ifelse_many.py

@@ -0,0 +1,8 @@
+if a:
+    a1
+elif b:
+    b1
+elif c:
+    c1
+elif d:
+    d1

crates/red_knot/resources/test/corpus/15_while.py

@@ -0,0 +1,2 @@
+while a:
+    b

crates/red_knot/resources/test/corpus/15_while_break.py

@@ -0,0 +1,2 @@
+while a:
+    break

crates/red_knot/resources/test/corpus/15_while_break_in_finally.py

@@ -0,0 +1,5 @@
+while a:
+    try:
+        continue
+    finally:
+        break

crates/red_knot/resources/test/corpus/15_while_break_non_empty.py

@@ -0,0 +1,4 @@
+while a:
+    if x:
+        break
+    y = 1

crates/red_knot/resources/test/corpus/15_while_break_non_exit.py

@@ -0,0 +1,7 @@
+while a:
+    try:
+       x
+    except:
+        break
+if x:
+    z = 1

crates/red_knot/resources/test/corpus/15_while_continue.py

@@ -0,0 +1,2 @@
+while a:
+    continue

crates/red_knot/resources/test/corpus/15_while_false.py

@@ -0,0 +1,2 @@
+while False:
+    b

crates/red_knot/resources/test/corpus/15_while_infinite.py

@@ -0,0 +1,2 @@
+while 1:
+    b

crates/red_knot/resources/test/corpus/15_while_true.py

@@ -0,0 +1,2 @@
+while True:
+    b

crates/red_knot/resources/test/corpus/16_for.py

@@ -0,0 +1,2 @@
+for a in b:
+    c

crates/red_knot/resources/test/corpus/16_for_break.py

@@ -0,0 +1,2 @@
+for a in b:
+    break

crates/red_knot/resources/test/corpus/16_for_continue.py

@@ -0,0 +1,2 @@
+for a in b:
+    continue

crates/red_knot/resources/test/corpus/16_for_else.py

@@ -0,0 +1,4 @@
+for a in b:
+    c
+else:
+    d

crates/red_knot/resources/test/corpus/16_for_list_literal.py

@@ -0,0 +1,2 @@
+for x in [a, b]:
+    pass

crates/red_knot/resources/test/corpus/16_for_nested_ifs.py

@@ -0,0 +1,8 @@
+for x in y:
+    if a:
+        if x:
+            y
+        else:
+            x
+    else:
+        b

crates/red_knot/resources/test/corpus/20_lambda.py

@@ -0,0 +1,3 @@
+lambda x: y
+
+lambda x: a and b

crates/red_knot/resources/test/corpus/20_lambda_const.py

@@ -0,0 +1 @@
+lambda x: y + 1

crates/red_knot/resources/test/corpus/20_lambda_default_arg.py

@@ -0,0 +1 @@
+lambda x=a: y

crates/red_knot/resources/test/corpus/20_lambda_ifelse.py

@@ -0,0 +1 @@
+lambda x: 0 if x else -1

crates/red_knot/resources/test/corpus/21_func1.py

@@ -0,0 +1,2 @@
+def foo():
+    a

crates/red_knot/resources/test/corpus/21_func1_ret.py

@@ -0,0 +1,2 @@
+def foo():
+    return a

crates/red_knot/resources/test/corpus/21_func_assign.py

@@ -0,0 +1,3 @@
+def foo():
+    a = 2
+    a

crates/red_knot/resources/test/corpus/21_func_assign2.py

@@ -0,0 +1,4 @@
+def foo():
+    # This is runtime, not compile-time error
+    a
+    a = 2

crates/red_knot/resources/test/corpus/22_func_arg.py

@@ -0,0 +1,2 @@
+def foo(a, b):
+    a + b

crates/red_knot/resources/test/corpus/22_func_vararg.py

@@ -0,0 +1,8 @@
+def foo1(*args):
+    func(args)
+
+def foo2(**kwargs):
+    func(kwargs)
+
+def foo3(a, *args, **kw):
+    func(a, args, kw)

crates/red_knot/resources/test/corpus/23_func_ret.py

@@ -0,0 +1,2 @@
+def foo(a):
+    return

crates/red_knot/resources/test/corpus/23_func_ret_val.py

@@ -0,0 +1,2 @@
+def foo(a):
+    return a

crates/red_knot/resources/test/corpus/24_func_if_ret.py

@@ -0,0 +1,4 @@
+def foo(a):
+    if a:
+        return b
+    return c

crates/red_knot/resources/test/corpus/24_func_ifelse_ret.py

@@ -0,0 +1,5 @@
+def foo(a):
+    if a:
+        return b
+    else:
+        return c