Rename ruff_python crate to ruff_python_stdlib (#3354)

In hindsight, `ruff_python` is too general. A good giveaway is that it's actually a prefix of some other crates. The intent of this crate is to reimplement pieces of the Python standard library and CPython itself, so `ruff_python_stdlib` feels appropriate.

crates/ruff_python_stdlib/src/builtins.rs

@@ -0,0 +1,168 @@
+pub const BUILTINS: &[&str] = &[
+    "ArithmeticError",
+    "AssertionError",
+    "AttributeError",
+    "BaseException",
+    "BaseExceptionGroup",
+    "BlockingIOError",
+    "BrokenPipeError",
+    "BufferError",
+    "BytesWarning",
+    "ChildProcessError",
+    "ConnectionAbortedError",
+    "ConnectionError",
+    "ConnectionRefusedError",
+    "ConnectionResetError",
+    "DeprecationWarning",
+    "EOFError",
+    "Ellipsis",
+    "EncodingWarning",
+    "EnvironmentError",
+    "Exception",
+    "ExceptionGroup",
+    "False",
+    "FileExistsError",
+    "FileNotFoundError",
+    "FloatingPointError",
+    "FutureWarning",
+    "GeneratorExit",
+    "IOError",
+    "ImportError",
+    "ImportWarning",
+    "IndentationError",
+    "IndexError",
+    "InterruptedError",
+    "IsADirectoryError",
+    "KeyError",
+    "KeyboardInterrupt",
+    "LookupError",
+    "MemoryError",
+    "ModuleNotFoundError",
+    "NameError",
+    "None",
+    "NotADirectoryError",
+    "NotImplemented",
+    "NotImplementedError",
+    "OSError",
+    "OverflowError",
+    "PendingDeprecationWarning",
+    "PermissionError",
+    "ProcessLookupError",
+    "RecursionError",
+    "ReferenceError",
+    "ResourceWarning",
+    "RuntimeError",
+    "RuntimeWarning",
+    "StopAsyncIteration",
+    "StopIteration",
+    "SyntaxError",
+    "SyntaxWarning",
+    "SystemError",
+    "SystemExit",
+    "TabError",
+    "TimeoutError",
+    "True",
+    "TypeError",
+    "UnboundLocalError",
+    "UnicodeDecodeError",
+    "UnicodeEncodeError",
+    "UnicodeError",
+    "UnicodeTranslateError",
+    "UnicodeWarning",
+    "UserWarning",
+    "ValueError",
+    "Warning",
+    "ZeroDivisionError",
+    "__build_class__",
+    "__debug__",
+    "__doc__",
+    "__import__",
+    "__loader__",
+    "__name__",
+    "__package__",
+    "__spec__",
+    "abs",
+    "aiter",
+    "all",
+    "anext",
+    "any",
+    "ascii",
+    "bin",
+    "bool",
+    "breakpoint",
+    "bytearray",
+    "bytes",
+    "callable",
+    "chr",
+    "classmethod",
+    "compile",
+    "complex",
+    "copyright",
+    "credits",
+    "delattr",
+    "dict",
+    "dir",
+    "divmod",
+    "enumerate",
+    "eval",
+    "exec",
+    "exit",
+    "filter",
+    "float",
+    "format",
+    "frozenset",
+    "getattr",
+    "globals",
+    "hasattr",
+    "hash",
+    "help",
+    "hex",
+    "id",
+    "input",
+    "int",
+    "isinstance",
+    "issubclass",
+    "iter",
+    "len",
+    "license",
+    "list",
+    "locals",
+    "map",
+    "max",
+    "memoryview",
+    "min",
+    "next",
+    "object",
+    "oct",
+    "open",
+    "ord",
+    "pow",
+    "print",
+    "property",
+    "quit",
+    "range",
+    "repr",
+    "reversed",
+    "round",
+    "set",
+    "setattr",
+    "slice",
+    "sorted",
+    "staticmethod",
+    "str",
+    "sum",
+    "super",
+    "tuple",
+    "type",
+    "vars",
+    "zip",
+];
+
+// Globally defined names which are not attributes of the builtins module, or
+// are only present on some platforms.
+pub const MAGIC_GLOBALS: &[&str] = &[
+    "WindowsError",
+    "__annotations__",
+    "__builtins__",
+    "__file__",
+];

crates/ruff_python_stdlib/src/bytes.rs

@@ -0,0 +1,10 @@
+/// See: <https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals>
+pub const TRIPLE_QUOTE_PREFIXES: &[&str] = &[
+    "br'''", "rb'''", "bR'''", "Rb'''", "Br'''", "rB'''", "RB'''", "BR'''", "b'''", "br\"\"\"",
+    "rb\"\"\"", "bR\"\"\"", "Rb\"\"\"", "Br\"\"\"", "rB\"\"\"", "RB\"\"\"", "BR\"\"\"", "b\"\"\"",
+    "B\"\"\"",
+];
+pub const SINGLE_QUOTE_PREFIXES: &[&str] = &[
+    "br'", "rb'", "bR'", "Rb'", "Br'", "rB'", "RB'", "BR'", "b'", "br\"", "rb\"", "bR\"", "Rb\"",
+    "Br\"", "rB\"", "RB\"", "BR\"", "b\"", "B\"",
+];

crates/ruff_python_stdlib/src/future.rs

@@ -0,0 +1,13 @@
+/// A copy of `__future__.all_feature_names`.
+pub const ALL_FEATURE_NAMES: &[&str] = &[
+    "nested_scopes",
+    "generators",
+    "division",
+    "absolute_import",
+    "with_statement",
+    "print_function",
+    "unicode_literals",
+    "barry_as_FLUFL",
+    "generator_stop",
+    "annotations",
+];

crates/ruff_python_stdlib/src/identifiers.rs

@@ -0,0 +1,49 @@
+/// Returns `true` if a string is a valid Python identifier (e.g., variable
+/// name).
+pub fn is_identifier(s: &str) -> bool {
+    // Is the first character a letter or underscore?
+    if !s
+        .chars()
+        .next()
+        .map_or(false, |c| c.is_alphabetic() || c == '_')
+    {
+        return false;
+    }
+
+    // Are the rest of the characters letters, digits, or underscores?
+    s.chars().skip(1).all(|c| c.is_alphanumeric() || c == '_')
+}
+
+/// Returns `true` if a string is a private identifier, such that, when the
+/// identifier is defined in a class definition, it will be mangled prior to
+/// code generation.
+///
+/// See: <https://docs.python.org/3.5/reference/expressions.html?highlight=mangling#index-5>.
+pub fn is_mangled_private(id: &str) -> bool {
+    id.starts_with("__") && !id.ends_with("__")
+}
+
+/// Returns `true` if a string is a PEP 8-compliant module name (i.e., consists of lowercase
+/// letters, numbers, and underscores).
+pub fn is_module_name(s: &str) -> bool {
+    s.chars()
+        .all(|c| c.is_lowercase() || c.is_numeric() || c == '_')
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::identifiers::is_module_name;
+
+    #[test]
+    fn test_is_module_name() {
+        assert!(is_module_name("a"));
+        assert!(is_module_name("abc"));
+        assert!(is_module_name("abc0"));
+        assert!(is_module_name("abc_"));
+        assert!(is_module_name("a_b_c"));
+        assert!(is_module_name("0abc"));
+        assert!(is_module_name("_abc"));
+        assert!(!is_module_name("a-b-c"));
+        assert!(!is_module_name("a_B_c"));
+    }
+}

crates/ruff_python_stdlib/src/keyword.rs

@@ -0,0 +1,7 @@
+// See: https://github.com/python/cpython/blob/9d692841691590c25e6cf5b2250a594d3bf54825/Lib/keyword.py#L18
+pub const KWLIST: [&str; 35] = [
+    "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue",
+    "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import",
+    "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while",
+    "with", "yield",
+];

crates/ruff_python_stdlib/src/lib.rs

@@ -0,0 +1,8 @@
+pub mod builtins;
+pub mod bytes;
+pub mod future;
+pub mod identifiers;
+pub mod keyword;
+pub mod str;
+pub mod sys;
+pub mod typing;

crates/ruff_python_stdlib/src/str.rs

@@ -0,0 +1,87 @@
+use once_cell::sync::Lazy;
+use regex::Regex;
+
+/// See: <https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals>
+pub const TRIPLE_QUOTE_PREFIXES: &[&str] = &[
+    "u\"\"\"", "u'''", "r\"\"\"", "r'''", "U\"\"\"", "U'''", "R\"\"\"", "R'''", "\"\"\"", "'''",
+];
+pub const SINGLE_QUOTE_PREFIXES: &[&str] = &[
+    "u\"", "u'", "r\"", "r'", "U\"", "U'", "R\"", "R'", "\"", "'",
+];
+pub const TRIPLE_QUOTE_SUFFIXES: &[&str] = &["\"\"\"", "'''"];
+pub const SINGLE_QUOTE_SUFFIXES: &[&str] = &["\"", "'"];
+
+pub static STRING_QUOTE_PREFIX_REGEX: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r#"^(?i)[urb]*['"](?P<raw>.*)['"]$"#).unwrap());
+
+pub fn is_lower(s: &str) -> bool {
+    let mut cased = false;
+    for c in s.chars() {
+        if c.is_uppercase() {
+            return false;
+        } else if !cased && c.is_lowercase() {
+            cased = true;
+        }
+    }
+    cased
+}
+
+pub fn is_upper(s: &str) -> bool {
+    let mut cased = false;
+    for c in s.chars() {
+        if c.is_lowercase() {
+            return false;
+        } else if !cased && c.is_uppercase() {
+            cased = true;
+        }
+    }
+    cased
+}
+
+/// Remove prefixes (u, r, b) and quotes around a string. This expects the given
+/// string to be a valid Python string representation, it doesn't do any
+/// validation.
+pub fn strip_quotes_and_prefixes(s: &str) -> &str {
+    match STRING_QUOTE_PREFIX_REGEX.captures(s) {
+        Some(caps) => match caps.name("raw") {
+            Some(m) => m.as_str(),
+            None => s,
+        },
+        None => s,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::str::{is_lower, is_upper, strip_quotes_and_prefixes};
+
+    #[test]
+    fn test_is_lower() {
+        assert!(is_lower("abc"));
+        assert!(is_lower("a_b_c"));
+        assert!(is_lower("a2c"));
+        assert!(!is_lower("aBc"));
+        assert!(!is_lower("ABC"));
+        assert!(!is_lower(""));
+        assert!(!is_lower("_"));
+    }
+
+    #[test]
+    fn test_is_upper() {
+        assert!(is_upper("ABC"));
+        assert!(is_upper("A_B_C"));
+        assert!(is_upper("A2C"));
+        assert!(!is_upper("aBc"));
+        assert!(!is_upper("abc"));
+        assert!(!is_upper(""));
+        assert!(!is_upper("_"));
+    }
+
+    #[test]
+    fn test_strip_quotes_and_prefixes() {
+        assert_eq!(strip_quotes_and_prefixes(r#"'a'"#), "a");
+        assert_eq!(strip_quotes_and_prefixes(r#"bur'a'"#), "a");
+        assert_eq!(strip_quotes_and_prefixes(r#"UrB'a'"#), "a");
+        assert_eq!(strip_quotes_and_prefixes(r#""a""#), "a");
+    }
+}

crates/ruff_python_stdlib/src/typing.rs

@@ -0,0 +1,199 @@
+use once_cell::sync::Lazy;
+use rustc_hash::FxHashSet;
+
+// See: https://pypi.org/project/typing-extensions/
+pub static TYPING_EXTENSIONS: Lazy<FxHashSet<&'static str>> = Lazy::new(|| {
+    FxHashSet::from_iter([
+        "Annotated",
+        "Any",
+        "AsyncContextManager",
+        "AsyncGenerator",
+        "AsyncIterable",
+        "AsyncIterator",
+        "Awaitable",
+        "ChainMap",
+        "ClassVar",
+        "Concatenate",
+        "ContextManager",
+        "Coroutine",
+        "Counter",
+        "DefaultDict",
+        "Deque",
+        "Final",
+        "Literal",
+        "LiteralString",
+        "NamedTuple",
+        "Never",
+        "NewType",
+        "NotRequired",
+        "OrderedDict",
+        "ParamSpec",
+        "ParamSpecArgs",
+        "ParamSpecKwargs",
+        "Protocol",
+        "Required",
+        "Self",
+        "TYPE_CHECKING",
+        "Text",
+        "Type",
+        "TypeAlias",
+        "TypeGuard",
+        "TypeVar",
+        "TypeVarTuple",
+        "TypedDict",
+        "Unpack",
+        "assert_never",
+        "assert_type",
+        "clear_overloads",
+        "final",
+        "get_type_hints",
+        "get_args",
+        "get_origin",
+        "get_overloads",
+        "is_typeddict",
+        "overload",
+        "override",
+        "reveal_type",
+        "runtime_checkable",
+    ])
+});
+
+// See: https://docs.python.org/3/library/typing.html
+pub const SUBSCRIPTS: &[&[&str]] = &[
+    // builtins
+    &["", "dict"],
+    &["", "frozenset"],
+    &["", "list"],
+    &["", "set"],
+    &["", "tuple"],
+    &["", "type"],
+    // `collections`
+    &["collections", "ChainMap"],
+    &["collections", "Counter"],
+    &["collections", "OrderedDict"],
+    &["collections", "defaultdict"],
+    &["collections", "deque"],
+    // `collections.abc`
+    &["collections", "abc", "AsyncGenerator"],
+    &["collections", "abc", "AsyncIterable"],
+    &["collections", "abc", "AsyncIterator"],
+    &["collections", "abc", "Awaitable"],
+    &["collections", "abc", "ByteString"],
+    &["collections", "abc", "Callable"],
+    &["collections", "abc", "Collection"],
+    &["collections", "abc", "Container"],
+    &["collections", "abc", "Coroutine"],
+    &["collections", "abc", "Generator"],
+    &["collections", "abc", "ItemsView"],
+    &["collections", "abc", "Iterable"],
+    &["collections", "abc", "Iterator"],
+    &["collections", "abc", "KeysView"],
+    &["collections", "abc", "Mapping"],
+    &["collections", "abc", "MappingView"],
+    &["collections", "abc", "MutableMapping"],
+    &["collections", "abc", "MutableSequence"],
+    &["collections", "abc", "MutableSet"],
+    &["collections", "abc", "Reversible"],
+    &["collections", "abc", "Sequence"],
+    &["collections", "abc", "Set"],
+    &["collections", "abc", "ValuesView"],
+    // `contextlib`
+    &["contextlib", "AbstractAsyncContextManager"],
+    &["contextlib", "AbstractContextManager"],
+    // `re`
+    &["re", "Match"],
+    &["re", "Pattern"],
+    // `typing`
+    &["typing", "AbstractSet"],
+    &["typing", "AsyncContextManager"],
+    &["typing", "AsyncGenerator"],
+    &["typing", "AsyncIterator"],
+    &["typing", "Awaitable"],
+    &["typing", "BinaryIO"],
+    &["typing", "ByteString"],
+    &["typing", "Callable"],
+    &["typing", "ChainMap"],
+    &["typing", "ClassVar"],
+    &["typing", "Collection"],
+    &["typing", "Concatenate"],
+    &["typing", "Container"],
+    &["typing", "ContextManager"],
+    &["typing", "Coroutine"],
+    &["typing", "Counter"],
+    &["typing", "DefaultDict"],
+    &["typing", "Deque"],
+    &["typing", "Dict"],
+    &["typing", "Final"],
+    &["typing", "FrozenSet"],
+    &["typing", "Generator"],
+    &["typing", "Generic"],
+    &["typing", "IO"],
+    &["typing", "ItemsView"],
+    &["typing", "Iterable"],
+    &["typing", "Iterator"],
+    &["typing", "KeysView"],
+    &["typing", "List"],
+    &["typing", "Mapping"],
+    &["typing", "Match"],
+    &["typing", "MutableMapping"],
+    &["typing", "MutableSequence"],
+    &["typing", "MutableSet"],
+    &["typing", "Optional"],
+    &["typing", "OrderedDict"],
+    &["typing", "Pattern"],
+    &["typing", "Reversible"],
+    &["typing", "Sequence"],
+    &["typing", "Set"],
+    &["typing", "TextIO"],
+    &["typing", "Tuple"],
+    &["typing", "Type"],
+    &["typing", "TypeGuard"],
+    &["typing", "Union"],
+    &["typing", "Unpack"],
+    &["typing", "ValuesView"],
+    // `typing.io`
+    &["typing", "io", "BinaryIO"],
+    &["typing", "io", "IO"],
+    &["typing", "io", "TextIO"],
+    // `typing.re`
+    &["typing", "re", "Match"],
+    &["typing", "re", "Pattern"],
+    // `typing_extensions`
+    &["typing_extensions", "AsyncContextManager"],
+    &["typing_extensions", "AsyncGenerator"],
+    &["typing_extensions", "AsyncIterable"],
+    &["typing_extensions", "AsyncIterator"],
+    &["typing_extensions", "Awaitable"],
+    &["typing_extensions", "ChainMap"],
+    &["typing_extensions", "ClassVar"],
+    &["typing_extensions", "Concatenate"],
+    &["typing_extensions", "ContextManager"],
+    &["typing_extensions", "Coroutine"],
+    &["typing_extensions", "Counter"],
+    &["typing_extensions", "DefaultDict"],
+    &["typing_extensions", "Deque"],
+    &["typing_extensions", "Type"],
+    // `weakref`
+    &["weakref", "WeakKeyDictionary"],
+    &["weakref", "WeakSet"],
+    &["weakref", "WeakValueDictionary"],
+];
+
+// See: https://docs.python.org/3/library/typing.html
+pub const PEP_593_SUBSCRIPTS: &[&[&str]] = &[
+    // `typing`
+    &["typing", "Annotated"],
+    // `typing_extensions`
+    &["typing_extensions", "Annotated"],
+];
+
+// See: https://peps.python.org/pep-0585/
+pub const PEP_585_BUILTINS_ELIGIBLE: &[&[&str]] = &[
+    &["typing", "Dict"],
+    &["typing", "FrozenSet"],
+    &["typing", "List"],
+    &["typing", "Set"],
+    &["typing", "Tuple"],
+    &["typing", "Type"],
+    &["typing_extensions", "Type"],
+];