Add known-standard-library for each Python version (#2491)

This commit is contained in:
Jonathan Plasse 2023-02-02 22:22:47 +01:00 committed by GitHub
parent b232c43824
commit ec8b827d26
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 1222 additions and 226 deletions

View file

@ -0,0 +1,91 @@
"""Vendored from [scripts/mkstdlibs.py in PyCQA/isort](https://github.com/PyCQA/isort/blob/e321a670d0fefdea0e04ed9d8d696434cf49bdec/scripts/mkstdlibs.py).
Only the generation of the file has been modified for use in this project.
"""
from pathlib import Path
from sphinx.ext.intersphinx import fetch_inventory
URL = "https://docs.python.org/{}/objects.inv"
PATH = Path("src") / "python" / "sys.rs"
VERSIONS = [
("3", "7"),
("3", "8"),
("3", "9"),
("3", "10"),
("3", "11"),
]
class FakeConfig: # noqa: D101
intersphinx_timeout = None
tls_verify = True
user_agent = ""
class FakeApp: # noqa: D101
srcdir = ""
config = FakeConfig()
with PATH.open("w") as f:
f.write(
"""\
//! This file is generated by `scripts/generate_known_standard_library.py`
use once_cell::sync::Lazy;
use rustc_hash::{FxHashMap, FxHashSet};
use crate::settings::types::PythonVersion;
// See: https://pycqa.github.io/isort/docs/configuration/options.html#known-standard-library
pub static KNOWN_STANDARD_LIBRARY: Lazy<FxHashMap<PythonVersion, FxHashSet<&'static str>>> =
Lazy::new(|| {
FxHashMap::from_iter([
""",
)
for version_info in VERSIONS:
version = ".".join(version_info)
url = URL.format(version)
invdata = fetch_inventory(FakeApp(), "", url)
# Any modules we want to enforce across Python versions stdlib can be included in set init
modules = {
"_ast",
"posixpath",
"ntpath",
"sre_constants",
"sre_parse",
"sre_compile",
"sre",
}
for module in invdata["py:module"]:
root, *_ = module.split(".")
if root not in ["__future__", "__main__"]:
modules.add(root)
f.write(
f"""\
(
PythonVersion::Py{"".join(version_info)},
FxHashSet::from_iter([
""",
)
for module in sorted(modules):
f.write(
f"""\
"{module}",
""",
)
f.write(
"""\
]),
),
""",
)
f.write(
"""\
])
});
""",
)

View file

@ -1,3 +1,8 @@
[project]
name = "scripts"
version = "0.0.1"
dependencies = ["sphinx"]
[tool.ruff]
select = ["ALL"]
ignore = [

View file

@ -1,8 +1,15 @@
//! This file is generated by `scripts/generate_known_standard_library.py`
use once_cell::sync::Lazy;
use rustc_hash::FxHashSet;
use rustc_hash::{FxHashMap, FxHashSet};
use crate::settings::types::PythonVersion;
// See: https://pycqa.github.io/isort/docs/configuration/options.html#known-standard-library
pub static KNOWN_STANDARD_LIBRARY: Lazy<FxHashSet<&'static str>> = Lazy::new(|| {
pub static KNOWN_STANDARD_LIBRARY: Lazy<FxHashMap<PythonVersion, FxHashSet<&'static str>>> =
Lazy::new(|| {
FxHashMap::from_iter([
(
PythonVersion::Py37,
FxHashSet::from_iter([
"_ast",
"_dummy_thread",
@ -67,7 +74,6 @@ pub static KNOWN_STANDARD_LIBRARY: Lazy<FxHashSet<&'static str>> = Lazy::new(||
"fileinput",
"fnmatch",
"formatter",
"fpectl",
"fractions",
"ftplib",
"functools",
@ -76,7 +82,6 @@ pub static KNOWN_STANDARD_LIBRARY: Lazy<FxHashSet<&'static str>> = Lazy::new(||
"getpass",
"gettext",
"glob",
"graphlib",
"grp",
"gzip",
"hashlib",
@ -194,6 +199,880 @@ pub static KNOWN_STANDARD_LIBRARY: Lazy<FxHashSet<&'static str>> = Lazy::new(||
"tkinter",
"token",
"tokenize",
"trace",
"traceback",
"tracemalloc",
"tty",
"turtle",
"turtledemo",
"types",
"typing",
"unicodedata",
"unittest",
"urllib",
"uu",
"uuid",
"venv",
"warnings",
"wave",
"weakref",
"webbrowser",
"winreg",
"winsound",
"wsgiref",
"xdrlib",
"xml",
"xmlrpc",
"zipapp",
"zipfile",
"zipimport",
"zlib",
]),
),
(
PythonVersion::Py38,
FxHashSet::from_iter([
"_ast",
"_dummy_thread",
"_thread",
"abc",
"aifc",
"argparse",
"array",
"ast",
"asynchat",
"asyncio",
"asyncore",
"atexit",
"audioop",
"base64",
"bdb",
"binascii",
"binhex",
"bisect",
"builtins",
"bz2",
"cProfile",
"calendar",
"cgi",
"cgitb",
"chunk",
"cmath",
"cmd",
"code",
"codecs",
"codeop",
"collections",
"colorsys",
"compileall",
"concurrent",
"configparser",
"contextlib",
"contextvars",
"copy",
"copyreg",
"crypt",
"csv",
"ctypes",
"curses",
"dataclasses",
"datetime",
"dbm",
"decimal",
"difflib",
"dis",
"distutils",
"doctest",
"dummy_threading",
"email",
"encodings",
"ensurepip",
"enum",
"errno",
"faulthandler",
"fcntl",
"filecmp",
"fileinput",
"fnmatch",
"formatter",
"fractions",
"ftplib",
"functools",
"gc",
"getopt",
"getpass",
"gettext",
"glob",
"grp",
"gzip",
"hashlib",
"heapq",
"hmac",
"html",
"http",
"imaplib",
"imghdr",
"imp",
"importlib",
"inspect",
"io",
"ipaddress",
"itertools",
"json",
"keyword",
"lib2to3",
"linecache",
"locale",
"logging",
"lzma",
"mailbox",
"mailcap",
"marshal",
"math",
"mimetypes",
"mmap",
"modulefinder",
"msilib",
"msvcrt",
"multiprocessing",
"netrc",
"nis",
"nntplib",
"ntpath",
"numbers",
"operator",
"optparse",
"os",
"ossaudiodev",
"parser",
"pathlib",
"pdb",
"pickle",
"pickletools",
"pipes",
"pkgutil",
"platform",
"plistlib",
"poplib",
"posix",
"posixpath",
"pprint",
"profile",
"pstats",
"pty",
"pwd",
"py_compile",
"pyclbr",
"pydoc",
"queue",
"quopri",
"random",
"re",
"readline",
"reprlib",
"resource",
"rlcompleter",
"runpy",
"sched",
"secrets",
"select",
"selectors",
"shelve",
"shlex",
"shutil",
"signal",
"site",
"smtpd",
"smtplib",
"sndhdr",
"socket",
"socketserver",
"spwd",
"sqlite3",
"sre",
"sre_compile",
"sre_constants",
"sre_parse",
"ssl",
"stat",
"statistics",
"string",
"stringprep",
"struct",
"subprocess",
"sunau",
"symbol",
"symtable",
"sys",
"sysconfig",
"syslog",
"tabnanny",
"tarfile",
"telnetlib",
"tempfile",
"termios",
"test",
"textwrap",
"threading",
"time",
"timeit",
"tkinter",
"token",
"tokenize",
"trace",
"traceback",
"tracemalloc",
"tty",
"turtle",
"turtledemo",
"types",
"typing",
"unicodedata",
"unittest",
"urllib",
"uu",
"uuid",
"venv",
"warnings",
"wave",
"weakref",
"webbrowser",
"winreg",
"winsound",
"wsgiref",
"xdrlib",
"xml",
"xmlrpc",
"zipapp",
"zipfile",
"zipimport",
"zlib",
]),
),
(
PythonVersion::Py39,
FxHashSet::from_iter([
"_ast",
"_thread",
"abc",
"aifc",
"argparse",
"array",
"ast",
"asynchat",
"asyncio",
"asyncore",
"atexit",
"audioop",
"base64",
"bdb",
"binascii",
"binhex",
"bisect",
"builtins",
"bz2",
"cProfile",
"calendar",
"cgi",
"cgitb",
"chunk",
"cmath",
"cmd",
"code",
"codecs",
"codeop",
"collections",
"colorsys",
"compileall",
"concurrent",
"configparser",
"contextlib",
"contextvars",
"copy",
"copyreg",
"crypt",
"csv",
"ctypes",
"curses",
"dataclasses",
"datetime",
"dbm",
"decimal",
"difflib",
"dis",
"distutils",
"doctest",
"email",
"encodings",
"ensurepip",
"enum",
"errno",
"faulthandler",
"fcntl",
"filecmp",
"fileinput",
"fnmatch",
"formatter",
"fractions",
"ftplib",
"functools",
"gc",
"getopt",
"getpass",
"gettext",
"glob",
"graphlib",
"grp",
"gzip",
"hashlib",
"heapq",
"hmac",
"html",
"http",
"imaplib",
"imghdr",
"imp",
"importlib",
"inspect",
"io",
"ipaddress",
"itertools",
"json",
"keyword",
"lib2to3",
"linecache",
"locale",
"logging",
"lzma",
"mailbox",
"mailcap",
"marshal",
"math",
"mimetypes",
"mmap",
"modulefinder",
"msilib",
"msvcrt",
"multiprocessing",
"netrc",
"nis",
"nntplib",
"ntpath",
"numbers",
"operator",
"optparse",
"os",
"ossaudiodev",
"parser",
"pathlib",
"pdb",
"pickle",
"pickletools",
"pipes",
"pkgutil",
"platform",
"plistlib",
"poplib",
"posix",
"posixpath",
"pprint",
"profile",
"pstats",
"pty",
"pwd",
"py_compile",
"pyclbr",
"pydoc",
"queue",
"quopri",
"random",
"re",
"readline",
"reprlib",
"resource",
"rlcompleter",
"runpy",
"sched",
"secrets",
"select",
"selectors",
"shelve",
"shlex",
"shutil",
"signal",
"site",
"smtpd",
"smtplib",
"sndhdr",
"socket",
"socketserver",
"spwd",
"sqlite3",
"sre",
"sre_compile",
"sre_constants",
"sre_parse",
"ssl",
"stat",
"statistics",
"string",
"stringprep",
"struct",
"subprocess",
"sunau",
"symbol",
"symtable",
"sys",
"sysconfig",
"syslog",
"tabnanny",
"tarfile",
"telnetlib",
"tempfile",
"termios",
"test",
"textwrap",
"threading",
"time",
"timeit",
"tkinter",
"token",
"tokenize",
"trace",
"traceback",
"tracemalloc",
"tty",
"turtle",
"turtledemo",
"types",
"typing",
"unicodedata",
"unittest",
"urllib",
"uu",
"uuid",
"venv",
"warnings",
"wave",
"weakref",
"webbrowser",
"winreg",
"winsound",
"wsgiref",
"xdrlib",
"xml",
"xmlrpc",
"zipapp",
"zipfile",
"zipimport",
"zlib",
"zoneinfo",
]),
),
(
PythonVersion::Py310,
FxHashSet::from_iter([
"_ast",
"_thread",
"abc",
"aifc",
"argparse",
"array",
"ast",
"asynchat",
"asyncio",
"asyncore",
"atexit",
"audioop",
"base64",
"bdb",
"binascii",
"binhex",
"bisect",
"builtins",
"bz2",
"cProfile",
"calendar",
"cgi",
"cgitb",
"chunk",
"cmath",
"cmd",
"code",
"codecs",
"codeop",
"collections",
"colorsys",
"compileall",
"concurrent",
"configparser",
"contextlib",
"contextvars",
"copy",
"copyreg",
"crypt",
"csv",
"ctypes",
"curses",
"dataclasses",
"datetime",
"dbm",
"decimal",
"difflib",
"dis",
"distutils",
"doctest",
"email",
"encodings",
"ensurepip",
"enum",
"errno",
"faulthandler",
"fcntl",
"filecmp",
"fileinput",
"fnmatch",
"fractions",
"ftplib",
"functools",
"gc",
"getopt",
"getpass",
"gettext",
"glob",
"graphlib",
"grp",
"gzip",
"hashlib",
"heapq",
"hmac",
"html",
"http",
"idlelib",
"imaplib",
"imghdr",
"imp",
"importlib",
"inspect",
"io",
"ipaddress",
"itertools",
"json",
"keyword",
"lib2to3",
"linecache",
"locale",
"logging",
"lzma",
"mailbox",
"mailcap",
"marshal",
"math",
"mimetypes",
"mmap",
"modulefinder",
"msilib",
"msvcrt",
"multiprocessing",
"netrc",
"nis",
"nntplib",
"ntpath",
"numbers",
"operator",
"optparse",
"os",
"ossaudiodev",
"pathlib",
"pdb",
"pickle",
"pickletools",
"pipes",
"pkgutil",
"platform",
"plistlib",
"poplib",
"posix",
"posixpath",
"pprint",
"profile",
"pstats",
"pty",
"pwd",
"py_compile",
"pyclbr",
"pydoc",
"queue",
"quopri",
"random",
"re",
"readline",
"reprlib",
"resource",
"rlcompleter",
"runpy",
"sched",
"secrets",
"select",
"selectors",
"shelve",
"shlex",
"shutil",
"signal",
"site",
"smtpd",
"smtplib",
"sndhdr",
"socket",
"socketserver",
"spwd",
"sqlite3",
"sre",
"sre_compile",
"sre_constants",
"sre_parse",
"ssl",
"stat",
"statistics",
"string",
"stringprep",
"struct",
"subprocess",
"sunau",
"symtable",
"sys",
"sysconfig",
"syslog",
"tabnanny",
"tarfile",
"telnetlib",
"tempfile",
"termios",
"test",
"textwrap",
"threading",
"time",
"timeit",
"tkinter",
"token",
"tokenize",
"trace",
"traceback",
"tracemalloc",
"tty",
"turtle",
"turtledemo",
"types",
"typing",
"unicodedata",
"unittest",
"urllib",
"uu",
"uuid",
"venv",
"warnings",
"wave",
"weakref",
"webbrowser",
"winreg",
"winsound",
"wsgiref",
"xdrlib",
"xml",
"xmlrpc",
"zipapp",
"zipfile",
"zipimport",
"zlib",
"zoneinfo",
]),
),
(
PythonVersion::Py311,
FxHashSet::from_iter([
"_ast",
"_thread",
"abc",
"aifc",
"argparse",
"array",
"ast",
"asynchat",
"asyncio",
"asyncore",
"atexit",
"audioop",
"base64",
"bdb",
"binascii",
"bisect",
"builtins",
"bz2",
"cProfile",
"calendar",
"cgi",
"cgitb",
"chunk",
"cmath",
"cmd",
"code",
"codecs",
"codeop",
"collections",
"colorsys",
"compileall",
"concurrent",
"configparser",
"contextlib",
"contextvars",
"copy",
"copyreg",
"crypt",
"csv",
"ctypes",
"curses",
"dataclasses",
"datetime",
"dbm",
"decimal",
"difflib",
"dis",
"distutils",
"doctest",
"email",
"encodings",
"ensurepip",
"enum",
"errno",
"faulthandler",
"fcntl",
"filecmp",
"fileinput",
"fnmatch",
"fractions",
"ftplib",
"functools",
"gc",
"getopt",
"getpass",
"gettext",
"glob",
"graphlib",
"grp",
"gzip",
"hashlib",
"heapq",
"hmac",
"html",
"http",
"idlelib",
"imaplib",
"imghdr",
"imp",
"importlib",
"inspect",
"io",
"ipaddress",
"itertools",
"json",
"keyword",
"lib2to3",
"linecache",
"locale",
"logging",
"lzma",
"mailbox",
"mailcap",
"marshal",
"math",
"mimetypes",
"mmap",
"modulefinder",
"msilib",
"msvcrt",
"multiprocessing",
"netrc",
"nis",
"nntplib",
"ntpath",
"numbers",
"operator",
"optparse",
"os",
"ossaudiodev",
"pathlib",
"pdb",
"pickle",
"pickletools",
"pipes",
"pkgutil",
"platform",
"plistlib",
"poplib",
"posix",
"posixpath",
"pprint",
"profile",
"pstats",
"pty",
"pwd",
"py_compile",
"pyclbr",
"pydoc",
"queue",
"quopri",
"random",
"re",
"readline",
"reprlib",
"resource",
"rlcompleter",
"runpy",
"sched",
"secrets",
"select",
"selectors",
"shelve",
"shlex",
"shutil",
"signal",
"site",
"smtpd",
"smtplib",
"sndhdr",
"socket",
"socketserver",
"spwd",
"sqlite3",
"sre",
"sre_compile",
"sre_constants",
"sre_parse",
"ssl",
"stat",
"statistics",
"string",
"stringprep",
"struct",
"subprocess",
"sunau",
"symtable",
"sys",
"sysconfig",
"syslog",
"tabnanny",
"tarfile",
"telnetlib",
"tempfile",
"termios",
"test",
"textwrap",
"threading",
"time",
"timeit",
"tkinter",
"token",
"tokenize",
"tomllib",
"trace",
"traceback",
@ -224,5 +1103,7 @@ pub static KNOWN_STANDARD_LIBRARY: Lazy<FxHashSet<&'static str>> = Lazy::new(||
"zipimport",
"zlib",
"zoneinfo",
]),
),
])
});

View file

@ -166,6 +166,7 @@ pub fn typing_only_runtime_import(
&settings.isort.known_first_party,
&settings.isort.known_third_party,
&settings.isort.extra_standard_library,
settings.target_version,
) {
ImportType::LocalFolder | ImportType::FirstParty => Some(Diagnostic::new(
TypingOnlyFirstPartyImport {

View file

@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
use super::types::{ImportBlock, Importable};
use crate::python::sys::KNOWN_STANDARD_LIBRARY;
use crate::settings::types::PythonVersion;
#[derive(
Debug, PartialOrd, Ord, PartialEq, Eq, Clone, Serialize, Deserialize, JsonSchema, Hash,
@ -34,6 +35,7 @@ enum Reason<'a> {
NoMatch,
}
#[allow(clippy::too_many_arguments)]
pub fn categorize(
module_base: &str,
level: Option<&usize>,
@ -42,6 +44,7 @@ pub fn categorize(
known_first_party: &BTreeSet<String>,
known_third_party: &BTreeSet<String>,
extra_standard_library: &BTreeSet<String>,
target_version: PythonVersion,
) -> ImportType {
let (import_type, reason) = {
if level.map_or(false, |level| *level > 0) {
@ -54,7 +57,11 @@ pub fn categorize(
(ImportType::StandardLibrary, Reason::ExtraStandardLibrary)
} else if module_base == "__future__" {
(ImportType::Future, Reason::Future)
} else if KNOWN_STANDARD_LIBRARY.contains(module_base) {
} else if KNOWN_STANDARD_LIBRARY
.get(&target_version)
.unwrap()
.contains(module_base)
{
(ImportType::StandardLibrary, Reason::KnownStandardLibrary)
} else if same_package(package, module_base) {
(ImportType::FirstParty, Reason::SamePackage)
@ -98,6 +105,7 @@ pub fn categorize_imports<'a>(
known_first_party: &BTreeSet<String>,
known_third_party: &BTreeSet<String>,
extra_standard_library: &BTreeSet<String>,
target_version: PythonVersion,
) -> BTreeMap<ImportType, ImportBlock<'a>> {
let mut block_by_type: BTreeMap<ImportType, ImportBlock> = BTreeMap::default();
// Categorize `StmtKind::Import`.
@ -110,6 +118,7 @@ pub fn categorize_imports<'a>(
known_first_party,
known_third_party,
extra_standard_library,
target_version,
);
block_by_type
.entry(import_type)
@ -127,6 +136,7 @@ pub fn categorize_imports<'a>(
known_first_party,
known_third_party,
extra_standard_library,
target_version,
);
block_by_type
.entry(classification)
@ -144,6 +154,7 @@ pub fn categorize_imports<'a>(
known_first_party,
known_third_party,
extra_standard_library,
target_version,
);
block_by_type
.entry(classification)
@ -161,6 +172,7 @@ pub fn categorize_imports<'a>(
known_first_party,
known_third_party,
extra_standard_library,
target_version,
);
block_by_type
.entry(classification)

View file

@ -18,6 +18,7 @@ use types::EitherImport::{Import, ImportFrom};
use types::{AliasData, CommentSet, EitherImport, OrderedImportBlock, TrailingComma};
use crate::rules::isort::types::ImportBlock;
use crate::settings::types::PythonVersion;
use crate::source_code::{Locator, Stylist};
mod annotate;
@ -132,6 +133,7 @@ pub fn format_imports(
no_lines_before: &BTreeSet<ImportType>,
lines_after_imports: isize,
forced_separate: &[String],
target_version: PythonVersion,
) -> String {
let trailer = &block.trailer;
let block = annotate_imports(&block.imports, comments, locator, split_on_trailing_comma);
@ -162,6 +164,7 @@ pub fn format_imports(
constants,
variables,
no_lines_before,
target_version,
);
if !block_output.is_empty() && !output.is_empty() {
@ -218,6 +221,7 @@ fn format_import_block(
constants: &BTreeSet<String>,
variables: &BTreeSet<String>,
no_lines_before: &BTreeSet<ImportType>,
target_version: PythonVersion,
) -> String {
// Categorize by type (e.g., first-party vs. third-party).
let block_by_type = categorize_imports(
@ -227,6 +231,7 @@ fn format_import_block(
known_first_party,
known_third_party,
extra_standard_library,
target_version,
);
let mut output = String::new();

View file

@ -120,6 +120,7 @@ pub fn organize_imports(
&settings.isort.no_lines_before,
settings.isort.lines_after_imports,
&settings.isort.forced_separate,
settings.target_version,
);
// Expand the span the entire range, including leading and trailing space.