[ty] Add an evaluation for completions

This is still early days, but I hope the framework introduced here makes it very easy to add new truth data. Truth data should be seen as a form of regression test for non-ideal ranking of completion suggestions. I think it would help to read `crates/ty_completion_eval/README.md` first to get an idea of what you're reviewing.
2025-10-25 17:38:19 +00:00 · 2025-09-25 14:53:34 -04:00 · 2025-09-25 14:53:34 -04:00 · 3771f1567c
commit 3771f1567c
parent 6b94e620fe
63 changed files with 1213 additions and 4 deletions
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@ -707,6 +707,24 @@ jobs:
      - run: cargo binstall --no-confirm cargo-shear
      - run: cargo shear

+  ty-completion-evaluation:
+    name: "ty completion evaluation"
+    runs-on: depot-ubuntu-22.04-16
+    needs: determine_changes
+    if: ${{ needs.determine_changes.outputs.ty == 'true' || github.ref == 'refs/heads/main' }}
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+      - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1
+      - name: "Install Rust toolchain"
+        run: rustup show
+      - name: "Run ty completion evaluation"
+        run: cargo run --release --package ty_completion_eval -- all --threshold 0.1 --tasks /tmp/completion-evaluation-tasks.csv
+      - name: "Ensure there are no changes"
+        run: diff ./crates/ty_completion_eval/completion-evaluation-tasks.csv /tmp/completion-evaluation-tasks.csv
+
  python-package:
    name: "python package"
    runs-on: ubuntu-latest
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -16,7 +16,8 @@ exclude: |
    crates/ruff_python_formatter/resources/.*|
    crates/ruff_python_formatter/tests/snapshots/.*|
    crates/ruff_python_resolver/resources/.*|
-    crates/ruff_python_resolver/tests/snapshots/.*
+    crates/ruff_python_resolver/tests/snapshots/.*|
+    crates/ty_completion_eval/truth/.*
  )$

 repos:
--- a/Cargo.lock
+++ b/Cargo.lock
@ -818,6 +818,27 @@ dependencies = [
 "typenum",
 ]

+[[package]]
+name = "csv"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "ctrlc"
 version = "3.5.0"
@ -4228,6 +4249,26 @@ dependencies = [
 "ty_python_semantic",
 ]

+[[package]]
+name = "ty_completion_eval"
+version = "0.0.0"
+dependencies = [
+ "anyhow",
+ "bstr",
+ "clap",
+ "csv",
+ "regex",
+ "ruff_db",
+ "ruff_text_size",
+ "serde",
+ "tempfile",
+ "toml",
+ "ty_ide",
+ "ty_project",
+ "ty_python_semantic",
+ "walkdir",
+]
+
 [[package]]
 name = "ty_ide"
 version = "0.0.0"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -43,6 +43,7 @@ ruff_workspace = { path = "crates/ruff_workspace" }

 ty = { path = "crates/ty" }
 ty_combine = { path = "crates/ty_combine" }
+ty_completion_eval = { path = "crates/ty_completion_eval" }
 ty_ide = { path = "crates/ty_ide" }
 ty_project = { path = "crates/ty_project", default-features = false }
 ty_python_semantic = { path = "crates/ty_python_semantic" }
@ -69,6 +70,7 @@ camino = { version = "1.1.7" }
 clap = { version = "4.5.3", features = ["derive"] }
 clap_complete_command = { version = "0.6.0" }
 clearscreen = { version = "4.0.0" }
+csv = { version = "1.3.1" }
 divan = { package = "codspeed-divan-compat", version = "3.0.2" }
 codspeed-criterion-compat = { version = "3.0.2", default-features = false }
 colored = { version = "3.0.0" }
@ -203,7 +205,7 @@ wild = { version = "2" }
 zip = { version = "0.6.6", default-features = false }

 [workspace.metadata.cargo-shear]
-ignored = ["getrandom", "ruff_options_metadata", "uuid", "get-size2"]
+ignored = ["getrandom", "ruff_options_metadata", "uuid", "get-size2", "ty_completion_eval"]


 [workspace.lints.rust]
--- a/crates/ty_completion_eval/Cargo.toml
+++ b/crates/ty_completion_eval/Cargo.toml
@ -0,0 +1,32 @@
+[package]
+name = "ty_completion_eval"
+version = "0.0.0"
+publish = false
+authors = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+homepage = { workspace = true }
+documentation = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+
+[dependencies]
+ruff_db = { workspace = true, features = ["os"] }
+ruff_text_size = { workspace = true }
+
+ty_ide = { workspace = true }
+ty_project = { workspace = true }
+ty_python_semantic = { workspace = true }
+
+anyhow = { workspace = true }
+bstr = { workspace = true }
+clap = { workspace = true, features = ["wrap_help", "string", "env"] }
+csv = { workspace = true }
+regex = { workspace = true }
+serde = { workspace = true }
+tempfile = { workspace = true }
+toml = { workspace = true }
+walkdir = { workspace = true }
+
+[lints]
+workspace = true
--- a/crates/ty_completion_eval/README.md
+++ b/crates/ty_completion_eval/README.md
@ -0,0 +1,143 @@
+This directory contains a framework for evaluating completion suggestions
+returned by the ty LSP.
+
+# Running an evaluation
+
+To run a full evaluation, run the `ty_completion_eval` crate with the
+`all` command from the root of this repository:
+
+```console
+cargo run --release --package ty_completion_eval -- all
+```
+
+The output should look like this:
+
+```text
+    Finished `release` profile [optimized] target(s) in 0.09s
+     Running `target/release/ty_completion_eval all`
+mean reciprocal rank: 0.20409790112917506
+MRR exceeds threshold of 0.001
+```
+
+If you want to look at the results of each individual evaluation task,
+you can ask the evaluation to write CSV data that contains the rank of
+the expected answer in each completion request:
+
+```console
+cargo r -r -p ty_completion_eval -- all --tasks ./crates/ty_completion_eval/completion-evaluation-tasks.csv
+```
+
+To debug a _specific_ task and look at the actual results, use the `show-one`
+command:
+
+```console
+cargo r -q -p ty_completion_eval show-one higher-level-symbols-preferred --index 1
+```
+
+(The `--index` flag is only needed if there are multiple `<CURSOR>` directives in the same file.)
+
+Has output that should look like this:
+
+```text
+ZQZQZQ_SOMETHING_IMPORTANT (*, 1/31)
+__annotations__
+__class__
+__delattr__
+__dict__
+__dir__
+__doc__
+__eq__
+__file__
+__format__
+__getattr__
+__getattribute__
+__getstate__
+__hash__
+__init__
+__init_subclass__
+__loader__
+__module__
+__name__
+__ne__
+__new__
+__package__
+__path__
+__reduce__
+__reduce_ex__
+__repr__
+__setattr__
+__sizeof__
+__spec__
+__str__
+__subclasshook__
+-----
+found 31 completions
+```
+
+The expected answer is marked with a `*`. The higher the rank, the better. In this example, the
+rank is perfect. Note that the expected answer may not always appear in the completion results!
+(Which is considered the worst possible outcome by this evaluation framework.)
+
+# Evaluation model
+
+This evaluation is based on [mean reciprocal rank] (MRR). That is, it assumes
+that for every evaluation task (i.e., a single completion request) there is
+precisely one correct answer. The higher the correct answer appears in each
+completion request, the better. The mean reciprocal rank is computed as the
+average of `1/rank` across all evaluation tasks. The higher the mean reciprocal
+rank, the better.
+
+The evaluation starts by preparing its truth data, which is contained in the `./truth` directory.
+Within `./truth` is a list of Python projects. Every project contains one or more `<CURSOR>`
+directives. Each `<CURSOR>` directive corresponds to an instruction to initiate a completion
+request at that position. For example:
+
+```python
+class Foo:
+    def frobnicate(self): pass
+
+foo = Foo()
+foo.frob<CURSOR: frobnicate>
+```
+
+The above example says that completions should be requested immediately after `foo.frob`
+_and_ that the expected answer is `frobnicate`.
+
+When testing auto-import, one should also include the module in the expected answer.
+For example:
+
+```python
+RegexFl<CURSOR: re.RegexFlag>
+```
+
+Settings for completion requests can be configured via a `completion.toml` file within
+each Python project directory.
+
+When an evaluation is run, the truth data is copied to a temporary directory.
+`uv sync` is then run within each directory to prepare it.
+
+# Continuous Integration
+
+At time of writing (2025-10-07), an evaluation is run in CI. CI will fail if the MRR is
+below a set threshold. When this occurs, it means that the evaluation's results have likely
+gotten worse in some measurable way. Ideally, the way to fix this would be to fix whatever
+regression occurred in ranking. One can follow the steps above to run an evaluation and
+emit the individual task results in CSV format. This difference between this CSV data and
+whatever is committed at `./crates/ty_completion_eval/completion-evaluation-tasks.csv` should
+point to where the regression occurs.
+
+If the change is not a regression or is otherwise expected, then the MRR threshold can be
+lowered. This requires changing how `ty_completion_eval` is executed within CI.
+
+CI will also fail if the individual task results have changed.
+To make CI pass, you can just re-run the evaluation locally and commit the results:
+
+```console
+cargo r -r -p ty_completion_eval -- all --tasks ./crates/ty_completion_eval/completion-evaluation-tasks.csv
+```
+
+CI fails in this case because it would be best to scrutinize the differences here.
+It's possible that the ranking has improved in some measurable way, for example.
+(Think of this as if it were a snapshot test.)
+
+[mean reciprocal rank]: https://en.wikipedia.org/wiki/Mean_reciprocal_rank
--- a/crates/ty_completion_eval/completion-evaluation-tasks.csv
+++ b/crates/ty_completion_eval/completion-evaluation-tasks.csv
@ -0,0 +1,17 @@
+name,file,index,rank
+higher-level-symbols-preferred,main.py,0,
+higher-level-symbols-preferred,main.py,1,1
+import-deprioritizes-dunder,main.py,0,195
+import-deprioritizes-sunder,main.py,0,195
+internal-typeshed-hidden,main.py,0,43
+numpy-array,main.py,0,
+numpy-array,main.py,1,32
+object-attr-instance-methods,main.py,0,7
+object-attr-instance-methods,main.py,1,1
+raise-uses-base-exception,main.py,0,42
+scope-existing-over-new-import,main.py,0,495
+scope-prioritize-closer,main.py,0,152
+scope-simple-long-identifier,main.py,0,140
+ty-extensions-lower-stdlib,main.py,0,142
+type-var-typing-over-ast,main.py,0,65
+type-var-typing-over-ast,main.py,1,353
--- a/crates/ty_completion_eval/src/main.rs
+++ b/crates/ty_completion_eval/src/main.rs
@ -0,0 +1,618 @@
+/*!
+A simple command line tool for running a completion evaluation.
+
+See `crates/ty_completion_eval/README.md` for examples and more docs.
+*/
+
+use std::io::Write;
+use std::process::ExitCode;
+use std::sync::LazyLock;
+
+use anyhow::{Context, anyhow};
+use clap::Parser;
+use regex::bytes::Regex;
+
+use ruff_db::files::system_path_to_file;
+use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf};
+use ty_ide::Completion;
+use ty_project::{ProjectDatabase, ProjectMetadata};
+use ty_python_semantic::ModuleName;
+
+#[derive(Debug, clap::Parser)]
+#[command(
+    author,
+    name = "ty_completion_eval",
+    about = "Run a information retrieval evaluation on ty-powered completions."
+)]
+struct Cli {
+    #[command(subcommand)]
+    command: Command,
+}
+
+#[derive(Debug, clap::Subcommand)]
+enum Command {
+    /// Run an evaluation on all tasks.
+    All(AllCommand),
+    /// Show the completions for a single task.
+    ///
+    /// This is useful for debugging one single completion task. For
+    /// example, let's say you make a change to a ranking heuristic and
+    /// everything looks good except for a few tasks where the rank for
+    /// the expected answer regressed. Just use this command to run a
+    /// specific task and you'll get the actual completions for that
+    /// task printed to stdout.
+    ///
+    /// If the expected answer is found in the completion list, then
+    /// it is marked with an `*` along with its rank.
+    ShowOne(ShowOneCommand),
+}
+
+#[derive(Debug, clap::Parser)]
+struct AllCommand {
+    /// The mean reciprocal rank threshold that the evaluation must
+    /// meet or exceed in order for the evaluation to pass.
+    #[arg(
+        long,
+        help = "The mean reciprocal rank threshold.",
+        value_name = "FLOAT",
+        default_value_t = 0.001
+    )]
+    threshold: f64,
+    /// If given, a CSV file of the results for each individual task
+    /// is written to the path given.
+    #[arg(
+        long,
+        help = "When provided, write individual task results in CSV format.",
+        value_name = "FILE"
+    )]
+    tasks: Option<String>,
+    /// Whether to keep the temporary evaluation directory around
+    /// after finishing or not. Keeping it around is useful for
+    /// debugging when something has gone wrong.
+    #[arg(
+        long,
+        help = "Whether to keep the temporary evaluation directory around or not."
+    )]
+    keep_tmp_dir: bool,
+}
+
+#[derive(Debug, clap::Parser)]
+struct ShowOneCommand {
+    /// The name of one or more completion tasks to run in isolation.
+    ///
+    /// The name corresponds to the name of a directory in
+    /// `./crates/ty_completion_eval/truth/`.
+    #[arg(help = "The task name to run.", value_name = "TASK_NAME")]
+    task_name: String,
+    /// The name of the file, relative to the root of the
+    /// Python project, that contains one or more completion
+    /// tasks to run in isolation.
+    #[arg(long, help = "The file name to run.", value_name = "FILE_NAME")]
+    file_name: Option<String>,
+    /// The index of the cursor directive within `file_name`
+    /// to select.
+    #[arg(
+        long,
+        help = "The index of the cursor directive to run.",
+        value_name = "INDEX"
+    )]
+    index: Option<usize>,
+    /// Whether to keep the temporary evaluation directory around
+    /// after finishing or not. Keeping it around is useful for
+    /// debugging when something has gone wrong.
+    #[arg(
+        long,
+        help = "Whether to keep the temporary evaluation directory around or not."
+    )]
+    keep_tmp_dir: bool,
+}
+
+impl ShowOneCommand {
+    fn matches_source_task(&self, task_source: &TaskSource) -> bool {
+        self.task_name == task_source.name
+    }
+
+    fn matches_task(&self, task: &Task) -> bool {
+        self.task_name == task.name
+            && self
+                .file_name
+                .as_ref()
+                .is_some_and(|name| name == task.cursor_name())
+            && self.index.is_some_and(|index| index == task.cursor.index)
+    }
+}
+
+fn main() -> anyhow::Result<ExitCode> {
+    let args = Cli::parse();
+
+    // The base path to which all CLI arguments are relative to.
+    let cwd = {
+        let cwd = std::env::current_dir().context("Failed to get the current working directory")?;
+        SystemPathBuf::from_path_buf(cwd).map_err(|path| {
+            anyhow!(
+                "The current working directory `{}` contains non-Unicode characters. \
+                 ty only supports Unicode paths.",
+                path.display()
+            )
+        })?
+    };
+    // Where we store our truth data.
+    let truth = cwd.join("crates").join("ty_completion_eval").join("truth");
+    anyhow::ensure!(
+        truth.as_std_path().exists(),
+        "{truth} does not exist: ty's completion evaluation must be run from the root \
+         of the ruff repository",
+        truth = truth.as_std_path().display(),
+    );
+
+    // The temporary directory at which we copy our truth
+    // data to. We do this because we can't use the truth
+    // data as-is with its `<CURSOR>` annotations (and perhaps
+    // any other future annotations we add).
+    let mut tmp_eval_dir = tempfile::Builder::new()
+        .prefix("ty-completion-eval-")
+        .tempdir()
+        .context("Failed to create temporary directory")?;
+    let tmp_eval_path = SystemPath::from_std_path(tmp_eval_dir.path())
+        .ok_or_else(|| {
+            anyhow::anyhow!(
+                "Temporary directory path is not valid UTF-8: {}",
+                tmp_eval_dir.path().display()
+            )
+        })?
+        .to_path_buf();
+
+    let sources = TaskSource::all(&truth)?;
+    match args.command {
+        Command::ShowOne(ref cmd) => {
+            tmp_eval_dir.disable_cleanup(cmd.keep_tmp_dir);
+
+            let Some(source) = sources
+                .iter()
+                .find(|source| cmd.matches_source_task(source))
+            else {
+                anyhow::bail!("could not find task named `{}`", cmd.task_name);
+            };
+            let tasks = source.to_tasks(&tmp_eval_path)?;
+            let matching: Vec<&Task> = tasks.iter().filter(|task| cmd.matches_task(task)).collect();
+            anyhow::ensure!(
+                !matching.is_empty(),
+                "could not find any tasks matching the given criteria",
+            );
+            anyhow::ensure!(
+                matching.len() < 2,
+                "found more than one task matching the given criteria",
+            );
+            let task = &matching[0];
+            let completions = task.completions()?;
+
+            let mut stdout = std::io::stdout().lock();
+            for (i, c) in completions.iter().enumerate() {
+                write!(stdout, "{}", c.name.as_str())?;
+                if let Some(module_name) = c.module_name {
+                    write!(stdout, " (module: {module_name})")?;
+                }
+                if task.cursor.answer.matches(c) {
+                    write!(stdout, " (*, {}/{})", i + 1, completions.len())?;
+                }
+                writeln!(stdout)?;
+            }
+            writeln!(stdout, "-----")?;
+            writeln!(stdout, "found {} completions", completions.len())?;
+            Ok(ExitCode::SUCCESS)
+        }
+        Command::All(AllCommand {
+            threshold,
+            tasks,
+            keep_tmp_dir,
+        }) => {
+            tmp_eval_dir.disable_cleanup(keep_tmp_dir);
+
+            let mut precision_sum = 0.0;
+            let mut task_count = 0.0f64;
+            let mut results_wtr = None;
+            if let Some(ref tasks) = tasks {
+                let mut wtr = csv::Writer::from_path(SystemPath::new(tasks))?;
+                wtr.serialize(("name", "file", "index", "rank"))?;
+                results_wtr = Some(wtr);
+            }
+            for source in &sources {
+                for task in source.to_tasks(&tmp_eval_path)? {
+                    task_count += 1.0;
+
+                    let completions = task.completions()?;
+                    let rank = task.rank(&completions)?;
+                    precision_sum += rank.map(|rank| 1.0 / f64::from(rank)).unwrap_or(0.0);
+                    if let Some(ref mut wtr) = results_wtr {
+                        wtr.serialize((&task.name, &task.cursor_name(), task.cursor.index, rank))?;
+                    }
+                }
+            }
+            let mrr = precision_sum / task_count;
+            if let Some(ref mut wtr) = results_wtr {
+                wtr.flush()?;
+            }
+
+            let mut out = std::io::stdout().lock();
+            writeln!(out, "mean reciprocal rank: {mrr:.4}")?;
+            if mrr < threshold {
+                writeln!(
+                    out,
+                    "Failure: MRR does not exceed minimum threshold of {threshold}"
+                )?;
+                Ok(ExitCode::FAILURE)
+            } else {
+                writeln!(out, "Success: MRR exceeds minimum threshold of {threshold}")?;
+                Ok(ExitCode::SUCCESS)
+            }
+        }
+    }
+}
+
+/// A single completion task.
+///
+/// The task is oriented in such a way that we have a single "cursor"
+/// position in a Python project. This allows us to ask for completions
+/// at that position.
+struct Task {
+    db: ProjectDatabase,
+    dir: SystemPathBuf,
+    name: String,
+    cursor: Cursor,
+    settings: ty_ide::CompletionSettings,
+}
+
+impl Task {
+    /// Create a new task for the Python project at `project_path`.
+    ///
+    /// `truth` should correspond to the completion configuration and the
+    /// expected answer for completions at the given `cursor` position.
+    fn new(
+        project_path: &SystemPath,
+        truth: &CompletionTruth,
+        cursor: Cursor,
+    ) -> anyhow::Result<Task> {
+        let name = project_path.file_name().ok_or_else(|| {
+            anyhow::anyhow!("project directory `{project_path}` does not contain a base name")
+        })?;
+
+        let system = OsSystem::new(project_path);
+        let mut project_metadata = ProjectMetadata::discover(project_path, &system)?;
+        project_metadata.apply_configuration_files(&system)?;
+        let db = ProjectDatabase::new(project_metadata, system)?;
+        Ok(Task {
+            db,
+            dir: project_path.to_path_buf(),
+            name: name.to_string(),
+            cursor,
+            settings: (&truth.settings).into(),
+        })
+    }
+
+    /// Returns the rank of the expected answer in the completions
+    /// given.
+    ///
+    /// The rank is the position (one indexed) at which the expected
+    /// answer appears in the slice given, or `None` if the answer
+    /// isn't found at all. A position of zero is maximally correct. A
+    /// missing position is maximally wrong. Anything in the middle is
+    /// a grey area with a lower rank being better.
+    ///
+    /// Because the rank is one indexed, if this returns a rank, then
+    /// it is guaranteed to be non-zero.
+    fn rank(&self, completions: &[Completion<'_>]) -> anyhow::Result<Option<u32>> {
+        completions
+            .iter()
+            .position(|completion| self.cursor.answer.matches(completion))
+            .map(|rank| u32::try_from(rank + 1).context("rank of completion is too big"))
+            .transpose()
+    }
+
+    /// Return completions for this task.
+    fn completions(&self) -> anyhow::Result<Vec<Completion<'_>>> {
+        let file = system_path_to_file(&self.db, &self.cursor.path)
+            .with_context(|| format!("failed to get database file for `{}`", self.cursor.path))?;
+        let offset = ruff_text_size::TextSize::try_from(self.cursor.offset).with_context(|| {
+            format!(
+                "failed to convert `<CURSOR>` file offset `{}` to 32-bit integer",
+                self.cursor.offset
+            )
+        })?;
+        let completions = ty_ide::completion(&self.db, &self.settings, file, offset);
+        Ok(completions)
+    }
+
+    /// Returns the file name, relative to this project's root
+    /// directory, that contains the cursor directive that we
+    /// are evaluating.
+    fn cursor_name(&self) -> &str {
+        self.cursor
+            .path
+            .strip_prefix(&self.dir)
+            .expect("task directory is a parent of cursor")
+            .as_str()
+    }
+}
+
+impl std::fmt::Debug for Task {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        f.debug_struct("Test")
+            .field("db", &"<ProjectDatabase>")
+            .field("dir", &self.dir)
+            .field("name", &self.name)
+            .field("cursor", &self.cursor)
+            .field("settings", &self.settings)
+            .finish()
+    }
+}
+
+/// Truth data for a single completion evaluation test.
+#[derive(Debug, Default, serde::Deserialize)]
+#[serde(rename_all = "kebab-case")]
+struct CompletionTruth {
+    #[serde(default)]
+    settings: CompletionSettings,
+}
+
+/// Settings to forward to our completion routine.
+#[derive(Debug, Default, serde::Deserialize)]
+#[serde(rename_all = "kebab-case")]
+struct CompletionSettings {
+    #[serde(default)]
+    auto_import: bool,
+}
+
+impl From<&CompletionSettings> for ty_ide::CompletionSettings {
+    fn from(x: &CompletionSettings) -> ty_ide::CompletionSettings {
+        ty_ide::CompletionSettings {
+            auto_import: x.auto_import,
+        }
+    }
+}
+
+/// The "source" of a task, as found in ty's git repository.
+#[derive(Debug)]
+struct TaskSource {
+    /// The directory containing this task.
+    dir: SystemPathBuf,
+    /// The name of this task (the basename of `dir`).
+    name: String,
+    /// The "truth" data for this task along with any
+    /// settings. This is pulled from `{dir}/completion.toml`.
+    truth: CompletionTruth,
+}
+
+impl TaskSource {
+    fn all(src_dir: &SystemPath) -> anyhow::Result<Vec<TaskSource>> {
+        let mut sources = vec![];
+        let read_dir = src_dir
+            .as_std_path()
+            .read_dir()
+            .with_context(|| format!("failed to read directory entries in `{src_dir}`"))?;
+        for result in read_dir {
+            let dent = result
+                .with_context(|| format!("failed to get directory entry from `{src_dir}`"))?;
+            let path = dent.path();
+            if !path.is_dir() {
+                continue;
+            }
+
+            let dir = SystemPath::from_std_path(&path).ok_or_else(|| {
+                anyhow::anyhow!(
+                    "truth source directory `{path}` contains invalid UTF-8",
+                    path = path.display()
+                )
+            })?;
+            sources.push(TaskSource::new(dir)?);
+        }
+        // Sort our sources so that we always run in the same order.
+        // And also so that the CSV output is deterministic across
+        // all platforms.
+        sources.sort_by(|source1, source2| source1.name.cmp(&source2.name));
+        Ok(sources)
+    }
+
+    fn new(dir: &SystemPath) -> anyhow::Result<TaskSource> {
+        let name = dir.file_name().ok_or_else(|| {
+            anyhow::anyhow!("truth source directory `{dir}` does not contain a base name")
+        })?;
+
+        let truth_path = dir.join("completion.toml");
+        let truth_data = std::fs::read(truth_path.as_std_path())
+            .with_context(|| format!("failed to read truth data at `{truth_path}`"))?;
+        let truth = toml::from_slice(&truth_data).with_context(|| {
+            format!("failed to parse TOML completion truth data from `{truth_path}`")
+        })?;
+
+        Ok(TaskSource {
+            dir: dir.to_path_buf(),
+            name: name.to_string(),
+            truth,
+        })
+    }
+
+    /// Convert this "source" task (from the Ruff repository) into
+    /// one or more evaluation tasks within a single Python project.
+    /// Exactly one task is created for each cursor directive found in
+    /// this source task.
+    ///
+    /// This includes running `uv sync` to set up a full virtual
+    /// environment.
+    fn to_tasks(&self, parent_dst_dir: &SystemPath) -> anyhow::Result<Vec<Task>> {
+        let dir = parent_dst_dir.join(&self.name);
+        let cursors = copy_project(&self.dir, &dir)?;
+        let uv_sync_output = std::process::Command::new("uv")
+            .arg("sync")
+            .current_dir(dir.as_std_path())
+            .output()
+            .with_context(|| format!("failed to run `uv sync` in `{dir}`"))?;
+        if !uv_sync_output.status.success() {
+            let code = uv_sync_output
+                .status
+                .code()
+                .map(|code| code.to_string())
+                .unwrap_or_else(|| "UNKNOWN".to_string());
+            let stderr = bstr::BStr::new(&uv_sync_output.stderr);
+            anyhow::bail!("`uv sync` failed to run with exit code `{code}`, stderr: {stderr}")
+        }
+        cursors
+            .into_iter()
+            .map(|cursor| Task::new(&dir, &self.truth, cursor))
+            .collect()
+    }
+}
+
+/// A single cursor directive within a single Python project.
+///
+/// Each cursor directive looks like:
+/// `<CURSOR [expected-module.]expected-symbol>`.
+///
+/// That is, each cursor directive corresponds to a single completion
+/// request, and each request is a single evaluation task.
+#[derive(Clone, Debug)]
+struct Cursor {
+    /// The path to the file containing this directive.
+    path: SystemPathBuf,
+    /// The index (starting at 0) of this cursor directive
+    /// within `path`.
+    index: usize,
+    /// The byte offset at which this cursor was located
+    /// within `path`.
+    offset: usize,
+    /// The expected symbol (and optionally module) for this
+    /// completion request.
+    answer: CompletionAnswer,
+}
+
+/// The answer for a single completion request.
+#[derive(Clone, Debug, Default, serde::Deserialize)]
+#[serde(rename_all = "kebab-case")]
+struct CompletionAnswer {
+    symbol: String,
+    module: Option<String>,
+}
+
+impl CompletionAnswer {
+    /// Returns true when this answer matches the completion given.
+    fn matches(&self, completion: &Completion) -> bool {
+        self.symbol == completion.name.as_str()
+            && self.module.as_deref() == completion.module_name.map(ModuleName::as_str)
+    }
+}
+
+/// Copy the Python project from `src_dir` to `dst_dir`.
+///
+/// This also looks for occurrences of cursor directives among the
+/// project files and returns them. The original cursor directives are
+/// deleted.
+///
+/// Hidden files or directories are skipped.
+///
+/// # Errors
+///
+/// Any underlying I/O errors are bubbled up. Also, if no cursor
+/// directives are found, then an error is returned. This guarantees
+/// that the `Vec<Cursor>` is always non-empty.
+fn copy_project(src_dir: &SystemPath, dst_dir: &SystemPath) -> anyhow::Result<Vec<Cursor>> {
+    std::fs::create_dir_all(dst_dir).with_context(|| dst_dir.to_string())?;
+
+    let mut cursors = vec![];
+    for result in walkdir::WalkDir::new(src_dir.as_std_path()) {
+        let dent =
+            result.with_context(|| format!("failed to get directory entry from {src_dir}"))?;
+        if dent
+            .file_name()
+            .to_str()
+            .is_some_and(|name| name.starts_with('.'))
+        {
+            continue;
+        }
+
+        let src = SystemPath::from_std_path(dent.path()).ok_or_else(|| {
+            anyhow::anyhow!("path `{}` is not valid UTF-8", dent.path().display())
+        })?;
+        let name = src
+            .strip_prefix(src_dir)
+            .expect("descendent of `src_dir` must start with `src`");
+        // let name = src
+        // .file_name()
+        // .ok_or_else(|| anyhow::anyhow!("path `{src}` is missing a basename"))?;
+        let dst = dst_dir.join(name);
+        if dent.file_type().is_dir() {
+            std::fs::create_dir_all(dst.as_std_path())
+                .with_context(|| format!("failed to create directory `{dst}`"))?;
+        } else {
+            cursors.extend(copy_file(src, &dst)?);
+        }
+    }
+    anyhow::ensure!(
+        !cursors.is_empty(),
+        "could not find any `<CURSOR>` directives in any of the files in `{src_dir}`",
+    );
+    Ok(cursors)
+}
+
+/// Copies `src` to `dst` while looking for cursor directives.
+///
+/// Each cursor directive looks like:
+/// `<CURSOR [expected-module.]expected-symbol>`.
+///
+/// When occurrences of cursor directives are found, then they are
+/// replaced with the empty string. The position of each occurrence is
+/// recorded, which points to the correct place in a document where all
+/// cursor directives are omitted.
+///
+/// # Errors
+///
+/// When an underlying I/O error occurs.
+fn copy_file(src: &SystemPath, dst: &SystemPath) -> anyhow::Result<Vec<Cursor>> {
+    static RE: LazyLock<Regex> = LazyLock::new(|| {
+        // Our module/symbol identifier regex here is certainly more
+        // permissive than necessary, but I think that should be fine
+        // for this silly little syntax. ---AG
+        Regex::new(r"<CURSOR:\s*(?:(?<module>[\S--.]+)\.)?(?<symbol>[\S--.]+)>").unwrap()
+    });
+
+    let src_data =
+        std::fs::read(src).with_context(|| format!("failed to read `{src}` for copying"))?;
+    let mut cursors = vec![];
+    // The new data, without cursor directives.
+    let mut new = Vec::with_capacity(src_data.len());
+    // An index into `src_data` corresponding to either the start of
+    // the data or the end of the previous cursor directive that we
+    // found.
+    let mut prev_match_end = 0;
+    // The total bytes removed so far by replacing cursor directives
+    // with empty strings.
+    let mut bytes_removed = 0;
+    for (index, caps) in RE.captures_iter(&src_data).enumerate() {
+        let overall = caps.get(0).expect("zeroth group is always available");
+        new.extend_from_slice(&src_data[prev_match_end..overall.start()]);
+        prev_match_end = overall.end();
+        let offset = overall.start() - bytes_removed;
+        bytes_removed += overall.len();
+
+        let symbol = str::from_utf8(&caps["symbol"])
+            .context("expected symbol in cursor directive in `{src}` is not valid UTF-8")?
+            .to_string();
+        let module = caps
+            .name("module")
+            .map(|module| {
+                str::from_utf8(module.as_bytes())
+                    .context("expected module in cursor directive in `{src}` is not valid UTF-8")
+            })
+            .transpose()?
+            .map(ToString::to_string);
+        let answer = CompletionAnswer { symbol, module };
+        cursors.push(Cursor {
+            path: dst.to_path_buf(),
+            index,
+            offset,
+            answer,
+        });
+    }
+    new.extend_from_slice(&src_data[prev_match_end..]);
+    std::fs::write(dst, &new)
+        .with_context(|| format!("failed to write contents of `{src}` to `{dst}`"))?;
+    Ok(cursors)
+}
--- a/crates/ty_completion_eval/truth/README.md
+++ b/crates/ty_completion_eval/truth/README.md
@ -0,0 +1,11 @@
+This directory contains truth data for ty's completion evaluation.
+
+# Adding new truth data
+
+To add new truth data, you can either add a new `<CURSOR>` directive to an
+existing Python project in this directory or create a new Python project. To
+create a new directory, just `cp -a existing new` and modify it as needed. Then:
+
+1. Check `completion.toml` for relevant settings.
+2. Run `uv.lock` after updating `pyproject.toml` (if necessary) to ensure the
+   dependency versions are locked.
--- a/crates/ty_completion_eval/truth/higher-level-symbols-preferred/completion.toml
+++ b/crates/ty_completion_eval/truth/higher-level-symbols-preferred/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = true
--- a/crates/ty_completion_eval/truth/higher-level-symbols-preferred/main.py
+++ b/crates/ty_completion_eval/truth/higher-level-symbols-preferred/main.py
@ -0,0 +1,9 @@
+# This is similar to the `numpy-array` test case,
+# where the completions returned don't contain
+# the expected symbol at all.
+ZQZQZQ_<CURSOR: sub1.ZQZQZQ_SOMETHING_IMPORTANT>
+
+import sub1
+# This works though, so ty sees the symbol where
+# as our auto-import symbol finder does not.
+sub1.ZQZQZQ_<CURSOR: ZQZQZQ_SOMETHING_IMPORTANT>
--- a/crates/ty_completion_eval/truth/higher-level-symbols-preferred/pyproject.toml
+++ b/crates/ty_completion_eval/truth/higher-level-symbols-preferred/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/higher-level-symbols-preferred/sub1/init.py
+++ b/crates/ty_completion_eval/truth/higher-level-symbols-preferred/sub1/init.py
@ -0,0 +1 @@
+from .sub2 import ZQZQZQ_SOMETHING_IMPORTANT
--- a/crates/ty_completion_eval/truth/higher-level-symbols-preferred/sub1/sub2.py
+++ b/crates/ty_completion_eval/truth/higher-level-symbols-preferred/sub1/sub2.py
@ -0,0 +1 @@
+ZQZQZQ_SOMETHING_IMPORTANT = 1
--- a/crates/ty_completion_eval/truth/higher-level-symbols-preferred/uv.lock
+++ b/crates/ty_completion_eval/truth/higher-level-symbols-preferred/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
--- a/crates/ty_completion_eval/truth/import-deprioritizes-dunder/zqzqzq/init__.py
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-dunder/zqzqzq/init__.py
--- a/crates/ty_completion_eval/truth/import-deprioritizes-dunder/completion.toml
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-dunder/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = false
--- a/crates/ty_completion_eval/truth/import-deprioritizes-dunder/main.py
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-dunder/main.py
@ -0,0 +1,4 @@
+# This checks that we prioritize modules without
+# preceding double underscores over modules with
+# preceding double underscores.
+import zqzq<CURSOR: zqzqzq>
--- a/crates/ty_completion_eval/truth/import-deprioritizes-dunder/pyproject.toml
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-dunder/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/import-deprioritizes-dunder/uv.lock
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-dunder/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
--- a/crates/ty_completion_eval/truth/import-deprioritizes-dunder/zqzqzq/init.py
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-dunder/zqzqzq/init.py
--- a/crates/ty_completion_eval/truth/import-deprioritizes-sunder/_zqzqzq/init.py
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-sunder/_zqzqzq/init.py
--- a/crates/ty_completion_eval/truth/import-deprioritizes-sunder/completion.toml
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-sunder/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = false
--- a/crates/ty_completion_eval/truth/import-deprioritizes-sunder/main.py
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-sunder/main.py
@ -0,0 +1,4 @@
+# This checks that we prioritize modules without
+# preceding underscores over modules with
+# preceding underscores.
+import zqzq<CURSOR: zqzqzq>
--- a/crates/ty_completion_eval/truth/import-deprioritizes-sunder/pyproject.toml
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-sunder/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/import-deprioritizes-sunder/uv.lock
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-sunder/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
--- a/crates/ty_completion_eval/truth/import-deprioritizes-sunder/zqzqzq/init.py
+++ b/crates/ty_completion_eval/truth/import-deprioritizes-sunder/zqzqzq/init.py
--- a/crates/ty_completion_eval/truth/internal-typeshed-hidden/completion.toml
+++ b/crates/ty_completion_eval/truth/internal-typeshed-hidden/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = true
--- a/crates/ty_completion_eval/truth/internal-typeshed-hidden/main.py
+++ b/crates/ty_completion_eval/truth/internal-typeshed-hidden/main.py
@ -0,0 +1,9 @@
+# This is a case where a symbol from an internal module appears
+# before the desired symbol from `typing`.
+#
+# We use a slightly different example than the one reported in
+# the issue to capture the deficiency via ranking. That is, in
+# astral-sh/ty#1274, the (current) top suggestion is the correct one.
+#
+# ref: https://github.com/astral-sh/ty/issues/1274#issuecomment-3345923575
+NoneTy<CURSOR: types.NoneType>
--- a/crates/ty_completion_eval/truth/internal-typeshed-hidden/pyproject.toml
+++ b/crates/ty_completion_eval/truth/internal-typeshed-hidden/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/internal-typeshed-hidden/uv.lock
+++ b/crates/ty_completion_eval/truth/internal-typeshed-hidden/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
--- a/crates/ty_completion_eval/truth/numpy-array/completion.toml
+++ b/crates/ty_completion_eval/truth/numpy-array/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = true
--- a/crates/ty_completion_eval/truth/numpy-array/main.py
+++ b/crates/ty_completion_eval/truth/numpy-array/main.py
@ -0,0 +1,16 @@
+# This one is tricky because `array` is an exported
+# symbol in a whole bunch of numpy internal modules.
+#
+# At time of writing (2025-10-07), the right completion
+# doesn't actually show up at all in the suggestions
+# returned. In fact, nothing from the top-level `numpy`
+# module shows up.
+arra<CURSOR: numpy.array>
+
+import numpy as np
+# In contrast to above, this *does* include the correct
+# completion. So there is likely some kind of bug in our
+# symbol discovery code for auto-import that isn't present
+# when using ty to discover symbols (which is likely far
+# too expensive to use across all dependencies).
+np.arra<CURSOR: array>
--- a/crates/ty_completion_eval/truth/numpy-array/pyproject.toml
+++ b/crates/ty_completion_eval/truth/numpy-array/pyproject.toml
@ -0,0 +1,7 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = [
+    "numpy>=2.3.3",
+]
--- a/crates/ty_completion_eval/truth/numpy-array/uv.lock
+++ b/crates/ty_completion_eval/truth/numpy-array/uv.lock
@ -0,0 +1,66 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "numpy"
+version = "2.3.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d0/19/95b3d357407220ed24c139018d2518fab0a61a948e68286a25f1a4d049ff/numpy-2.3.3.tar.gz", hash = "sha256:ddc7c39727ba62b80dfdbedf400d1c10ddfa8eefbd7ec8dcb118be8b56d31029", size = 20576648, upload-time = "2025-09-09T16:54:12.543Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7d/b9/984c2b1ee61a8b803bf63582b4ac4242cf76e2dbd663efeafcb620cc0ccb/numpy-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f5415fb78995644253370985342cd03572ef8620b934da27d77377a2285955bf", size = 20949588, upload-time = "2025-09-09T15:56:59.087Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/e4/07970e3bed0b1384d22af1e9912527ecbeb47d3b26e9b6a3bced068b3bea/numpy-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d00de139a3324e26ed5b95870ce63be7ec7352171bc69a4cf1f157a48e3eb6b7", size = 14177802, upload-time = "2025-09-09T15:57:01.73Z" },
+    { url = "https://files.pythonhosted.org/packages/35/c7/477a83887f9de61f1203bad89cf208b7c19cc9fef0cebef65d5a1a0619f2/numpy-2.3.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9dc13c6a5829610cc07422bc74d3ac083bd8323f14e2827d992f9e52e22cd6a6", size = 5106537, upload-time = "2025-09-09T15:57:03.765Z" },
+    { url = "https://files.pythonhosted.org/packages/52/47/93b953bd5866a6f6986344d045a207d3f1cfbad99db29f534ea9cee5108c/numpy-2.3.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d79715d95f1894771eb4e60fb23f065663b2298f7d22945d66877aadf33d00c7", size = 6640743, upload-time = "2025-09-09T15:57:07.921Z" },
+    { url = "https://files.pythonhosted.org/packages/23/83/377f84aaeb800b64c0ef4de58b08769e782edcefa4fea712910b6f0afd3c/numpy-2.3.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:952cfd0748514ea7c3afc729a0fc639e61655ce4c55ab9acfab14bda4f402b4c", size = 14278881, upload-time = "2025-09-09T15:57:11.349Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b83648633d46f77039c29078751f80da65aa64d5622a3cd62aaef9d835b6c93", size = 16636301, upload-time = "2025-09-09T15:57:14.245Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/59/1287924242eb4fa3f9b3a2c30400f2e17eb2707020d1c5e3086fe7330717/numpy-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b001bae8cea1c7dfdb2ae2b017ed0a6f2102d7a70059df1e338e307a4c78a8ae", size = 16053645, upload-time = "2025-09-09T15:57:16.534Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/93/b3d47ed882027c35e94ac2320c37e452a549f582a5e801f2d34b56973c97/numpy-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8e9aced64054739037d42fb84c54dd38b81ee238816c948c8f3ed134665dcd86", size = 18578179, upload-time = "2025-09-09T15:57:18.883Z" },
+    { url = "https://files.pythonhosted.org/packages/20/d9/487a2bccbf7cc9d4bfc5f0f197761a5ef27ba870f1e3bbb9afc4bbe3fcc2/numpy-2.3.3-cp313-cp313-win32.whl", hash = "sha256:9591e1221db3f37751e6442850429b3aabf7026d3b05542d102944ca7f00c8a8", size = 6312250, upload-time = "2025-09-09T15:57:21.296Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/b5/263ebbbbcede85028f30047eab3d58028d7ebe389d6493fc95ae66c636ab/numpy-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f0dadeb302887f07431910f67a14d57209ed91130be0adea2f9793f1a4f817cf", size = 12783269, upload-time = "2025-09-09T15:57:23.034Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/75/67b8ca554bbeaaeb3fac2e8bce46967a5a06544c9108ec0cf5cece559b6c/numpy-2.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:3c7cf302ac6e0b76a64c4aecf1a09e51abd9b01fc7feee80f6c43e3ab1b1dbc5", size = 10195314, upload-time = "2025-09-09T15:57:25.045Z" },
+    { url = "https://files.pythonhosted.org/packages/11/d0/0d1ddec56b162042ddfafeeb293bac672de9b0cfd688383590090963720a/numpy-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:eda59e44957d272846bb407aad19f89dc6f58fecf3504bd144f4c5cf81a7eacc", size = 21048025, upload-time = "2025-09-09T15:57:27.257Z" },
+    { url = "https://files.pythonhosted.org/packages/36/9e/1996ca6b6d00415b6acbdd3c42f7f03ea256e2c3f158f80bd7436a8a19f3/numpy-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:823d04112bc85ef5c4fda73ba24e6096c8f869931405a80aa8b0e604510a26bc", size = 14301053, upload-time = "2025-09-09T15:57:30.077Z" },
+    { url = "https://files.pythonhosted.org/packages/05/24/43da09aa764c68694b76e84b3d3f0c44cb7c18cdc1ba80e48b0ac1d2cd39/numpy-2.3.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:40051003e03db4041aa325da2a0971ba41cf65714e65d296397cc0e32de6018b", size = 5229444, upload-time = "2025-09-09T15:57:32.733Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/14/50ffb0f22f7218ef8af28dd089f79f68289a7a05a208db9a2c5dcbe123c1/numpy-2.3.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:6ee9086235dd6ab7ae75aba5662f582a81ced49f0f1c6de4260a78d8f2d91a19", size = 6738039, upload-time = "2025-09-09T15:57:34.328Z" },
+    { url = "https://files.pythonhosted.org/packages/55/52/af46ac0795e09657d45a7f4db961917314377edecf66db0e39fa7ab5c3d3/numpy-2.3.3-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94fcaa68757c3e2e668ddadeaa86ab05499a70725811e582b6a9858dd472fb30", size = 14352314, upload-time = "2025-09-09T15:57:36.255Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/b1/dc226b4c90eb9f07a3fff95c2f0db3268e2e54e5cce97c4ac91518aee71b/numpy-2.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da1a74b90e7483d6ce5244053399a614b1d6b7bc30a60d2f570e5071f8959d3e", size = 16701722, upload-time = "2025-09-09T15:57:38.622Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/9d/9d8d358f2eb5eced14dba99f110d83b5cd9a4460895230f3b396ad19a323/numpy-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2990adf06d1ecee3b3dcbb4977dfab6e9f09807598d647f04d385d29e7a3c3d3", size = 16132755, upload-time = "2025-09-09T15:57:41.16Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/27/b3922660c45513f9377b3fb42240bec63f203c71416093476ec9aa0719dc/numpy-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ed635ff692483b8e3f0fcaa8e7eb8a75ee71aa6d975388224f70821421800cea", size = 18651560, upload-time = "2025-09-09T15:57:43.459Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/8e/3ab61a730bdbbc201bb245a71102aa609f0008b9ed15255500a99cd7f780/numpy-2.3.3-cp313-cp313t-win32.whl", hash = "sha256:a333b4ed33d8dc2b373cc955ca57babc00cd6f9009991d9edc5ddbc1bac36bcd", size = 6442776, upload-time = "2025-09-09T15:57:45.793Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/3a/e22b766b11f6030dc2decdeff5c2fb1610768055603f9f3be88b6d192fb2/numpy-2.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:4384a169c4d8f97195980815d6fcad04933a7e1ab3b530921c3fef7a1c63426d", size = 12927281, upload-time = "2025-09-09T15:57:47.492Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/42/c2e2bc48c5e9b2a83423f99733950fbefd86f165b468a3d85d52b30bf782/numpy-2.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:75370986cc0bc66f4ce5110ad35aae6d182cc4ce6433c40ad151f53690130bf1", size = 10265275, upload-time = "2025-09-09T15:57:49.647Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/01/342ad585ad82419b99bcf7cebe99e61da6bedb89e213c5fd71acc467faee/numpy-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cd052f1fa6a78dee696b58a914b7229ecfa41f0a6d96dc663c1220a55e137593", size = 20951527, upload-time = "2025-09-09T15:57:52.006Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/d8/204e0d73fc1b7a9ee80ab1fe1983dd33a4d64a4e30a05364b0208e9a241a/numpy-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:414a97499480067d305fcac9716c29cf4d0d76db6ebf0bf3cbce666677f12652", size = 14186159, upload-time = "2025-09-09T15:57:54.407Z" },
+    { url = "https://files.pythonhosted.org/packages/22/af/f11c916d08f3a18fb8ba81ab72b5b74a6e42ead4c2846d270eb19845bf74/numpy-2.3.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:50a5fe69f135f88a2be9b6ca0481a68a136f6febe1916e4920e12f1a34e708a7", size = 5114624, upload-time = "2025-09-09T15:57:56.5Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/11/0ed919c8381ac9d2ffacd63fd1f0c34d27e99cab650f0eb6f110e6ae4858/numpy-2.3.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:b912f2ed2b67a129e6a601e9d93d4fa37bef67e54cac442a2f588a54afe5c67a", size = 6642627, upload-time = "2025-09-09T15:57:58.206Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/83/deb5f77cb0f7ba6cb52b91ed388b47f8f3c2e9930d4665c600408d9b90b9/numpy-2.3.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e318ee0596d76d4cb3d78535dc005fa60e5ea348cd131a51e99d0bdbe0b54fe", size = 14296926, upload-time = "2025-09-09T15:58:00.035Z" },
+    { url = "https://files.pythonhosted.org/packages/77/cc/70e59dcb84f2b005d4f306310ff0a892518cc0c8000a33d0e6faf7ca8d80/numpy-2.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce020080e4a52426202bdb6f7691c65bb55e49f261f31a8f506c9f6bc7450421", size = 16638958, upload-time = "2025-09-09T15:58:02.738Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/5a/b2ab6c18b4257e099587d5b7f903317bd7115333ad8d4ec4874278eafa61/numpy-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6687dc183aa55dae4a705b35f9c0f8cb178bcaa2f029b241ac5356221d5c021", size = 16071920, upload-time = "2025-09-09T15:58:05.029Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/f1/8b3fdc44324a259298520dd82147ff648979bed085feeacc1250ef1656c0/numpy-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d8f3b1080782469fdc1718c4ed1d22549b5fb12af0d57d35e992158a772a37cf", size = 18577076, upload-time = "2025-09-09T15:58:07.745Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/a1/b87a284fb15a42e9274e7fcea0dad259d12ddbf07c1595b26883151ca3b4/numpy-2.3.3-cp314-cp314-win32.whl", hash = "sha256:cb248499b0bc3be66ebd6578b83e5acacf1d6cb2a77f2248ce0e40fbec5a76d0", size = 6366952, upload-time = "2025-09-09T15:58:10.096Z" },
+    { url = "https://files.pythonhosted.org/packages/70/5f/1816f4d08f3b8f66576d8433a66f8fa35a5acfb3bbd0bf6c31183b003f3d/numpy-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:691808c2b26b0f002a032c73255d0bd89751425f379f7bcd22d140db593a96e8", size = 12919322, upload-time = "2025-09-09T15:58:12.138Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/de/072420342e46a8ea41c324a555fa90fcc11637583fb8df722936aed1736d/numpy-2.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:9ad12e976ca7b10f1774b03615a2a4bab8addce37ecc77394d8e986927dc0dfe", size = 10478630, upload-time = "2025-09-09T15:58:14.64Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/df/ee2f1c0a9de7347f14da5dd3cd3c3b034d1b8607ccb6883d7dd5c035d631/numpy-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9cc48e09feb11e1db00b320e9d30a4151f7369afb96bd0e48d942d09da3a0d00", size = 21047987, upload-time = "2025-09-09T15:58:16.889Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/92/9453bdc5a4e9e69cf4358463f25e8260e2ffc126d52e10038b9077815989/numpy-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:901bf6123879b7f251d3631967fd574690734236075082078e0571977c6a8e6a", size = 14301076, upload-time = "2025-09-09T15:58:20.343Z" },
+    { url = "https://files.pythonhosted.org/packages/13/77/1447b9eb500f028bb44253105bd67534af60499588a5149a94f18f2ca917/numpy-2.3.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:7f025652034199c301049296b59fa7d52c7e625017cae4c75d8662e377bf487d", size = 5229491, upload-time = "2025-09-09T15:58:22.481Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/f9/d72221b6ca205f9736cb4b2ce3b002f6e45cd67cd6a6d1c8af11a2f0b649/numpy-2.3.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:533ca5f6d325c80b6007d4d7fb1984c303553534191024ec6a524a4c92a5935a", size = 6737913, upload-time = "2025-09-09T15:58:24.569Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/5f/d12834711962ad9c46af72f79bb31e73e416ee49d17f4c797f72c96b6ca5/numpy-2.3.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0edd58682a399824633b66885d699d7de982800053acf20be1eaa46d92009c54", size = 14352811, upload-time = "2025-09-09T15:58:26.416Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/0d/fdbec6629d97fd1bebed56cd742884e4eead593611bbe1abc3eb40d304b2/numpy-2.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:367ad5d8fbec5d9296d18478804a530f1191e24ab4d75ab408346ae88045d25e", size = 16702689, upload-time = "2025-09-09T15:58:28.831Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/09/0a35196dc5575adde1eb97ddfbc3e1687a814f905377621d18ca9bc2b7dd/numpy-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8f6ac61a217437946a1fa48d24c47c91a0c4f725237871117dea264982128097", size = 16133855, upload-time = "2025-09-09T15:58:31.349Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/ca/c9de3ea397d576f1b6753eaa906d4cdef1bf97589a6d9825a349b4729cc2/numpy-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:179a42101b845a816d464b6fe9a845dfaf308fdfc7925387195570789bb2c970", size = 18652520, upload-time = "2025-09-09T15:58:33.762Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/c2/e5ed830e08cd0196351db55db82f65bc0ab05da6ef2b72a836dcf1936d2f/numpy-2.3.3-cp314-cp314t-win32.whl", hash = "sha256:1250c5d3d2562ec4174bce2e3a1523041595f9b651065e4a4473f5f48a6bc8a5", size = 6515371, upload-time = "2025-09-09T15:58:36.04Z" },
+    { url = "https://files.pythonhosted.org/packages/47/c7/b0f6b5b67f6788a0725f744496badbb604d226bf233ba716683ebb47b570/numpy-2.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b37a0b2e5935409daebe82c1e42274d30d9dd355852529eab91dab8dcca7419f", size = 13112576, upload-time = "2025-09-09T15:58:37.927Z" },
+    { url = "https://files.pythonhosted.org/packages/06/b9/33bba5ff6fb679aa0b1f8a07e853f002a6b04b9394db3069a1270a7784ca/numpy-2.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:78c9f6560dc7e6b3990e32df7ea1a50bbd0e2a111e05209963f5ddcab7073b0b", size = 10545953, upload-time = "2025-09-09T15:58:40.576Z" },
+]
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "numpy" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "numpy", specifier = ">=2.3.3" }]
--- a/crates/ty_completion_eval/truth/object-attr-instance-methods/completion.toml
+++ b/crates/ty_completion_eval/truth/object-attr-instance-methods/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = false
--- a/crates/ty_completion_eval/truth/object-attr-instance-methods/main.py
+++ b/crates/ty_completion_eval/truth/object-attr-instance-methods/main.py
@ -0,0 +1,16 @@
+class Quux:
+    def __init__(self): pass
+    def lion(self): pass
+    def tiger(self): pass
+    def bear(self): pass
+    def chicken(self): pass
+    def turkey(self): pass
+    def wasp(self): pass
+    def rabbit(self): pass
+    def squirrel(self): pass
+
+quux = Quux()
+quux.tur<CURSOR: turkey>
+
+quux = Quux()
+quux.be<CURSOR: bear>
--- a/crates/ty_completion_eval/truth/object-attr-instance-methods/pyproject.toml
+++ b/crates/ty_completion_eval/truth/object-attr-instance-methods/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/object-attr-instance-methods/uv.lock
+++ b/crates/ty_completion_eval/truth/object-attr-instance-methods/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
--- a/crates/ty_completion_eval/truth/raise-uses-base-exception/completion.toml
+++ b/crates/ty_completion_eval/truth/raise-uses-base-exception/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = false
--- a/crates/ty_completion_eval/truth/raise-uses-base-exception/main.py
+++ b/crates/ty_completion_eval/truth/raise-uses-base-exception/main.py
@ -0,0 +1,2 @@
+# ref: https://github.com/astral-sh/ty/issues/1262
+raise NotImplement<CURSOR: NotImplementedError>
--- a/crates/ty_completion_eval/truth/raise-uses-base-exception/pyproject.toml
+++ b/crates/ty_completion_eval/truth/raise-uses-base-exception/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/raise-uses-base-exception/uv.lock
+++ b/crates/ty_completion_eval/truth/raise-uses-base-exception/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
--- a/crates/ty_completion_eval/truth/scope-existing-over-new-import/completion.toml
+++ b/crates/ty_completion_eval/truth/scope-existing-over-new-import/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = true
--- a/crates/ty_completion_eval/truth/scope-existing-over-new-import/main.py
+++ b/crates/ty_completion_eval/truth/scope-existing-over-new-import/main.py
@ -0,0 +1,3 @@
+# ref: https://github.com/astral-sh/ty/issues/1274#issuecomment-3345942698
+from typing import Iterator
+Iter<CURSOR: Iterator>
--- a/crates/ty_completion_eval/truth/scope-existing-over-new-import/pyproject.toml
+++ b/crates/ty_completion_eval/truth/scope-existing-over-new-import/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/scope-existing-over-new-import/uv.lock
+++ b/crates/ty_completion_eval/truth/scope-existing-over-new-import/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
--- a/crates/ty_completion_eval/truth/scope-prioritize-closer/completion.toml
+++ b/crates/ty_completion_eval/truth/scope-prioritize-closer/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = false
--- a/crates/ty_completion_eval/truth/scope-prioritize-closer/main.py
+++ b/crates/ty_completion_eval/truth/scope-prioritize-closer/main.py
@ -0,0 +1,5 @@
+zqzqzq_global_identifier = 1
+
+def foo():
+    zqzqzq_local_identifier = 1
+    zqzqzq_<CURSOR: zqzqzq_local_identifier>
--- a/crates/ty_completion_eval/truth/scope-prioritize-closer/pyproject.toml
+++ b/crates/ty_completion_eval/truth/scope-prioritize-closer/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/scope-prioritize-closer/uv.lock
+++ b/crates/ty_completion_eval/truth/scope-prioritize-closer/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
--- a/crates/ty_completion_eval/truth/scope-simple-long-identifier/completion.toml
+++ b/crates/ty_completion_eval/truth/scope-simple-long-identifier/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = false
--- a/crates/ty_completion_eval/truth/scope-simple-long-identifier/main.py
+++ b/crates/ty_completion_eval/truth/scope-simple-long-identifier/main.py
@ -0,0 +1,2 @@
+simple_long_identifier = 1
+simple<CURSOR: simple_long_identifier>
--- a/crates/ty_completion_eval/truth/scope-simple-long-identifier/pyproject.toml
+++ b/crates/ty_completion_eval/truth/scope-simple-long-identifier/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/scope-simple-long-identifier/uv.lock
+++ b/crates/ty_completion_eval/truth/scope-simple-long-identifier/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
--- a/crates/ty_completion_eval/truth/ty-extensions-lower-stdlib/completion.toml
+++ b/crates/ty_completion_eval/truth/ty-extensions-lower-stdlib/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = true
--- a/crates/ty_completion_eval/truth/ty-extensions-lower-stdlib/main.py
+++ b/crates/ty_completion_eval/truth/ty-extensions-lower-stdlib/main.py
@ -0,0 +1,2 @@
+# ref: https://github.com/astral-sh/ty/issues/1274#issuecomment-3345879257
+reveal<CURSOR: typing.reveal_type>
--- a/crates/ty_completion_eval/truth/ty-extensions-lower-stdlib/pyproject.toml
+++ b/crates/ty_completion_eval/truth/ty-extensions-lower-stdlib/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/ty-extensions-lower-stdlib/uv.lock
+++ b/crates/ty_completion_eval/truth/ty-extensions-lower-stdlib/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
--- a/crates/ty_completion_eval/truth/type-var-typing-over-ast/completion.toml
+++ b/crates/ty_completion_eval/truth/type-var-typing-over-ast/completion.toml
@ -0,0 +1,2 @@
+[settings]
+auto-import = true
--- a/crates/ty_completion_eval/truth/type-var-typing-over-ast/main.py
+++ b/crates/ty_completion_eval/truth/type-var-typing-over-ast/main.py
@ -0,0 +1,12 @@
+# This one demands that `TypeVa` complete to `typing.TypeVar`
+# even though there is also an `ast.TypeVar`. Getting this one
+# right seems tricky, and probably requires module-specific
+# heuristics.
+#
+# ref: https://github.com/astral-sh/ty/issues/1274#issuecomment-3345884227
+TypeVa<CURSOR: typing.TypeVar>
+
+# This is a similar case of `ctypes.cast` being preferred over
+# `typing.cast`. Maybe `typing` should just get a slightly higher
+# weight than most other stdlib modules?
+cas<CURSOR: typing.cast>
--- a/crates/ty_completion_eval/truth/type-var-typing-over-ast/pyproject.toml
+++ b/crates/ty_completion_eval/truth/type-var-typing-over-ast/pyproject.toml
@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []
--- a/crates/ty_completion_eval/truth/type-var-typing-over-ast/uv.lock
+++ b/crates/ty_completion_eval/truth/type-var-typing-over-ast/uv.lock
@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "test"
+version = "0.1.0"
+source = { virtual = "." }
				`@ -0,0 +1 @@`
				`from .sub2 import ZQZQZQ_SOMETHING_IMPORTANT`