[ty] First cut at semantic token provider (#19108)

This PR implements a basic semantic token provider for ty's language
server. This allows for more accurate semantic highlighting / coloring
within editors that support this LSP functionality.

Here are screen shots that show how code appears in VS Code using the
"rainbow" theme both before and after this change.


![461737617-15630625-d4a9-4ec5-9886-77b00eb7a41a](https://github.com/user-attachments/assets/f963b55b-3195-41d1-ba38-ac2e7508d5f5)


![461737624-d6dcf5f0-7b9b-47de-a410-e202c63e2058](https://github.com/user-attachments/assets/111ca2c5-bb4f-4c8a-a0b5-6c1b2b6f246b)

The token types and modifier tags in this implementation largely mirror
those used in Microsoft's default language server for Python.

The implementation supports two LSP interfaces. The first provides
semantic tokens for an entire document, and the second returns semantic
tokens for a requested range within a document.

The PR includes unit tests. It also includes comments that document
known limitations and areas for future improvements.

---------

Co-authored-by: UnboundVariable <unbound@gmail.com>

crates/ty_python_semantic/src/types.rs

@@ -46,7 +46,7 @@ use crate::types::generics::{
     GenericContext, PartialSpecialization, Specialization, walk_generic_context,
     walk_partial_specialization, walk_specialization,
 };
-pub use crate::types::ide_support::all_members;
+pub use crate::types::ide_support::{all_members, definition_kind_for_name};
 use crate::types::infer::infer_unpack_types;
 use crate::types::mro::{Mro, MroError, MroIterator};
 pub(crate) use crate::types::narrow::infer_narrowing_constraint;

crates/ty_python_semantic/src/types/ide_support.rs

@@ -1,10 +1,13 @@
 use crate::place::{Place, imported_symbol, place_from_bindings, place_from_declarations};
+use crate::semantic_index::definition::DefinitionKind;
 use crate::semantic_index::place::ScopeId;
 use crate::semantic_index::{
     attribute_scopes, global_scope, imported_modules, place_table, semantic_index, use_def_map,
 };
 use crate::types::{ClassBase, ClassLiteral, KnownClass, KnownInstanceType, Type};
 use crate::{Db, NameKind};
+use ruff_db::files::File;
+use ruff_python_ast as ast;
 use ruff_python_ast::name::Name;
 use rustc_hash::FxHashSet;
 
@@ -241,3 +244,37 @@ impl AllMembers {
 pub fn all_members<'db>(db: &'db dyn Db, ty: Type<'db>) -> FxHashSet<Name> {
     AllMembers::of(db, ty).members
 }
+
+/// Get the primary definition kind for a name expression within a specific file.
+/// Returns the first definition kind that is reachable for this name in its scope.
+/// This is useful for IDE features like semantic tokens.
+pub fn definition_kind_for_name<'db>(
+    db: &'db dyn Db,
+    file: File,
+    name: &ast::ExprName,
+) -> Option<DefinitionKind<'db>> {
+    let index = semantic_index(db, file);
+    let name_str = name.id.as_str();
+
+    // Get the scope for this name expression
+    let file_scope = index.try_expression_scope_id(&ast::Expr::Name(name.clone()))?;
+
+    // Get the place table for this scope
+    let place_table = index.place_table(file_scope);
+
+    // Look up the place by name
+    let place_id = place_table.place_id_by_name(name_str)?;
+
+    // Get the use-def map and look up definitions for this place
+    let use_def_map = index.use_def_map(file_scope);
+    let declarations = use_def_map.all_reachable_declarations(place_id);
+
+    // Find the first valid definition and return its kind
+    for declaration in declarations {
+        if let Some(def) = declaration.declaration.definition() {
+            return Some(def.kind(db).clone());
+        }
+    }
+
+    None
+}