Merge remote-tracking branch 'origin/main' into glue-getters-rtfeldman

crates/compiler/load_internal/Cargo.toml

@@ -27,6 +27,7 @@ roc_mono = { path = "../mono" }
 roc_intern = { path = "../intern" }
 roc_target = { path = "../roc_target" }
 roc_tracing = { path = "../../tracing" }
+roc_packaging = { path = "../../packaging" }
 roc_reporting = { path = "../../reporting" }
 roc_debug_flags = { path = "../debug_flags" }
 
@@ -35,6 +36,7 @@ ven_pretty = { path = "../../vendor/pretty" }
 bumpalo.workspace = true
 parking_lot.workspace = true
 crossbeam.workspace = true
+tempfile.workspace = true
 
 [dev-dependencies]
 roc_test_utils = { path = "../../test_utils" }

crates/compiler/load_internal/src/docs.rs

@@ -1,28 +1,21 @@
 use crate::docs::DocEntry::DetachedDoc;
 use crate::docs::TypeAnnotation::{Apply, BoundVariable, Function, NoTypeAnn, Record, TagUnion};
-use crate::file::LoadedModule;
 use roc_can::scope::Scope;
+use roc_collections::VecSet;
 use roc_module::ident::ModuleName;
-use roc_module::symbol::IdentIds;
+use roc_module::symbol::{IdentIds, ModuleId, ModuleIds, Symbol};
 use roc_parse::ast::AssignedField;
 use roc_parse::ast::{self, ExtractSpaces, TypeHeader};
 use roc_parse::ast::{CommentOrNewline, TypeDef, ValueDef};
 
 // Documentation generation requirements
 
-#[derive(Debug)]
-pub struct Documentation {
-    pub name: String,
-    pub version: String,
-    pub docs: String,
-    pub modules: Vec<LoadedModule>,
-}
-
 #[derive(Debug)]
 pub struct ModuleDocumentation {
     pub name: String,
     pub entries: Vec<DocEntry>,
     pub scope: Scope,
+    pub exposed_symbols: VecSet<Symbol>,
 }
 
 #[derive(Debug, Clone)]
@@ -34,6 +27,7 @@ pub enum DocEntry {
 #[derive(Debug, Clone)]
 pub struct DocDef {
     pub name: String,
+    pub symbol: Symbol,
     pub type_vars: Vec<String>,
     pub type_annotation: TypeAnnotation,
     pub docs: Option<String>,
@@ -96,17 +90,31 @@ pub struct Tag {
     pub values: Vec<TypeAnnotation>,
 }
 
+#[allow(clippy::too_many_arguments)]
 pub fn generate_module_docs(
     scope: Scope,
+    home: ModuleId,
+    module_ids: &ModuleIds,
     module_name: ModuleName,
     parsed_defs: &roc_parse::ast::Defs,
+    exposed_module_ids: &[ModuleId],
+    exposed_symbols: VecSet<Symbol>,
+    header_comments: &[CommentOrNewline<'_>],
 ) -> ModuleDocumentation {
-    let entries = generate_entry_docs(&scope.locals.ident_ids, parsed_defs);
+    let entries = generate_entry_docs(
+        home,
+        &scope.locals.ident_ids,
+        module_ids,
+        parsed_defs,
+        exposed_module_ids,
+        header_comments,
+    );
 
     ModuleDocumentation {
         name: module_name.as_str().to_string(),
         scope,
         entries,
+        exposed_symbols,
     }
 }
 
@@ -137,13 +145,22 @@ fn detached_docs_from_comments_and_new_lines<'a>(
     detached_docs
 }
 
-fn generate_entry_docs<'a>(
-    ident_ids: &'a IdentIds,
-    defs: &roc_parse::ast::Defs<'a>,
+fn generate_entry_docs(
+    home: ModuleId,
+    ident_ids: &IdentIds,
+    module_ids: &ModuleIds,
+    defs: &roc_parse::ast::Defs<'_>,
+    exposed_module_ids: &[ModuleId],
+    header_comments: &[CommentOrNewline<'_>],
 ) -> Vec<DocEntry> {
     use roc_parse::ast::Pattern;
 
-    let mut acc = Vec::with_capacity(defs.tags.len());
+    let mut acc = Vec::with_capacity(defs.tags.len() + 1);
+
+    if let Some(docs) = comments_or_new_lines_to_docs(header_comments) {
+        acc.push(DetachedDoc(docs));
+    }
+
     let mut before_comments_or_new_lines: Option<&[CommentOrNewline]> = None;
     let mut scratchpad = Vec::new();
 
@@ -165,11 +182,12 @@ fn generate_entry_docs<'a>(
             Err(value_index) => match &defs.value_defs[value_index.index()] {
                 ValueDef::Annotation(loc_pattern, loc_ann) => {
                     if let Pattern::Identifier(identifier) = loc_pattern.value {
-                        // Check if the definition is exposed
-                        if ident_ids.get_id(identifier).is_some() {
+                        // Check if this module exposes the def
+                        if let Some(ident_id) = ident_ids.get_id(identifier) {
                             let name = identifier.to_string();
                             let doc_def = DocDef {
                                 name,
+                                symbol: Symbol::new(home, ident_id),
                                 type_annotation: type_to_docs(false, loc_ann.value),
                                 type_vars: Vec::new(),
                                 docs,
@@ -185,12 +203,13 @@ fn generate_entry_docs<'a>(
                     ..
                 } => {
                     if let Pattern::Identifier(identifier) = ann_pattern.value {
-                        // Check if the definition is exposed
-                        if ident_ids.get_id(identifier).is_some() {
+                        // Check if this module exposes the def
+                        if let Some(ident_id) = ident_ids.get_id(identifier) {
                             let doc_def = DocDef {
                                 name: identifier.to_string(),
                                 type_annotation: type_to_docs(false, ann_type.value),
                                 type_vars: Vec::new(),
+                                symbol: Symbol::new(home, ident_id),
                                 docs,
                             };
                             acc.push(DocEntry::DocDef(doc_def));
@@ -198,7 +217,13 @@ fn generate_entry_docs<'a>(
                     }
                 }
 
-                ValueDef::Body(_, _) => (),
+                ValueDef::Body(_, _) => {
+                    // TODO generate docs for un-annotated bodies
+                }
+
+                ValueDef::Dbg { .. } => {
+                    // Don't generate docs for `dbg`s
+                }
 
                 ValueDef::Expect { .. } => {
                     // Don't generate docs for `expect`s
@@ -221,11 +246,24 @@ fn generate_entry_docs<'a>(
                         }
                     }
 
+                    let type_annotation =
+                        // If this alias contains an unexposed type, then don't try to render a
+                        // type annotation for it. You're not allowed to see that!
+                        // (This comes up when exporting an alias like Task ok err : InnerTask ok err
+                        // where Task is exposed but InnerTask isn't.)
+                        if contains_unexposed_type(&ann.value, exposed_module_ids, module_ids) {
+                            TypeAnnotation::NoTypeAnn
+                        } else {
+                            type_to_docs(false, ann.value)
+                        };
+
+                    let ident_id = ident_ids.get_id(name.value).unwrap();
                     let doc_def = DocDef {
                         name: name.value.to_string(),
-                        type_annotation: type_to_docs(false, ann.value),
+                        type_annotation,
                         type_vars,
                         docs,
+                        symbol: Symbol::new(home, ident_id),
                     };
                     acc.push(DocEntry::DocDef(doc_def));
                 }
@@ -242,11 +280,13 @@ fn generate_entry_docs<'a>(
                         }
                     }
 
+                    let ident_id = ident_ids.get_id(name.value).unwrap();
                     let doc_def = DocDef {
                         name: name.value.to_string(),
                         type_annotation: TypeAnnotation::NoTypeAnn,
                         type_vars,
                         docs,
+                        symbol: Symbol::new(home, ident_id),
                     };
                     acc.push(DocEntry::DocDef(doc_def));
                 }
@@ -280,9 +320,11 @@ fn generate_entry_docs<'a>(
                         })
                         .collect();
 
+                    let ident_id = ident_ids.get_id(name.value).unwrap();
                     let doc_def = DocDef {
                         name: name.value.to_string(),
                         type_annotation: TypeAnnotation::Ability { members },
+                        symbol: Symbol::new(home, ident_id),
                         type_vars,
                         docs,
                     };
@@ -304,6 +346,123 @@ fn generate_entry_docs<'a>(
     acc
 }
 
+/// Does this type contain any types which are not exposed outside the package?
+/// (If so, we shouldn't try to render a type annotation for it.)
+fn contains_unexposed_type(
+    ann: &ast::TypeAnnotation,
+    exposed_module_ids: &[ModuleId],
+    module_ids: &ModuleIds,
+) -> bool {
+    use ast::TypeAnnotation::*;
+
+    match ann {
+        // Apply is the one case that can directly return true.
+        Apply(module_name, _ident, loc_args) => {
+            // If the *ident* was unexposed, we would have gotten a naming error
+            // during canonicalization, so all we need to check is the module.
+            let module_id = module_ids.get_id(&(*module_name).into()).unwrap();
+
+            !exposed_module_ids.contains(&module_id)
+                || loc_args.iter().any(|loc_arg| {
+                    contains_unexposed_type(&loc_arg.value, exposed_module_ids, module_ids)
+                })
+        }
+        Malformed(_) | Inferred | Wildcard | BoundVariable(_) => false,
+        Function(loc_args, loc_ret) => {
+            contains_unexposed_type(&loc_ret.value, exposed_module_ids, module_ids)
+                || loc_args.iter().any(|loc_arg| {
+                    contains_unexposed_type(&loc_arg.value, exposed_module_ids, module_ids)
+                })
+        }
+        Record { fields, ext } => {
+            if let Some(loc_ext) = ext {
+                if contains_unexposed_type(&loc_ext.value, exposed_module_ids, module_ids) {
+                    return true;
+                }
+            }
+
+            let mut fields_to_process =
+                Vec::from_iter(fields.iter().map(|loc_field| loc_field.value));
+
+            while let Some(field) = fields_to_process.pop() {
+                match field {
+                    AssignedField::RequiredValue(_field, _spaces, loc_val)
+                    | AssignedField::OptionalValue(_field, _spaces, loc_val) => {
+                        if contains_unexposed_type(&loc_val.value, exposed_module_ids, module_ids) {
+                            return true;
+                        }
+                    }
+                    AssignedField::Malformed(_) | AssignedField::LabelOnly(_) => {
+                        // contains no unexposed types, so continue
+                    }
+                    AssignedField::SpaceBefore(field, _) | AssignedField::SpaceAfter(field, _) => {
+                        fields_to_process.push(*field);
+                    }
+                }
+            }
+
+            false
+        }
+        Tuple { fields, ext } => {
+            if let Some(loc_ext) = ext {
+                if contains_unexposed_type(&loc_ext.value, exposed_module_ids, module_ids) {
+                    return true;
+                }
+            }
+
+            fields.iter().any(|loc_field| {
+                contains_unexposed_type(&loc_field.value, exposed_module_ids, module_ids)
+            })
+        }
+        TagUnion { ext, tags } => {
+            use ast::Tag;
+
+            if let Some(loc_ext) = ext {
+                if contains_unexposed_type(&loc_ext.value, exposed_module_ids, module_ids) {
+                    return true;
+                }
+            }
+
+            let mut tags_to_process = Vec::from_iter(tags.iter().map(|loc_tag| loc_tag.value));
+
+            while let Some(tag) = tags_to_process.pop() {
+                match tag {
+                    Tag::Apply { name: _, args } => {
+                        for loc_ann in args.iter() {
+                            if contains_unexposed_type(
+                                &loc_ann.value,
+                                exposed_module_ids,
+                                module_ids,
+                            ) {
+                                return true;
+                            }
+                        }
+                    }
+                    Tag::Malformed(_) => {
+                        // contains no unexposed types, so continue
+                    }
+                    Tag::SpaceBefore(tag, _) | Tag::SpaceAfter(tag, _) => {
+                        tags_to_process.push(*tag);
+                    }
+                }
+            }
+
+            false
+        }
+        Where(loc_ann, _loc_has_clauses) => {
+            // We assume all the abilities in the `has` clause are from exported modules.
+            // TODO don't assume this! Instead, look them up and verify.
+            contains_unexposed_type(&loc_ann.value, exposed_module_ids, module_ids)
+        }
+        As(loc_ann, _spaces, _type_header) => {
+            contains_unexposed_type(&loc_ann.value, exposed_module_ids, module_ids)
+        }
+        SpaceBefore(ann, _) | ast::TypeAnnotation::SpaceAfter(ann, _) => {
+            contains_unexposed_type(ann, exposed_module_ids, module_ids)
+        }
+    }
+}
+
 fn type_to_docs(in_func_type_ann: bool, type_annotation: ast::TypeAnnotation) -> TypeAnnotation {
     match type_annotation {
         ast::TypeAnnotation::TagUnion { tags, ext } => {

crates/compiler/load_internal/src/wasm_instant.rs

@@ -29,6 +29,10 @@ impl Duration {
     pub fn checked_sub(&self, _: Duration) -> Option<Duration> {
         Some(Duration)
     }
+
+    pub fn as_secs_f64(&self) -> f64 {
+        0.0
+    }
 }
 
 impl Default for Duration {

crates/compiler/load_internal/src/work.rs

@@ -454,6 +454,8 @@ impl<'a> Dependencies<'a> {
     pub fn load_find_and_make_specializations_after_check(&mut self) -> MutSet<(ModuleId, Phase)> {
         let mut output = MutSet::default();
 
+        // Take out the specialization dependency graph, as this should not be modified as we
+        // reload the build graph. We'll make sure the state is unaffected at the end of this call.
         let mut make_specializations_dependents = MakeSpecializationsDependents::default();
         let default_make_specializations_dependents_len = make_specializations_dependents.0.len();
         std::mem::swap(
@@ -484,8 +486,9 @@ impl<'a> Dependencies<'a> {
                 self.add_dependency(module_dep, module, Phase::MakeSpecializations);
                 self.add_dependency(ModuleId::DERIVED_GEN, module, Phase::MakeSpecializations);
 
-                // `module_dep` can't make its specializations until the current module does.
-                info.has_pred = true;
+                // That `module_dep` can't make its specializations until the current module does
+                // should already be accounted for in `make_specializations_dependents`, which we
+                // populated when initially building the graph.
             }
 
             if module != ModuleId::DERIVED_GEN {

crates/compiler/load_internal/tests/fixtures/build/interface_with_deps/AStar.roc

@@ -13,7 +13,7 @@ Model position :
     }
 
 
-initialModel : position -> Model position
+initialModel : position -> Model position | position has Hash & Eq
 initialModel = \start ->
     { evaluated : Set.empty
     , openSet : Set.single start
@@ -22,7 +22,7 @@ initialModel = \start ->
     }
 
 
-cheapestOpen : (position -> F64), Model position -> Result position [KeyNotFound] | position has Eq
+cheapestOpen : (position -> F64), Model position -> Result position [KeyNotFound] | position has Hash & Eq
 cheapestOpen = \costFunction, model ->
 
     folder = \resSmallestSoFar, position ->
@@ -47,7 +47,7 @@ cheapestOpen = \costFunction, model ->
 
 
 
-reconstructPath : Dict position position, position -> List position | position has Eq
+reconstructPath : Dict position position, position -> List position | position has Hash & Eq
 reconstructPath = \cameFrom, goal ->
     when Dict.get cameFrom goal is
         Err KeyNotFound ->
@@ -56,7 +56,7 @@ reconstructPath = \cameFrom, goal ->
         Ok next ->
             List.append (reconstructPath cameFrom next) goal
 
-updateCost : position, position, Model position -> Model position | position has Eq
+updateCost : position, position, Model position -> Model position | position has Hash & Eq
 updateCost = \current, neighbour, model ->
     newCameFrom = Dict.insert model.cameFrom neighbour current
 
@@ -80,12 +80,12 @@ updateCost = \current, neighbour, model ->
                 model
 
 
-findPath : { costFunction: (position, position -> F64), moveFunction: (position -> Set position), start : position, end : position } -> Result (List position) [KeyNotFound] | position has Eq
+findPath : { costFunction: (position, position -> F64), moveFunction: (position -> Set position), start : position, end : position } -> Result (List position) [KeyNotFound] | position has Hash & Eq
 findPath = \{ costFunction, moveFunction, start, end } ->
     astar costFunction moveFunction end (initialModel start)
 
 
-astar : (position, position -> F64), (position -> Set position), position, Model position -> [Err [KeyNotFound], Ok (List position)] | position has Eq
+astar : (position, position -> F64), (position -> Set position), position, Model position -> [Err [KeyNotFound], Ok (List position)] | position has Hash & Eq
 astar = \costFn, moveFn, goal, model ->
     when cheapestOpen (\position -> costFn goal position) model is
         Err _ ->

crates/compiler/load_internal/tests/test_load.rs

@@ -21,6 +21,7 @@ use roc_load_internal::file::{ExecutionMode, LoadConfig, Threading};
 use roc_load_internal::file::{LoadResult, LoadStart, LoadedModule, LoadingProblem};
 use roc_module::ident::ModuleName;
 use roc_module::symbol::{Interns, ModuleId};
+use roc_packaging::cache::RocCacheDir;
 use roc_problem::can::Problem;
 use roc_region::all::LineInfo;
 use roc_reporting::report::RenderTarget;
@@ -40,7 +41,13 @@ fn load_and_typecheck(
 ) -> Result<LoadedModule, LoadingProblem> {
     use LoadResult::*;
 
-    let load_start = LoadStart::from_path(arena, filename, RenderTarget::Generic, DEFAULT_PALETTE)?;
+    let load_start = LoadStart::from_path(
+        arena,
+        filename,
+        RenderTarget::Generic,
+        RocCacheDir::Disallowed,
+        DEFAULT_PALETTE,
+    )?;
     let load_config = LoadConfig {
         target_info,
         render: RenderTarget::Generic,
@@ -54,6 +61,7 @@ fn load_and_typecheck(
         load_start,
         exposed_types,
         Default::default(), // these tests will re-compile the builtins
+        RocCacheDir::Disallowed,
         load_config,
     )? {
         Monomorphized(_) => unreachable!(""),
@@ -483,12 +491,12 @@ fn load_astar() {
     expect_types(
         loaded_module,
         hashmap! {
-            "findPath" => "{ costFunction : position, position -> F64, end : position, moveFunction : position -> Set position, start : position } -> Result (List position) [KeyNotFound] | position has Eq",
-            "initialModel" => "position -> Model position",
-            "reconstructPath" => "Dict position position, position -> List position | position has Eq",
-            "updateCost" => "position, position, Model position -> Model position | position has Eq",
-            "cheapestOpen" => "(position -> F64), Model position -> Result position [KeyNotFound] | position has Eq",
-            "astar" => "(position, position -> F64), (position -> Set position), position, Model position -> [Err [KeyNotFound], Ok (List position)] | position has Eq",
+            "findPath" => "{ costFunction : position, position -> F64, end : position, moveFunction : position -> Set position, start : position } -> Result (List position) [KeyNotFound] | position has Hash & Eq",
+            "initialModel" => "position -> Model position | position has Hash & Eq",
+            "reconstructPath" => "Dict position position, position -> List position | position has Hash & Eq",
+            "updateCost" => "position, position, Model position -> Model position | position has Hash & Eq",
+            "cheapestOpen" => "(position -> F64), Model position -> Result position [KeyNotFound] | position has Hash & Eq",
+            "astar" => "(position, position -> F64), (position -> Set position), position, Model position -> [Err [KeyNotFound], Ok (List position)] | position has Hash & Eq",
         },
     );
 }
@@ -584,16 +592,19 @@ fn parse_problem() {
                 "
                     ── UNFINISHED LIST ──────────────────────────────────── tmp/parse_problem/Main ─
 
-                    I cannot find the end of this list:
+                    I am partway through started parsing a list, but I got stuck here:
 
                     3│  main = [
-                                ^
+                    4│
+                    5│
+                        ^
 
-                    You could change it to something like [1, 2, 3] or even just [].
-                    Anything where there is an open and a close square bracket, and where
-                    the elements of the list are separated by commas.
+                    I was expecting to see a closing square bracket before this, so try
+                    adding a ] and see if that helps?
 
-                    Note: I may be confused by indentation"
+                    Note: When I get stuck like this, it usually means that there is a
+                    missing parenthesis or bracket somewhere earlier. It could also be a
+                    stray keyword or operator."
             )
         ),
         Ok(_) => unreachable!("we expect failure here"),
@@ -646,7 +657,8 @@ fn platform_does_not_exist() {
 
     match multiple_modules("platform_does_not_exist", modules) {
         Err(report) => {
-            assert!(report.contains("FILE NOT FOUND"), "report=({})", report);
+            // TODO restore this assert once it can pass.
+            // assert!(report.contains("FILE NOT FOUND"), "report=({})", report);
             assert!(
                 report.contains("zzz-does-not-exist/main.roc"),
                 "report=({})",