diff --git a/compiler/load/src/file.rs b/compiler/load/src/file.rs
index fc6177c4bd..842b23799e 100644
--- a/compiler/load/src/file.rs
+++ b/compiler/load/src/file.rs
@@ -3773,12 +3773,7 @@ fn make_specializations<'a>(
     // TODO: for now this final specialization pass is sequential,
     // with no parallelization at all. We should try to parallelize
     // this, but doing so will require a redesign of Procs.
-    procs = roc_mono::ir::specialize_all(
-        &mut mono_env,
-        procs,
-        &mut layout_cache,
-        // &finished_info.vars_by_symbol,
-    );
+    procs = roc_mono::ir::specialize_all(&mut mono_env, procs, &mut layout_cache);
 
     let external_specializations_requested = procs.externals_we_need.clone();
     let procedures = procs.get_specialized_procs_without_rc(mono_env.arena);
diff --git a/compiler/mono/src/ir.rs b/compiler/mono/src/ir.rs
index bf1aad9bd0..58267ef608 100644
--- a/compiler/mono/src/ir.rs
+++ b/compiler/mono/src/ir.rs
@@ -558,13 +558,7 @@ impl<'a> Procs<'a> {
                                     }
                                 }
                                 Err(error) => {
-                                    let error_msg = format!(
-                                        "TODO generate a RuntimeError message for {:?}",
-                                        error
-                                    );
-                                    self.runtime_errors
-                                        .insert(symbol, env.arena.alloc(error_msg));
-                                    panic!();
+                                    panic!("TODO generate a RuntimeError message for {:?}", error);
                                 }
                             }
                         }
@@ -671,11 +665,7 @@ impl<'a> Procs<'a> {
                         self.specialized.insert((symbol, layout), Done(proc));
                     }
                     Err(error) => {
-                        let error_msg =
-                            format!("TODO generate a RuntimeError message for {:?}", error);
-                        self.runtime_errors
-                            .insert(symbol, env.arena.alloc(error_msg));
-                        panic!();
+                        panic!("TODO generate a RuntimeError message for {:?}", error);
                     }
                 }
             }
@@ -699,7 +689,6 @@ fn add_pending<'a>(
 #[derive(Default)]
 pub struct Specializations<'a> {
     by_symbol: MutMap<Symbol, MutMap<Layout<'a>, Proc<'a>>>,
-    runtime_errors: MutSet<Symbol>,
 }
 
 impl<'a> Specializations<'a> {
@@ -715,32 +704,15 @@ impl<'a> Specializations<'a> {
             !procs_by_layout.contains_key(&layout) || procs_by_layout.get(&layout) == Some(&proc)
         );
 
-        // We shouldn't already have a runtime error recorded for this symbol
-        debug_assert!(!self.runtime_errors.contains(&symbol));
-
         procs_by_layout.insert(layout, proc);
     }
 
-    pub fn runtime_error(&mut self, symbol: Symbol) {
-        // We shouldn't already have a normal proc recorded for this symbol
-        debug_assert!(!self.by_symbol.contains_key(&symbol));
-
-        self.runtime_errors.insert(symbol);
-    }
-
-    pub fn into_owned(self) -> (MutMap<Symbol, MutMap<Layout<'a>, Proc<'a>>>, MutSet<Symbol>) {
-        (self.by_symbol, self.runtime_errors)
-    }
-
     pub fn len(&self) -> usize {
-        let runtime_errors: usize = self.runtime_errors.len();
-        let specializations: usize = self.by_symbol.len();
-
-        runtime_errors + specializations
+        self.by_symbol.len()
     }
 
     pub fn is_empty(&self) -> bool {
-        self.len() == 0
+        self.by_symbol.is_empty()
     }
 }
 
@@ -1696,13 +1668,15 @@ pub fn specialize_all<'a>(
             Ok((proc, layout)) => {
                 procs.specialized.insert((name, layout), Done(proc));
             }
-            Err(error) => {
-                let error_msg = env.arena.alloc(format!(
-                    "TODO generate a RuntimeError message for {:?}",
-                    error
-                ));
+            Err(SpecializeFailure {
+                problem: _,
+                attempted_layout,
+            }) => {
+                let proc = generate_runtime_error_function(env, name, attempted_layout);
 
-                procs.runtime_errors.insert(name, error_msg);
+                procs
+                    .specialized
+                    .insert((name, attempted_layout), Done(proc));
             }
         }
     }
@@ -1758,13 +1732,14 @@ pub fn specialize_all<'a>(
                             procs.specialized.insert((name, layout), Done(proc));
                         }
                     }
-                    Err(error) => {
-                        let error_msg = env.arena.alloc(format!(
-                            "TODO generate a RuntimeError message for {:?}",
-                            error
-                        ));
+                    Err(SpecializeFailure {
+                        attempted_layout, ..
+                    }) => {
+                        let proc = generate_runtime_error_function(env, name, attempted_layout);
 
-                        procs.runtime_errors.insert(name, error_msg);
+                        procs
+                            .specialized
+                            .insert((name, attempted_layout), Done(proc));
                     }
                 }
             }
@@ -1774,6 +1749,47 @@ pub fn specialize_all<'a>(
     procs
 }
 
+fn generate_runtime_error_function<'a>(
+    env: &mut Env<'a, '_>,
+    name: Symbol,
+    layout: Layout<'a>,
+) -> Proc<'a> {
+    let (arg_layouts, ret_layout) = match layout {
+        Layout::FunctionPointer(a, r) => (a, *r),
+        _ => (&[] as &[_], layout),
+    };
+
+    let mut args = Vec::with_capacity_in(arg_layouts.len(), env.arena);
+
+    for arg in arg_layouts {
+        args.push((*arg, env.unique_symbol()));
+    }
+
+    let mut msg = bumpalo::collections::string::String::with_capacity_in(80, env.arena);
+    use std::fmt::Write;
+    write!(
+        &mut msg,
+        "The {:?} function could not be generated, likely due to a type error.",
+        name
+    )
+    .unwrap();
+
+    eprintln!("emitted runtime error function {:?}", &msg);
+
+    let runtime_error = Stmt::RuntimeError(msg.into_bump_str());
+
+    Proc {
+        name,
+        args: args.into_bump_slice(),
+        body: runtime_error,
+        closure_data_layout: None,
+        ret_layout,
+        is_self_recursive: SelfRecursive::NotSelfRecursive,
+        must_own_arguments: false,
+        host_exposed_layouts: HostExposedLayouts::NotHostExposed,
+    }
+}
+
 fn specialize_external<'a>(
     env: &mut Env<'a, '_>,
     procs: &mut Procs<'a>,
@@ -2270,6 +2286,14 @@ fn build_specialized_proc<'a>(
     }
 }
 
+#[derive(Debug)]
+struct SpecializeFailure<'a> {
+    /// The layout we attempted to create
+    attempted_layout: Layout<'a>,
+    /// The problem we ran into while creating it
+    problem: LayoutProblem,
+}
+
 fn specialize<'a>(
     env: &mut Env<'a, '_>,
     procs: &mut Procs<'a>,
@@ -2277,7 +2301,7 @@ fn specialize<'a>(
     layout_cache: &mut LayoutCache<'a>,
     pending: PendingSpecialization,
     partial_proc: PartialProc<'a>,
-) -> Result<(Proc<'a>, Layout<'a>), LayoutProblem> {
+) -> Result<(Proc<'a>, Layout<'a>), SpecializeFailure<'a>> {
     let PendingSpecialization {
         solved_type,
         host_exposed_aliases,
@@ -2321,7 +2345,7 @@ fn specialize_solved_type<'a>(
     solved_type: SolvedType,
     host_exposed_aliases: MutMap<Symbol, SolvedType>,
     partial_proc: PartialProc<'a>,
-) -> Result<(Proc<'a>, Layout<'a>), LayoutProblem> {
+) -> Result<(Proc<'a>, Layout<'a>), SpecializeFailure<'a>> {
     // add the specializations that other modules require of us
     use roc_solve::solve::instantiate_rigids;
 
@@ -2330,6 +2354,10 @@ fn specialize_solved_type<'a>(
 
     let fn_var = introduce_solved_type_to_subs(env, &solved_type);
 
+    let attempted_layout = layout_cache
+        .from_var(&env.arena, fn_var, env.subs)
+        .unwrap_or_else(|err| panic!("TODO handle invalid function {:?}", err));
+
     // make sure rigid variables in the annotation are converted to flex variables
     instantiate_rigids(env.subs, partial_proc.annotation);
 
@@ -2353,17 +2381,25 @@ fn specialize_solved_type<'a>(
 
     match specialized {
         Ok(proc) => {
-            let layout = layout_cache
-                .from_var(&env.arena, fn_var, env.subs)
-                .unwrap_or_else(|err| panic!("TODO handle invalid function {:?}", err));
+            // when successful, the layout after unification should be the layout before unification
+            debug_assert_eq!(
+                attempted_layout,
+                layout_cache
+                    .from_var(&env.arena, fn_var, env.subs)
+                    .unwrap_or_else(|err| panic!("TODO handle invalid function {:?}", err))
+            );
+
             env.subs.rollback_to(snapshot);
             layout_cache.rollback_to(cache_snapshot);
-            Ok((proc, layout))
+            Ok((proc, attempted_layout))
         }
         Err(error) => {
             env.subs.rollback_to(snapshot);
             layout_cache.rollback_to(cache_snapshot);
-            Err(error)
+            Err(SpecializeFailure {
+                problem: error,
+                attempted_layout,
+            })
         }
     }
 }
@@ -5951,6 +5987,20 @@ fn call_by_name<'a>(
 
     // Register a pending_specialization for this function
     match layout_cache.from_var(env.arena, fn_var, env.subs) {
+        Err(LayoutProblem::UnresolvedTypeVar(var)) => {
+            let msg = format!(
+                "Hit an unresolved type variable {:?} when creating a layout for {:?} (var {:?})",
+                var, proc_name, fn_var
+            );
+            Stmt::RuntimeError(env.arena.alloc(msg))
+        }
+        Err(LayoutProblem::Erroneous) => {
+            let msg = format!(
+                "Hit an erroneous type when creating a layout for {:?}",
+                proc_name
+            );
+            Stmt::RuntimeError(env.arena.alloc(msg))
+        }
         Ok(layout) => {
             // Build the CallByName node
             let arena = env.arena;
@@ -6086,123 +6136,42 @@ fn call_by_name<'a>(
                                             "\n\n{:?}\n\n{:?}",
                                             full_layout, layout
                                         );
-                                        let function_layout =
-                                            FunctionLayouts::from_layout(env.arena, layout);
 
-                                        procs.specialized.remove(&(proc_name, full_layout));
-
-                                        procs
-                                            .specialized
-                                            .insert((proc_name, function_layout.full), Done(proc));
-
-                                        if field_symbols.is_empty() {
-                                            debug_assert!(loc_args.is_empty());
-
-                                            // This happens when we return a function, e.g.
-                                            //
-                                            // foo = Num.add
-                                            //
-                                            // Even though the layout (and type) are functions,
-                                            // there are no arguments. This confuses our IR,
-                                            // and we have to fix it here.
-                                            match full_layout {
-                                                Layout::Closure(_, closure_layout, _) => {
-                                                    let call = self::Call {
-                                                        call_type: CallType::ByName {
-                                                            name: proc_name,
-                                                            ret_layout: function_layout.result,
-                                                            full_layout: function_layout.full,
-                                                            arg_layouts: function_layout.arguments,
-                                                        },
-                                                        arguments: field_symbols,
-                                                    };
-
-                                                    // in the case of a closure specifically, we
-                                                    // have to create a custom layout, to make sure
-                                                    // the closure data is part of the layout
-                                                    let closure_struct_layout = Layout::Struct(
-                                                        env.arena.alloc([
-                                                            function_layout.full,
-                                                            closure_layout
-                                                                .as_block_of_memory_layout(),
-                                                        ]),
-                                                    );
-
-                                                    build_call(
-                                                        env,
-                                                        call,
-                                                        assigned,
-                                                        closure_struct_layout,
-                                                        hole,
-                                                    )
-                                                }
-                                                _ => {
-                                                    let call = self::Call {
-                                                        call_type: CallType::ByName {
-                                                            name: proc_name,
-                                                            ret_layout: function_layout.result,
-                                                            full_layout: function_layout.full,
-                                                            arg_layouts: function_layout.arguments,
-                                                        },
-                                                        arguments: field_symbols,
-                                                    };
-
-                                                    build_call(
-                                                        env,
-                                                        call,
-                                                        assigned,
-                                                        function_layout.full,
-                                                        hole,
-                                                    )
-                                                }
-                                            }
-                                        } else {
-                                            debug_assert_eq!(
-                                                function_layout.arguments.len(),
-                                                field_symbols.len(),
-                                                "scroll up a bit for background"
-                                            );
-                                            let call = self::Call {
-                                                call_type: CallType::ByName {
-                                                    name: proc_name,
-                                                    ret_layout: function_layout.result,
-                                                    full_layout: function_layout.full,
-                                                    arg_layouts: function_layout.arguments,
-                                                },
-                                                arguments: field_symbols,
-                                            };
-
-                                            let iter = loc_args
-                                                .into_iter()
-                                                .rev()
-                                                .zip(field_symbols.iter().rev());
-
-                                            let result = build_call(
-                                                env,
-                                                call,
-                                                assigned,
-                                                function_layout.result,
-                                                hole,
-                                            );
-
-                                            assign_to_symbols(
-                                                env,
-                                                procs,
-                                                layout_cache,
-                                                iter,
-                                                result,
-                                            )
-                                        }
+                                        call_specialized_proc(
+                                            env,
+                                            procs,
+                                            proc_name,
+                                            proc,
+                                            layout,
+                                            field_symbols,
+                                            loc_args,
+                                            layout_cache,
+                                            assigned,
+                                            hole,
+                                        )
                                     }
-                                    Err(error) => {
-                                        let error_msg = env.arena.alloc(format!(
-                                            "TODO generate a RuntimeError message for {:?}",
-                                            error
-                                        ));
+                                    Err(SpecializeFailure {
+                                        attempted_layout,
+                                        problem: _,
+                                    }) => {
+                                        let proc = generate_runtime_error_function(
+                                            env,
+                                            proc_name,
+                                            attempted_layout,
+                                        );
 
-                                        procs.runtime_errors.insert(proc_name, error_msg);
-
-                                        Stmt::RuntimeError(error_msg)
+                                        call_specialized_proc(
+                                            env,
+                                            procs,
+                                            proc_name,
+                                            proc,
+                                            layout,
+                                            field_symbols,
+                                            loc_args,
+                                            layout_cache,
+                                            assigned,
+                                            hole,
+                                        )
                                     }
                                 }
                             }
@@ -6249,33 +6218,104 @@ fn call_by_name<'a>(
                             }
 
                             None => {
-                                // This must have been a runtime error.
-                                match procs.runtime_errors.get(&proc_name) {
-                                    Some(error) => Stmt::RuntimeError(
-                                        env.arena.alloc(format!("runtime error {:?}", error)),
-                                    ),
-                                    None => unreachable!("Proc name {:?} is invalid", proc_name),
-                                }
+                                unreachable!("Proc name {:?} is invalid", proc_name)
                             }
                         }
                     }
                 }
             }
         }
-        Err(LayoutProblem::UnresolvedTypeVar(var)) => {
-            let msg = format!(
-                "Hit an unresolved type variable {:?} when creating a layout for {:?} (var {:?})",
-                var, proc_name, fn_var
-            );
-            Stmt::RuntimeError(env.arena.alloc(msg))
-        }
-        Err(LayoutProblem::Erroneous) => {
-            let msg = format!(
-                "Hit an erroneous type when creating a layout for {:?}",
-                proc_name
-            );
-            Stmt::RuntimeError(env.arena.alloc(msg))
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+fn call_specialized_proc<'a>(
+    env: &mut Env<'a, '_>,
+    procs: &mut Procs<'a>,
+    proc_name: Symbol,
+    proc: Proc<'a>,
+    layout: Layout<'a>,
+    field_symbols: &'a [Symbol],
+    loc_args: std::vec::Vec<(Variable, Located<roc_can::expr::Expr>)>,
+    layout_cache: &mut LayoutCache<'a>,
+    assigned: Symbol,
+    hole: &'a Stmt<'a>,
+) -> Stmt<'a> {
+    let function_layout = FunctionLayouts::from_layout(env.arena, layout);
+
+    procs.specialized.remove(&(proc_name, layout));
+
+    procs
+        .specialized
+        .insert((proc_name, function_layout.full), Done(proc));
+
+    if field_symbols.is_empty() {
+        debug_assert!(loc_args.is_empty());
+
+        // This happens when we return a function, e.g.
+        //
+        // foo = Num.add
+        //
+        // Even though the layout (and type) are functions,
+        // there are no arguments. This confuses our IR,
+        // and we have to fix it here.
+        match layout {
+            Layout::Closure(_, closure_layout, _) => {
+                let call = self::Call {
+                    call_type: CallType::ByName {
+                        name: proc_name,
+                        ret_layout: function_layout.result,
+                        full_layout: function_layout.full,
+                        arg_layouts: function_layout.arguments,
+                    },
+                    arguments: field_symbols,
+                };
+
+                // in the case of a closure specifically, we
+                // have to create a custom layout, to make sure
+                // the closure data is part of the layout
+                let closure_struct_layout = Layout::Struct(env.arena.alloc([
+                    function_layout.full,
+                    closure_layout.as_block_of_memory_layout(),
+                ]));
+
+                build_call(env, call, assigned, closure_struct_layout, hole)
+            }
+            _ => {
+                let call = self::Call {
+                    call_type: CallType::ByName {
+                        name: proc_name,
+                        ret_layout: function_layout.result,
+                        full_layout: function_layout.full,
+                        arg_layouts: function_layout.arguments,
+                    },
+                    arguments: field_symbols,
+                };
+
+                build_call(env, call, assigned, function_layout.full, hole)
+            }
         }
+    } else {
+        debug_assert_eq!(
+            function_layout.arguments.len(),
+            field_symbols.len(),
+            "scroll up a bit for background"
+        );
+        let call = self::Call {
+            call_type: CallType::ByName {
+                name: proc_name,
+                ret_layout: function_layout.result,
+                full_layout: function_layout.full,
+                arg_layouts: function_layout.arguments,
+            },
+            arguments: field_symbols,
+        };
+
+        let iter = loc_args.into_iter().rev().zip(field_symbols.iter().rev());
+
+        let result = build_call(env, call, assigned, function_layout.result, hole);
+
+        assign_to_symbols(env, procs, layout_cache, iter, result)
     }
 }
 
diff --git a/compiler/mono/tests/test_mono.rs b/compiler/mono/tests/test_mono.rs
index e476947adf..739712a76c 100644
--- a/compiler/mono/tests/test_mono.rs
+++ b/compiler/mono/tests/test_mono.rs
@@ -341,13 +341,13 @@ mod test_mono {
             "#,
             indoc!(
                 r#"
-                procedure Num.46 (#Attr.2):
+                procedure Num.47 (#Attr.2):
                     let Test.3 = lowlevel NumRound #Attr.2;
                     ret Test.3;
 
                 procedure Test.0 ():
                     let Test.2 = 3.6f64;
-                    let Test.1 = CallByName Num.46 Test.2;
+                    let Test.1 = CallByName Num.47 Test.2;
                     ret Test.1;
                 "#
             ),
diff --git a/compiler/test_gen/src/gen_num.rs b/compiler/test_gen/src/gen_num.rs
index a0ed06f72e..8d55fdc7a8 100644
--- a/compiler/test_gen/src/gen_num.rs
+++ b/compiler/test_gen/src/gen_num.rs
@@ -404,6 +404,7 @@ mod gen_num {
         );
     }
 
+    #[test]
     fn f64_sqrt_zero() {
         assert_evals_to!(
             indoc!(
diff --git a/compiler/test_gen/src/gen_primitives.rs b/compiler/test_gen/src/gen_primitives.rs
index aa277f8a6a..edb65f4dd9 100644
--- a/compiler/test_gen/src/gen_primitives.rs
+++ b/compiler/test_gen/src/gen_primitives.rs
@@ -2276,3 +2276,23 @@ fn function_malformed_pattern() {
         i64
     );
 }
+
+#[test]
+#[should_panic(expected = "Hit an erroneous type when creating a layout for")]
+fn call_invalid_layout() {
+    assert_llvm_evals_to!(
+        indoc!(
+            r#"
+                f : I64 -> I64
+                f = \x -> x
+
+                f {}
+            "#
+        ),
+        3,
+        i64,
+        |x| x,
+        false,
+        true
+    );
+}
diff --git a/compiler/test_gen/src/helpers/eval.rs b/compiler/test_gen/src/helpers/eval.rs
index 3302ce5fcb..b5fcf9124a 100644
--- a/compiler/test_gen/src/helpers/eval.rs
+++ b/compiler/test_gen/src/helpers/eval.rs
@@ -37,6 +37,7 @@ pub fn helper<'a>(
     src: &str,
     stdlib: &'a roc_builtins::std::StdLib,
     leak: bool,
+    ignore_problems: bool,
     context: &'a inkwell::context::Context,
 ) -> (&'static str, String, Library) {
     use roc_gen::llvm::build::{build_proc, build_proc_header, Scope};
@@ -170,7 +171,7 @@ pub fn helper<'a>(
         println!("{}", lines.join("\n"));
 
         // only crash at this point if there were no delayed_errors
-        if delayed_errors.is_empty() {
+        if delayed_errors.is_empty() && !ignore_problems {
             assert_eq!(0, 1, "Mistakes were made");
         }
     }
@@ -331,7 +332,7 @@ pub fn helper<'a>(
 
 #[macro_export]
 macro_rules! assert_llvm_evals_to {
-    ($src:expr, $expected:expr, $ty:ty, $transform:expr, $leak:expr) => {
+    ($src:expr, $expected:expr, $ty:ty, $transform:expr, $leak:expr, $ignore_problems:expr) => {
         use bumpalo::Bump;
         use inkwell::context::Context;
         use roc_gen::run_jit_function;
@@ -343,7 +344,7 @@ macro_rules! assert_llvm_evals_to {
         let stdlib = arena.alloc(roc_builtins::std::standard_stdlib());
 
         let (main_fn_name, errors, lib) =
-            $crate::helpers::eval::helper(&arena, $src, stdlib, $leak, &context);
+            $crate::helpers::eval::helper(&arena, $src, stdlib, $leak, $ignore_problems, &context);
 
         let transform = |success| {
             let expected = $expected;
@@ -354,7 +355,7 @@ macro_rules! assert_llvm_evals_to {
     };
 
     ($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
-        assert_llvm_evals_to!($src, $expected, $ty, $transform, true);
+        assert_llvm_evals_to!($src, $expected, $ty, $transform, true, false);
     };
 }
 
@@ -375,7 +376,7 @@ macro_rules! assert_evals_to {
         // parsing the source, so that there's no chance their passing
         // or failing depends on leftover state from the previous one.
         {
-            assert_llvm_evals_to!($src, $expected, $ty, $transform, $leak);
+            assert_llvm_evals_to!($src, $expected, $ty, $transform, $leak, false);
         }
         {
             // NOTE at the moment, the optimized tests do the same thing
@@ -392,7 +393,7 @@ macro_rules! assert_non_opt_evals_to {
     ($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
         // Same as above, except with an additional transformation argument.
         {
-            assert_llvm_evals_to!($src, $expected, $ty, $transform, true);
+            assert_llvm_evals_to!($src, $expected, $ty, $transform, true, false);
         }
     };
     ($src:expr, $expected:expr, $ty:ty, $transform:expr, $leak:expr) => {{