diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs
index 3f06fbab10..f4beb5f5ef 100644
--- a/compiler/build/src/link.rs
+++ b/compiler/build/src/link.rs
@@ -291,13 +291,25 @@ pub fn rebuild_host(host_input_path: &Path) {
     }
 }
 
-fn env_glibc_path() -> PathBuf {
-    let glibc_env = env::var("GLIBC_PATH").unwrap_or_else(|_| panic!("We couldn't find glibc!"));
-    let glibc_path = Path::new(&glibc_env).to_path_buf();
-    if glibc_path.exists() {
-        glibc_path
+fn nixos_path() -> String {
+    env::var("NIXOS_GLIBC_PATH").unwrap_or_else(|_| {
+        panic!("We couldn't find glibc! We tried looking for NIXOS_GLIBC_PATH
+to find it via Nix, but that didn't work either. Please file a bug report.
+
+This will only be an issue until we implement surgical linking.",
+        )
+    })
+}
+
+fn library_path<const N: usize>(segments: [&str; N]) -> Option<PathBuf> {
+    let mut guess_path = PathBuf::new();
+    for s in segments {
+        guess_path.push(s);
+    }
+    if guess_path.exists() {
+        Some(guess_path)
     } else {
-        panic!("We couldn't find glibc!")
+        None
     }
 }
 
@@ -307,39 +319,31 @@ fn link_linux(
     input_paths: &[&str],
     link_type: LinkType,
 ) -> io::Result<(Child, PathBuf)> {
-    let usr_lib_path = Path::new("/usr/lib").to_path_buf();
-    let link_root = if usr_lib_path.exists() {
-        usr_lib_path
-    } else {
-        env_glibc_path()
-    };
+    let architecture = format!("{}-linux-gnu", target.architecture);
 
-    let usr_lib_gnu_path = link_root.join(format!("{}-linux-gnu", target.architecture));
-    let lib_gnu_path = Path::new("/lib/").join(format!("{}-linux-gnu", target.architecture));
-
-    let libcrt_path = if usr_lib_gnu_path.exists() {
-        &usr_lib_gnu_path
-    } else {
-        &link_root
-    };
+    let libcrt_path = library_path(["/usr", "lib", &architecture])
+        .or_else(|| library_path(["/usr", "lib"]))
+        .or_else(|| library_path([&nixos_path()]))
+        .unwrap();
 
     let libgcc_name = "libgcc_s.so.1";
-    let libgcc_path = if lib_gnu_path.join(libgcc_name).exists() {
-        lib_gnu_path.join(libgcc_name)
-    } else if usr_lib_gnu_path.join(libgcc_name).exists() {
-        usr_lib_gnu_path.join(libgcc_name)
-    } else {
-        link_root.join(libgcc_name)
-    };
+    let libgcc_path = library_path(["/lib", &architecture, libgcc_name])
+        .or_else(|| library_path(["/usr", "lib", &architecture, libgcc_name]))
+        .or_else(|| library_path(["/usr", "lib", libgcc_name]))
+        .or_else(|| library_path([&nixos_path(), libgcc_name]))
+        .unwrap();
 
     let ld_linux = match target.architecture {
-        Architecture::X86_64 => "/lib64/ld-linux-x86-64.so.2",
-        Architecture::Aarch64(_) => "/lib/ld-linux-aarch64.so.1",
+        Architecture::X86_64 => library_path(["/lib64", "ld-linux-x86-64.so.2"])
+            .or_else(|| library_path([&nixos_path(), "ld-linux-x86-64.so.2"])),
+        Architecture::Aarch64(_) => library_path(["/lib", "ld-linux-aarch64.so.1"]),
         _ => panic!(
             "TODO gracefully handle unsupported linux architecture: {:?}",
             target.architecture
         ),
     };
+    let ld_linux = ld_linux.unwrap();
+    let ld_linux = ld_linux.to_str().unwrap();
 
     let mut soname;
     let (base_args, output_path) = match link_type {
diff --git a/shell.nix b/shell.nix
index 183766f8b7..4cb95aa23e 100644
--- a/shell.nix
+++ b/shell.nix
@@ -71,7 +71,7 @@ in pkgs.mkShell {
 
   # Additional Env vars
   LLVM_SYS_120_PREFIX = "${llvmPkgs.llvm.dev}";
-  GLIBC_PATH = "${pkgs.glibc_multi.out}/lib";
+  NIXOS_GLIBC_PATH = "${pkgs.glibc_multi.out}/lib";
   LD_LIBRARY_PATH = with pkgs;
     lib.makeLibraryPath ([
       pkg-config