feat(nix): make buildRocPackage fetch dependencies recursively (#7729)

* update nix buildRocPackage script

* change list of prefetched urls to include all links from the repository

* update nix buildRocPackage script to also include gz files

add debug echo

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

* use descriptive variable names and add allowed domains list

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

* add fizzbuzz to examples folder

* limit fizzbuzz range to 10

* include buildRocPackage tests in flake checks

* revert formatter changes

* extend range to contain FizzBuzz

* run nix fmt

* add flake checks to CI

* hello world + fix warnings

* fix unnecessary deps

Signed-off-by: Anton-4 <17049058+Anton-4@users.noreply.github.com>

* misc fixes

* nix fixes

---------

Signed-off-by: Anton-4 <17049058+Anton-4@users.noreply.github.com>
Co-authored-by: Anton-4 <17049058+Anton-4@users.noreply.github.com>

.github/workflows/nix_linux_x86_64.yml

@@ -17,6 +17,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - run: nix flake check --impure
+
       - name: test building default.nix
         run: nix-build
 

devtools/flake.lock

@@ -75,13 +75,13 @@
         "rust-overlay": "rust-overlay"
       },
       "locked": {
-        "lastModified": 1738668249,
+        "lastModified": 1744991948,
-        "narHash": "sha256-VsJbSkTfh9gooNfEWEfrvfcukdg5xKpUXr186T0dtUI=",
+        "narHash": "sha256-YJ66VPH3Cm5vL356CNkAbFsoOB+uAeK9ZTSCcMOBFrU=",
-        "path": "/home/username/gitrepos/roc",
+        "path": "/home/username/gitrepos/forks/nxy7/roc",
         "type": "path"
       },
       "original": {
-        "path": "/home/username/gitrepos/roc",
+        "path": "/home/username/gitrepos/forks/nxy7/roc",
         "type": "path"
       }
     },

devtools/flake.nix

@@ -19,7 +19,7 @@
 
         isAarch64Darwin = pkgs.stdenv.hostPlatform.system == "aarch64-darwin";
 
-        rocShell = roc.devShell.${system};
+        rocShell = roc.devShells.${system}.default;
       in
       {
         devShell = pkgs.mkShell {

flake.nix

@@ -2,7 +2,8 @@
   description = "Roc flake";
 
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs?rev=184957277e885c06a505db112b35dfbec7c60494";
+    nixpkgs.url =
+      "github:nixos/nixpkgs?rev=184957277e885c06a505db112b35dfbec7c60494";
 
     # rust from nixpkgs has some libc problems, this is patched in the rust-overlay
     rust-overlay = {
@@ -21,29 +22,34 @@
 
   outputs = { self, nixpkgs, rust-overlay, flake-utils, ... }@inputs:
     let
-      supportedSystems = [ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" "aarch64-linux" ];
+      supportedSystems =
+        [ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" "aarch64-linux" ];
 
       templates = import ./nix/templates { };
-    in {
+      buildRocPackage = import ./nix/buildRocPackage.nix;
+    in
+    {
       inherit templates;
-      lib = { buildRocPackage = import ./nix/buildRocPackage.nix; };
+      lib = { inherit buildRocPackage; };
-    } //
+    } // flake-utils.lib.eachSystem supportedSystems (system:
-    flake-utils.lib.eachSystem supportedSystems (system:
       let
 
-        overlays = [ (import rust-overlay) ]
+        overlays = [ (import rust-overlay) ] ++ [
-        ++ [(final: prev: {
+          (final: prev: {
-          # using a custom simple-http-server fork because of github.com/TheWaWaR/simple-http-server/issues/111
+            # using a custom simple-http-server fork because of github.com/TheWaWaR/simple-http-server/issues/111
-          # the server is used for local testing of the roc website
+            # the server is used for local testing of the roc website
-          simple-http-server = final.callPackage ./nix/simple-http-server.nix { };
+            simple-http-server =
-        })];
+              final.callPackage ./nix/simple-http-server.nix { };
+          })
+        ];
         pkgs = import nixpkgs { inherit system overlays; };
 
         rocBuild = import ./nix { inherit pkgs; };
 
         compile-deps = rocBuild.compile-deps;
-        inherit (compile-deps) zigPkg llvmPkgs llvmVersion
+        inherit (compile-deps)
-          llvmMajorMinorStr glibcPath libGccSPath darwinInputs;
+          zigPkg llvmPkgs llvmVersion llvmMajorMinorStr glibcPath libGccSPath
+          darwinInputs;
 
         # DevInputs are not necessary to build roc as a user
         linuxDevInputs = with pkgs;
@@ -55,10 +61,10 @@
 
         # DevInputs are not necessary to build roc as a user
         darwinDevInputs = with pkgs;
-          lib.optionals stdenv.isDarwin
+          lib.optionals stdenv.isDarwin (with pkgs.darwin.apple_sdk.frameworks;
-            (with pkgs.darwin.apple_sdk.frameworks; [
+          [
-              curl # for wasm-bindgen-cli libcurl (see ./ci/www-repl.sh)
+            curl # for wasm-bindgen-cli libcurl (see ./ci/www-repl.sh)
-            ]);
+          ]);
 
         sharedInputs = (with pkgs; [
           # build libraries
@@ -94,7 +100,7 @@
           jq # used in several bash scripts
           cargo-nextest # used to give more info for segfaults for gen tests
           # cargo-udeps # to find unused dependencies
-           
+
           zls # zig language server
           watchexec
           simple-http-server # to view the website locally
@@ -109,8 +115,10 @@
       in
       {
 
-        devShell = pkgs.mkShell {
+        devShells = {
-          buildInputs = sharedInputs ++ sharedDevInputs ++ darwinInputs ++ darwinDevInputs ++ linuxDevInputs;
+          default = pkgs.mkShell {
+          buildInputs = sharedInputs ++ sharedDevInputs ++ darwinInputs
+          ++ darwinDevInputs ++ linuxDevInputs;
 
           # nix does not store libs in /usr/lib or /lib
           # for libgcc_s.so.1
@@ -120,7 +128,7 @@
           NIX_GLIBC_PATH =
             if pkgs.stdenv.isLinux then "${pkgs.glibc.out}/lib" else "";
 
-          LD_LIBRARY_PATH =  with pkgs;
+          LD_LIBRARY_PATH = with pkgs;
             lib.makeLibraryPath
               ([ pkg-config stdenv.cc.cc.lib libffi ncurses zlib ]
               ++ linuxDevInputs);
@@ -134,6 +142,7 @@
             unset NIX_LDFLAGS
           '';
         };
+      };
 
         formatter = pkgs.nixpkgs-fmt;
 
@@ -155,7 +164,43 @@
           default = {
             type = "app";
             program = "${rocBuild.roc-cli}/bin/roc";
+            meta = {
+              description = "Roc CLI";
+              mainProgram = "roc";
+            };
           };
         };
+
+        # test for nix/buildRocPackage.nix
+        checks.canBuildRocPackage =
+          let
+            helloWorldPackage = buildRocPackage {
+              inherit pkgs;
+              roc-cli = rocBuild.roc-cli;
+              linker = "legacy";
+              name = "helloworld";
+              optimize = true;
+              # use `src = ./myfolder;` for local usage
+              src = pkgs.fetchFromGitHub {
+                owner = "roc-lang";
+                repo = "examples";
+                rev = "main";
+                sha256 = "sha256-DqqkA5iASoK68XBFKv6Gbrso4687smKz8PqVUL2rRsE=";
+              };
+              entryPoint = "./examples/HelloWorld/main.roc";
+              outputHash = "sha256-Hg1K3tNE2hdz9o9f2HEB0aEuBIBoXrlpb70h6uyOABo=";
+            };
+          in
+          pkgs.runCommand "build helloworld" { } ''
+            expectedOutput="Hello, World!"
+            actualOutput=$(${helloWorldPackage}/bin/helloworld)
+            if [ "$actualOutput" = "$expectedOutput" ]; then
+              echo "helloworld output is correct."
+              touch $out
+            else
+              echo "helloworld output is incorrect, I expected '$expectedOutput' but got '$actualOutput'" >&2
+              exit 1
+            fi
+          '';
       });
 }

nix/buildRocPackage.nix

@@ -1,41 +1,141 @@
-{ pkgs, roc-cli, name, entryPoint, src, outputHash, linker ? "", ... }:
+
+{ pkgs
+, roc-cli
+, name
+, entryPoint
+, src
+, outputHash
+, linker ? ""
+, optimize ? true
+, ...
+}:
+# see checks.canBuildRocPackage in /flake.nix for example usage
 let
   packageDependencies = pkgs.stdenv.mkDerivation {
-    inherit src;
+    inherit src outputHash;
     name = "roc-dependencies";
     nativeBuildInputs = with pkgs; [ gnutar brotli ripgrep wget cacert ];
 
     buildPhase = ''
-      list=$(rg -o 'https://github.com[^"]*' ${entryPoint})
+      set -x
-      for url in $list; do
+
-        path=$(echo $url | awk -F'github.com/|/[^/]*$' '{print $2}')
+      declare -A visitedUrls
-        packagePath=$out/roc/packages/github.com/$path
+
-        mkdir -p $packagePath
+      # function that recursively prefetches roc dependencies
-        wget -P $packagePath $url
+      # so they're available during roc build stage
-        cd $packagePath
+      function prefetch () {
-        brotli -d *.tar.br
+        local searchPath=$1
-        tar -xf *.tar --one-top-level
+        local skipApp=$2 # to skip any code gen app files in dependencies
-        rm *.tar.br
+
-        rm *.tar
+        echo "Searching for deps in $searchPath"
-      done
+        
+        # Set default value for skipApp if not provided
+        if [ -z "$skipApp" ]; then
+          skipApp=false
+        fi
+
+        local dependenciesRegexp='https://[^"]*tar.br|https://[^"]*tar.gz'
+        
+        # If skipApp is true, exclude files containing app declarations
+        if [ "$skipApp" = true ]; then
+          # Find files containing app declarations
+          local appFiles=$(rg -l '^\s*app\s*\[' -IN $searchPath)
+          echo "appFiles $appFiles"
+          
+          # If app files were found, exclude them from search
+          if [ -n "$appFiles" ]; then
+            local filesWithUrls=$(rg -l "$dependenciesRegexp" -IN  $searchPath)
+
+            for appFile in $appFiles; do
+              local appFullPath=$(realpath "$appFile")
+              # Remove appFullPath from depsUrlsList
+              filesWithUrls=$(echo "$filesWithUrls" | grep -vxF "$appFullPath" || true)
+            done
+
+            if [ -n "$filesWithUrls" ]; then
+              local depsUrlsList=$(rg -o "$dependenciesRegexp" -IN  $filesWithUrls)
+            else
+              return 0
+            fi
+          else
+            local depsUrlsList=$(rg -o "$dependenciesRegexp" -IN  $searchPath)
+          fi
+        else
+          local depsUrlsList=$(rg -o "$dependenciesRegexp" -IN  $searchPath)
+        fi
+
+        depsUrlsList=$(echo "$depsUrlsList" | tr ' ' '\n' | sort | uniq | tr '\n' ' ' | sed 's/ $//')
+        echo "depsUrlsList: $depsUrlsList"
+
+        if [ -z "$depsUrlsList" ]; then
+          echo "No dependency URLs need to be downloaded."
+          return 0
+        fi
+
+        for url in $depsUrlsList; do
+          if [[ -n "''${visitedUrls["$url"]^^}" ]]; then
+            echo "Skipping already visited URL: $url"
+          else
+            echo "Prefetching $url"
+            visitedUrls["$url"]=1
+
+            local domain=$(echo $url | awk -F '/' '{print $3}')
+
+            local packagePath=$(echo $url | awk -F "$domain/|/[^/]*$" '{print $2}')
+            local outputPackagePath="$out/roc/packages/$domain/$packagePath"
+            echo "Package path: $outputPackagePath"
+
+            mkdir -p "$outputPackagePath"
+
+            # Download dependency
+            if ! (wget -P "$outputPackagePath" "$url" 2>/tmp/wget_error); then
+              echo "WARNING: Failed to download $url: $(cat /tmp/wget_error)"
+              exit 1
+            fi
+
+            # Unpack dependency
+            if [[ $url == *.br ]]; then
+              brotli -d "$outputPackagePath"/*.tar.br
+              tar -xf "$outputPackagePath"/*.tar --one-top-level -C $outputPackagePath
+            elif [[ $url == *.gz ]]; then
+              tar -xzf "$outputPackagePath"/*.tar.gz --one-top-level -C $outputPackagePath
+            fi
+
+            # Delete temporary files
+            rm "$outputPackagePath"/*tar*
+
+            # Recursively fetch dependencies of dependencies
+            # Pass the skipApp parameter to recursive calls
+            prefetch "$outputPackagePath" true
+          fi
+        done
+      }
+
+      prefetch ${src} false
+
+      if [ -d "$out/roc/packages" ]; then
+        echo "Successfully prefetched packages:"
+        find "$out/roc/packages" -type d -mindepth 3 -maxdepth 3 | sort
+      else
+        echo "WARNING: No packages were prefetched. This might indicate a problem."
+      fi
     '';
 
     outputHashMode = "recursive";
     outputHashAlgo = "sha256";
-    outputHash = outputHash;
   };
-in pkgs.stdenv.mkDerivation {
+in
+pkgs.stdenv.mkDerivation {
   inherit name src;
   nativeBuildInputs = [ roc-cli ];
   XDG_CACHE_HOME = packageDependencies;
 
   buildPhase = ''
-    roc build ${entryPoint} --output ${name} --optimize ${
+    roc build ${entryPoint} --output ${name} \
-      if linker != "" then "--linker=${linker}" else ""
+    ${if optimize == true then "--optimize" else ""} \
-    }
+    ${if linker != "" then "--linker=${linker}" else ""}
 
     mkdir -p $out/bin
     mv ${name} $out/bin/${name}
   '';
 }
-

shell.nix

@@ -2,4 +2,4 @@
 # This file is kept to maintain compatibility with tools like lorri until they support flakes (https://github.com/target/lorri/issues/460).
 { system ? builtins.currentSystem }:
 
-(builtins.getFlake (toString ./.)).devShell.${system}
+(builtins.getFlake (toString ./.)).devShells.${system}