mirror of
https://github.com/roc-lang/roc.git
synced 2025-07-07 14:44:59 +00:00
feat(nix): make buildRocPackage fetch dependencies recursively (#7729)
* update nix buildRocPackage script * change list of prefetched urls to include all links from the repository * update nix buildRocPackage script to also include gz files add debug echo WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP * use descriptive variable names and add allowed domains list WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP WIP * add fizzbuzz to examples folder * limit fizzbuzz range to 10 * include buildRocPackage tests in flake checks * revert formatter changes * extend range to contain FizzBuzz * run nix fmt * add flake checks to CI * hello world + fix warnings * fix unnecessary deps Signed-off-by: Anton-4 <17049058+Anton-4@users.noreply.github.com> * misc fixes * nix fixes --------- Signed-off-by: Anton-4 <17049058+Anton-4@users.noreply.github.com> Co-authored-by: Anton-4 <17049058+Anton-4@users.noreply.github.com>
This commit is contained in:
Dawid Danieluk
GitHub
parent
1c86a3e346
commit
15f162f83c
6 changed files with 195 additions and 48 deletions
2
.github/workflows/nix_linux_x86_64.yml
vendored
2
.github/workflows/nix_linux_x86_64.yml
vendored
|
|
@ -17,6 +17,8 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- run: nix flake check --impure
|
||||
|
||||
- name: test building default.nix
|
||||
run: nix-build
|
||||
|
||||
|
|
|
|||
8
devtools/flake.lock
generated
8
devtools/flake.lock
generated
|
|
@ -75,13 +75,13 @@
|
|||
"rust-overlay": "rust-overlay"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1738668249,
|
||||
"narHash": "sha256-VsJbSkTfh9gooNfEWEfrvfcukdg5xKpUXr186T0dtUI=",
|
||||
"path": "/home/username/gitrepos/roc",
|
||||
"lastModified": 1744991948,
|
||||
"narHash": "sha256-YJ66VPH3Cm5vL356CNkAbFsoOB+uAeK9ZTSCcMOBFrU=",
|
||||
"path": "/home/username/gitrepos/forks/nxy7/roc",
|
||||
"type": "path"
|
||||
},
|
||||
"original": {
|
||||
"path": "/home/username/gitrepos/roc",
|
||||
"path": "/home/username/gitrepos/forks/nxy7/roc",
|
||||
"type": "path"
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
|
||||
isAarch64Darwin = pkgs.stdenv.hostPlatform.system == "aarch64-darwin";
|
||||
|
||||
rocShell = roc.devShell.${system};
|
||||
rocShell = roc.devShells.${system}.default;
|
||||
in
|
||||
{
|
||||
devShell = pkgs.mkShell {
|
||||
|
|
|
|||
89
flake.nix
89
flake.nix
|
|
@ -2,7 +2,8 @@
|
|||
description = "Roc flake";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:nixos/nixpkgs?rev=184957277e885c06a505db112b35dfbec7c60494";
|
||||
nixpkgs.url =
|
||||
"github:nixos/nixpkgs?rev=184957277e885c06a505db112b35dfbec7c60494";
|
||||
|
||||
# rust from nixpkgs has some libc problems, this is patched in the rust-overlay
|
||||
rust-overlay = {
|
||||
|
|
@ -21,29 +22,34 @@
|
|||
|
||||
outputs = { self, nixpkgs, rust-overlay, flake-utils, ... }@inputs:
|
||||
let
|
||||
supportedSystems = [ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" "aarch64-linux" ];
|
||||
supportedSystems =
|
||||
[ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" "aarch64-linux" ];
|
||||
|
||||
templates = import ./nix/templates { };
|
||||
in {
|
||||
buildRocPackage = import ./nix/buildRocPackage.nix;
|
||||
in
|
||||
{
|
||||
inherit templates;
|
||||
lib = { buildRocPackage = import ./nix/buildRocPackage.nix; };
|
||||
} //
|
||||
flake-utils.lib.eachSystem supportedSystems (system:
|
||||
lib = { inherit buildRocPackage; };
|
||||
} // flake-utils.lib.eachSystem supportedSystems (system:
|
||||
let
|
||||
|
||||
overlays = [ (import rust-overlay) ]
|
||||
++ [(final: prev: {
|
||||
# using a custom simple-http-server fork because of github.com/TheWaWaR/simple-http-server/issues/111
|
||||
# the server is used for local testing of the roc website
|
||||
simple-http-server = final.callPackage ./nix/simple-http-server.nix { };
|
||||
})];
|
||||
overlays = [ (import rust-overlay) ] ++ [
|
||||
(final: prev: {
|
||||
# using a custom simple-http-server fork because of github.com/TheWaWaR/simple-http-server/issues/111
|
||||
# the server is used for local testing of the roc website
|
||||
simple-http-server =
|
||||
final.callPackage ./nix/simple-http-server.nix { };
|
||||
})
|
||||
];
|
||||
pkgs = import nixpkgs { inherit system overlays; };
|
||||
|
||||
rocBuild = import ./nix { inherit pkgs; };
|
||||
|
||||
compile-deps = rocBuild.compile-deps;
|
||||
inherit (compile-deps) zigPkg llvmPkgs llvmVersion
|
||||
llvmMajorMinorStr glibcPath libGccSPath darwinInputs;
|
||||
inherit (compile-deps)
|
||||
zigPkg llvmPkgs llvmVersion llvmMajorMinorStr glibcPath libGccSPath
|
||||
darwinInputs;
|
||||
|
||||
# DevInputs are not necessary to build roc as a user
|
||||
linuxDevInputs = with pkgs;
|
||||
|
|
@ -55,10 +61,10 @@
|
|||
|
||||
# DevInputs are not necessary to build roc as a user
|
||||
darwinDevInputs = with pkgs;
|
||||
lib.optionals stdenv.isDarwin
|
||||
(with pkgs.darwin.apple_sdk.frameworks; [
|
||||
curl # for wasm-bindgen-cli libcurl (see ./ci/www-repl.sh)
|
||||
]);
|
||||
lib.optionals stdenv.isDarwin (with pkgs.darwin.apple_sdk.frameworks;
|
||||
[
|
||||
curl # for wasm-bindgen-cli libcurl (see ./ci/www-repl.sh)
|
||||
]);
|
||||
|
||||
sharedInputs = (with pkgs; [
|
||||
# build libraries
|
||||
|
|
@ -94,7 +100,7 @@
|
|||
jq # used in several bash scripts
|
||||
cargo-nextest # used to give more info for segfaults for gen tests
|
||||
# cargo-udeps # to find unused dependencies
|
||||
|
||||
|
||||
zls # zig language server
|
||||
watchexec
|
||||
simple-http-server # to view the website locally
|
||||
|
|
@ -109,8 +115,10 @@
|
|||
in
|
||||
{
|
||||
|
||||
devShell = pkgs.mkShell {
|
||||
buildInputs = sharedInputs ++ sharedDevInputs ++ darwinInputs ++ darwinDevInputs ++ linuxDevInputs;
|
||||
devShells = {
|
||||
default = pkgs.mkShell {
|
||||
buildInputs = sharedInputs ++ sharedDevInputs ++ darwinInputs
|
||||
++ darwinDevInputs ++ linuxDevInputs;
|
||||
|
||||
# nix does not store libs in /usr/lib or /lib
|
||||
# for libgcc_s.so.1
|
||||
|
|
@ -120,7 +128,7 @@
|
|||
NIX_GLIBC_PATH =
|
||||
if pkgs.stdenv.isLinux then "${pkgs.glibc.out}/lib" else "";
|
||||
|
||||
LD_LIBRARY_PATH = with pkgs;
|
||||
LD_LIBRARY_PATH = with pkgs;
|
||||
lib.makeLibraryPath
|
||||
([ pkg-config stdenv.cc.cc.lib libffi ncurses zlib ]
|
||||
++ linuxDevInputs);
|
||||
|
|
@ -134,6 +142,7 @@
|
|||
unset NIX_LDFLAGS
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
formatter = pkgs.nixpkgs-fmt;
|
||||
|
||||
|
|
@ -155,7 +164,43 @@
|
|||
default = {
|
||||
type = "app";
|
||||
program = "${rocBuild.roc-cli}/bin/roc";
|
||||
meta = {
|
||||
description = "Roc CLI";
|
||||
mainProgram = "roc";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# test for nix/buildRocPackage.nix
|
||||
checks.canBuildRocPackage =
|
||||
let
|
||||
helloWorldPackage = buildRocPackage {
|
||||
inherit pkgs;
|
||||
roc-cli = rocBuild.roc-cli;
|
||||
linker = "legacy";
|
||||
name = "helloworld";
|
||||
optimize = true;
|
||||
# use `src = ./myfolder;` for local usage
|
||||
src = pkgs.fetchFromGitHub {
|
||||
owner = "roc-lang";
|
||||
repo = "examples";
|
||||
rev = "main";
|
||||
sha256 = "sha256-DqqkA5iASoK68XBFKv6Gbrso4687smKz8PqVUL2rRsE=";
|
||||
};
|
||||
entryPoint = "./examples/HelloWorld/main.roc";
|
||||
outputHash = "sha256-Hg1K3tNE2hdz9o9f2HEB0aEuBIBoXrlpb70h6uyOABo=";
|
||||
};
|
||||
in
|
||||
pkgs.runCommand "build helloworld" { } ''
|
||||
expectedOutput="Hello, World!"
|
||||
actualOutput=$(${helloWorldPackage}/bin/helloworld)
|
||||
if [ "$actualOutput" = "$expectedOutput" ]; then
|
||||
echo "helloworld output is correct."
|
||||
touch $out
|
||||
else
|
||||
echo "helloworld output is incorrect, I expected '$expectedOutput' but got '$actualOutput'" >&2
|
||||
exit 1
|
||||
fi
|
||||
'';
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,41 +1,141 @@
|
|||
{ pkgs, roc-cli, name, entryPoint, src, outputHash, linker ? "", ... }:
|
||||
|
||||
{ pkgs
|
||||
, roc-cli
|
||||
, name
|
||||
, entryPoint
|
||||
, src
|
||||
, outputHash
|
||||
, linker ? ""
|
||||
, optimize ? true
|
||||
, ...
|
||||
}:
|
||||
# see checks.canBuildRocPackage in /flake.nix for example usage
|
||||
let
|
||||
packageDependencies = pkgs.stdenv.mkDerivation {
|
||||
inherit src;
|
||||
inherit src outputHash;
|
||||
name = "roc-dependencies";
|
||||
nativeBuildInputs = with pkgs; [ gnutar brotli ripgrep wget cacert ];
|
||||
|
||||
buildPhase = ''
|
||||
list=$(rg -o 'https://github.com[^"]*' ${entryPoint})
|
||||
for url in $list; do
|
||||
path=$(echo $url | awk -F'github.com/|/[^/]*$' '{print $2}')
|
||||
packagePath=$out/roc/packages/github.com/$path
|
||||
mkdir -p $packagePath
|
||||
wget -P $packagePath $url
|
||||
cd $packagePath
|
||||
brotli -d *.tar.br
|
||||
tar -xf *.tar --one-top-level
|
||||
rm *.tar.br
|
||||
rm *.tar
|
||||
done
|
||||
set -x
|
||||
|
||||
declare -A visitedUrls
|
||||
|
||||
# function that recursively prefetches roc dependencies
|
||||
# so they're available during roc build stage
|
||||
function prefetch () {
|
||||
local searchPath=$1
|
||||
local skipApp=$2 # to skip any code gen app files in dependencies
|
||||
|
||||
echo "Searching for deps in $searchPath"
|
||||
|
||||
# Set default value for skipApp if not provided
|
||||
if [ -z "$skipApp" ]; then
|
||||
skipApp=false
|
||||
fi
|
||||
|
||||
local dependenciesRegexp='https://[^"]*tar.br|https://[^"]*tar.gz'
|
||||
|
||||
# If skipApp is true, exclude files containing app declarations
|
||||
if [ "$skipApp" = true ]; then
|
||||
# Find files containing app declarations
|
||||
local appFiles=$(rg -l '^\s*app\s*\[' -IN $searchPath)
|
||||
echo "appFiles $appFiles"
|
||||
|
||||
# If app files were found, exclude them from search
|
||||
if [ -n "$appFiles" ]; then
|
||||
local filesWithUrls=$(rg -l "$dependenciesRegexp" -IN $searchPath)
|
||||
|
||||
for appFile in $appFiles; do
|
||||
local appFullPath=$(realpath "$appFile")
|
||||
# Remove appFullPath from depsUrlsList
|
||||
filesWithUrls=$(echo "$filesWithUrls" | grep -vxF "$appFullPath" || true)
|
||||
done
|
||||
|
||||
if [ -n "$filesWithUrls" ]; then
|
||||
local depsUrlsList=$(rg -o "$dependenciesRegexp" -IN $filesWithUrls)
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
local depsUrlsList=$(rg -o "$dependenciesRegexp" -IN $searchPath)
|
||||
fi
|
||||
else
|
||||
local depsUrlsList=$(rg -o "$dependenciesRegexp" -IN $searchPath)
|
||||
fi
|
||||
|
||||
depsUrlsList=$(echo "$depsUrlsList" | tr ' ' '\n' | sort | uniq | tr '\n' ' ' | sed 's/ $//')
|
||||
echo "depsUrlsList: $depsUrlsList"
|
||||
|
||||
if [ -z "$depsUrlsList" ]; then
|
||||
echo "No dependency URLs need to be downloaded."
|
||||
return 0
|
||||
fi
|
||||
|
||||
for url in $depsUrlsList; do
|
||||
if [[ -n "''${visitedUrls["$url"]^^}" ]]; then
|
||||
echo "Skipping already visited URL: $url"
|
||||
else
|
||||
echo "Prefetching $url"
|
||||
visitedUrls["$url"]=1
|
||||
|
||||
local domain=$(echo $url | awk -F '/' '{print $3}')
|
||||
|
||||
local packagePath=$(echo $url | awk -F "$domain/|/[^/]*$" '{print $2}')
|
||||
local outputPackagePath="$out/roc/packages/$domain/$packagePath"
|
||||
echo "Package path: $outputPackagePath"
|
||||
|
||||
mkdir -p "$outputPackagePath"
|
||||
|
||||
# Download dependency
|
||||
if ! (wget -P "$outputPackagePath" "$url" 2>/tmp/wget_error); then
|
||||
echo "WARNING: Failed to download $url: $(cat /tmp/wget_error)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Unpack dependency
|
||||
if [[ $url == *.br ]]; then
|
||||
brotli -d "$outputPackagePath"/*.tar.br
|
||||
tar -xf "$outputPackagePath"/*.tar --one-top-level -C $outputPackagePath
|
||||
elif [[ $url == *.gz ]]; then
|
||||
tar -xzf "$outputPackagePath"/*.tar.gz --one-top-level -C $outputPackagePath
|
||||
fi
|
||||
|
||||
# Delete temporary files
|
||||
rm "$outputPackagePath"/*tar*
|
||||
|
||||
# Recursively fetch dependencies of dependencies
|
||||
# Pass the skipApp parameter to recursive calls
|
||||
prefetch "$outputPackagePath" true
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
prefetch ${src} false
|
||||
|
||||
if [ -d "$out/roc/packages" ]; then
|
||||
echo "Successfully prefetched packages:"
|
||||
find "$out/roc/packages" -type d -mindepth 3 -maxdepth 3 | sort
|
||||
else
|
||||
echo "WARNING: No packages were prefetched. This might indicate a problem."
|
||||
fi
|
||||
'';
|
||||
|
||||
outputHashMode = "recursive";
|
||||
outputHashAlgo = "sha256";
|
||||
outputHash = outputHash;
|
||||
};
|
||||
in pkgs.stdenv.mkDerivation {
|
||||
in
|
||||
pkgs.stdenv.mkDerivation {
|
||||
inherit name src;
|
||||
nativeBuildInputs = [ roc-cli ];
|
||||
XDG_CACHE_HOME = packageDependencies;
|
||||
|
||||
buildPhase = ''
|
||||
roc build ${entryPoint} --output ${name} --optimize ${
|
||||
if linker != "" then "--linker=${linker}" else ""
|
||||
}
|
||||
roc build ${entryPoint} --output ${name} \
|
||||
${if optimize == true then "--optimize" else ""} \
|
||||
${if linker != "" then "--linker=${linker}" else ""}
|
||||
|
||||
mkdir -p $out/bin
|
||||
mv ${name} $out/bin/${name}
|
||||
'';
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,4 +2,4 @@
|
|||
# This file is kept to maintain compatibility with tools like lorri until they support flakes (https://github.com/target/lorri/issues/460).
|
||||
{ system ? builtins.currentSystem }:
|
||||
|
||||
(builtins.getFlake (toString ./.)).devShell.${system}
|
||||
(builtins.getFlake (toString ./.)).devShells.${system}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue