From d2379bd71ad395891d45c307ca0e9f2174221e8b Mon Sep 17 00:00:00 2001 From: edouardparis Date: Thu, 28 Nov 2024 16:19:07 +0100 Subject: [PATCH 1/2] Use cargo-zigbuild for reproducible build with glibc 2.31 --- contrib/release/release.sh | 4 +- contrib/reproducible/guix/build.sh | 8 +- contrib/reproducible/guix/guix-build.sh | 2 +- contrib/reproducible/guix/manifest.scm | 358 +++++++++++++----------- 4 files changed, 206 insertions(+), 166 deletions(-) diff --git a/contrib/release/release.sh b/contrib/release/release.sh index 8e4e6e61..00ee6dde 100755 --- a/contrib/release/release.sh +++ b/contrib/release/release.sh @@ -38,13 +38,13 @@ TARGET_DIR="$BUILD_DIR" ./contrib/reproducible/docker/docker-build.sh ( cd "$BUILD_DIR" create_dir "$LINUX_DIR_NAME" - cp "$BUILD_DIR/release/lianad" "$BUILD_DIR/release/liana-cli" "$BUILD_DIR/release/liana-gui" ../README.md "$LINUX_DIR_NAME" + cp "$BUILD_DIR/x86_64-unknown-linux-gnu/release/lianad" "$BUILD_DIR/x86_64-unknown-linux-gnu/release/liana-cli" "$BUILD_DIR/x86_64-unknown-linux-gnu/release/liana-gui" ../README.md "$LINUX_DIR_NAME" tar -czf "$LINUX_ARCHIVE" "$LINUX_DIR_NAME" cp "$LINUX_ARCHIVE" "$RELEASE_DIR" unzip ../contrib/release/debian/package.zip sed -i "s/VERSION_PLACEHOLDER/$VERSION/g" ./package/DEBIAN/control - cp "$BUILD_DIR/release/lianad" "$BUILD_DIR/release/liana-cli" "$BUILD_DIR/release/liana-gui" ../README.md ./package/usr/bin/ + cp "$BUILD_DIR/x86_64-unknown-linux-gnu/release/lianad" "$BUILD_DIR/x86_64-unknown-linux-gnu/release/liana-cli" "$BUILD_DIR/x86_64-unknown-linux-gnu/release/liana-gui" ../README.md ./package/usr/bin/ DIRNAME="liana_$VERSION-1_amd64" mv ./package "$DIRNAME" dpkg-deb -Zxz --build "$DIRNAME" diff --git a/contrib/reproducible/guix/build.sh b/contrib/reproducible/guix/build.sh index 5d50833d..08b6e824 100755 --- a/contrib/reproducible/guix/build.sh +++ b/contrib/reproducible/guix/build.sh @@ -26,13 +26,13 @@ export CARGO_HOME="/liana/.cargo" # We need to set RUSTC_BOOTSTRAP=1 as a workaround to be able to use unstable # features in the GUI dependencies for package_name in "lianad" "liana-gui"; do - RUSTC_BOOTSTRAP=1 cargo -vvv \ + RUSTC_BOOTSTRAP=1 cargo zigbuild -vvv \ --color always \ --frozen \ --offline \ - rustc \ -p "$package_name" \ --jobs "$JOBS" \ + --target x86_64-unknown-linux-gnu.2.31 \ --release \ --target-dir "/out" done @@ -40,10 +40,10 @@ done for bin_name in "liana-gui" "lianad" "liana-cli"; do # Assume 64bits. Even bitcoind doesn't ship 32bits binaries for x86. # FIXME: is there a cleaner way than using patchelf for this? - patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 "/out/release/$bin_name" + patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 "/out/x86_64-unknown-linux-gnu/release/$bin_name" # FIXME: Find a way to use GUIX_LD_WRAPPER_DISABLE_RPATH=yes instead - patchelf --remove-rpath "/out/release/$bin_name" + patchelf --remove-rpath "/out/x86_64-unknown-linux-gnu/release/$bin_name" done set +ex diff --git a/contrib/reproducible/guix/guix-build.sh b/contrib/reproducible/guix/guix-build.sh index 6cdd380f..0217cbbe 100755 --- a/contrib/reproducible/guix/guix-build.sh +++ b/contrib/reproducible/guix/guix-build.sh @@ -48,7 +48,7 @@ fi # across time. time_machine() { guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=059d38dc3f8b087f4a42df586daeb05761ee18d7 \ + --commit=3ef5e20bcdb6caed49e5db46a135ee4c17d69b5f \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/reproducible/guix/manifest.scm b/contrib/reproducible/guix/manifest.scm index 46d89637..e7f78aab 100644 --- a/contrib/reproducible/guix/manifest.scm +++ b/contrib/reproducible/guix/manifest.scm @@ -1,183 +1,223 @@ (use-modules (gnu packages llvm) (gnu packages rust) + (gnu packages base) + (gnu packages crates-io) + (guix build-system cargo) + ((guix licenses) #:prefix license:) + (guix download) (guix packages) (guix utils)) -;; Stolen from Bitcoin Core. Some patches are needed to bootstrap a newer rustc. -(define-syntax-rule (search-our-patches file-name ...) - "Return the list of absolute file names corresponding to each -FILE-NAME found in ./patches relative to the current file." - (parameterize - ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) - (list (search-patch file-name) ...))) +(define-public rust-goblin-0.9 + (package + (name "rust-goblin") + (version "0.9.2") + (source (origin + (method url-fetch) + (uri (crate-uri "goblin" version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + "08yrnjj5j4nddh6y1r8kf35ys7p3iwg6npga3nc4cwfps4r3zask")))) + (build-system cargo-build-system) + (arguments + `(#:cargo-inputs (("rust-log" ,rust-log-0.4) + ("rust-plain" ,rust-plain-0.2) + ("rust-scroll" ,rust-scroll-0.12)) + #:cargo-development-inputs (("rust-stderrlog" ,rust-stderrlog-0.5)))) + (home-page "https://github.com/m4b/goblin") + (synopsis + "An impish, cross-platform, ELF, Mach-o, and PE binary parsing and loading crate") + (description + "An impish, cross-platform, ELF, Mach-o, and PE binary parsing and loading crate") + (license license:expat))) -;; What follows are the declaration of more recent rustc versions from the current Guix master -;; (https://git.savannah.gnu.org/cgit/guix.git/tree/gnu/packages/rust.scm), which we can't use -;; because it would require an unreasonable glibc version. +(define-public rust-fat-macho-0.4 + (package + (name "rust-fat-macho") + (version "0.4.9") + (source (origin + (method url-fetch) + (uri (crate-uri "fat-macho" version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + "0idkn366wipv2l757yqfgzgibqc6jvm89gdk9kpgmvf6lv54b72c")))) + (build-system cargo-build-system) + (arguments + `(#:cargo-inputs (("rust-goblin" ,rust-goblin-0.9) + ("rust-llvm-bitcode" ,rust-llvm-bitcode-0.1)))) + (home-page "https://github.com/messense/fat-macho-rs.git") + (synopsis "Mach-O Fat Binary Reader and Writer") + (description "Mach-O Fat Binary Reader and Writer") + (license license:expat))) -(define rust-bootstrapped-package - (@@ (gnu packages rust) rust-bootstrapped-package)) +(define-public rust-rustflags-0.1 + (package + (name "rust-rustflags") + (version "0.1.6") + (source (origin + (method url-fetch) + (uri (crate-uri "rustflags" version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + "1h1al0xhd9kzy8q8lzw6rxip5zjifxigfrm3blf462mmkwar5z6p")))) + (build-system cargo-build-system) + (arguments + `(#:cargo-development-inputs (("rust-cmake" ,rust-cmake-0.1)))) + (home-page "https://github.com/dtolnay/rustflags") + (synopsis "Parser for CARGO_ENCODED_RUSTFLAGS") + (description "Parser for CARGO_ENCODED_RUSTFLAGS") + (license (list license:expat license:asl2.0)))) -(define %cargo-reference-hash - (@@ (gnu packages rust) %cargo-reference-hash)) +(define-public rust-cargo-config2-0.1 + (package + (name "rust-cargo-config2") + (version "0.1.29") + (source + (origin + (method url-fetch) + (uri (crate-uri "cargo-config2" version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 "15cmkvxad1b33bvx2d1h7bnjlbvq1lpyi5jybk0jq9mrxi5ha90i")))) + (build-system cargo-build-system) + (arguments + `(#:cargo-inputs (("rust-home" ,rust-home-0.5) + ("rust-serde" ,rust-serde-1) + ("rust-serde-derive" ,rust-serde-derive-1) + ("rust-toml-edit" ,rust-toml-edit-0.22)) + #:cargo-development-inputs (("rust-anyhow" ,rust-anyhow-1) + ("rust-build-context" ,rust-build-context-0.1) + ("rust-clap" ,rust-clap-4) + ("rust-duct" ,rust-duct-0.13) + ("rust-fs-err" ,rust-fs-err-2) + ("rust-lexopt" ,rust-lexopt-0.3) + ("rust-rustversion" ,rust-rustversion-1) + ("rust-serde-json" ,rust-serde-json-1) + ("rust-shell-escape" ,rust-shell-escape-0.1) + ("rust-static-assertions" ,rust-static-assertions-1) + ("rust-tempfile" ,rust-tempfile-3) + ("rust-toml" ,rust-toml-0.8)))) + (home-page "https://github.com/taiki-e/cargo-config2") + (synopsis "Load and resolve Cargo configuration.") + (description "This package provides Load and resolve Cargo configuration.") + (license (list license:asl2.0 license:expat)))) -(define rust-1.65 - (@@ (gnu packages rust) rust-1.65)) +(define-public rust-cargo-options-0.7 + (package + (name "rust-cargo-options") + (version "0.7.4") + (source + (origin + (method url-fetch) + (uri (crate-uri "cargo-options" version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 "07rj2hkqyan8c2jmvfa35ji6wy0pqs6j7k2a6bmpcym3q13h4m7k")))) + (build-system cargo-build-system) + (arguments + `(#:cargo-inputs (("rust-anstyle" ,rust-anstyle-1) + ("rust-clap" ,rust-clap-4)) + #:cargo-development-inputs (("rust-trycmd" ,rust-trycmd-0.14)))) + (home-page "https://github.com/messense/cargo-options") + (synopsis "Reusable common Cargo command line options") + (description + "This package provides Reusable common Cargo command line options.") + (license license:expat))) -(define rust-1.66 - (rust-bootstrapped-package - rust-1.65 "1.66.1" "1fjr94gsicsxd2ypz4zm8aad1zdbiccr7qjfbmq8f8f7jhx96g2v")) +(define-public rust-crc-3 + (package + (name "rust-crc") + (version "3.2.1") + (source + (origin + (method url-fetch) + (uri (crate-uri "crc" version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 "0dnn23x68qakzc429s1y9k9y3g8fn5v9jwi63jcz151sngby9rk9")))) + (build-system cargo-build-system) + (arguments + `(#:cargo-inputs (("rust-crc-catalog" ,rust-crc-catalog-2)) + #:cargo-development-inputs (("rust-criterion" ,rust-criterion-0.4)))) + (home-page "https://github.com/mrhooray/crc-rs.git") + (synopsis "Rust implementation of CRC with support of various standards") + (description + "This package provides Rust implementation of CRC with support of various standards.") + (license (list license:expat license:asl2.0)))) -(define rust-1.67 - (let ((base-rust - (rust-bootstrapped-package - rust-1.66 "1.67.1" "0vpzv6rm3w1wbni17ryvcw83k5klhghklylfdza3nnp8blz3sj26"))) - (package - (inherit base-rust) - (source - (origin - (inherit (package-source base-rust)) - (snippet - '(begin - (for-each delete-file-recursively - '("src/llvm-project" - "vendor/openssl-src/openssl" - "vendor/tikv-jemalloc-sys/jemalloc")) - ;; Adjust rustix to always build with cc. - (substitute* '("Cargo.lock" - "src/bootstrap/Cargo.lock") - (("\"errno\",") "\"cc\",\n \"errno\",")) - ;; Add a dependency on the the 'cc' feature of rustix. - (substitute* "vendor/fd-lock/Cargo.toml" - (("\"fs\"") "\"fs\", \"cc\"")) - (substitute* "vendor/is-terminal/Cargo.toml" - (("\"termios\"") "\"termios\", \"cc\"")) - ;; Remove vendored dynamically linked libraries. - ;; find . -not -type d -executable -exec file {} \+ | grep ELF - (delete-file "vendor/vte/vim10m_match") - (delete-file "vendor/vte/vim10m_table") - ;; Also remove the bundled (mostly Windows) libraries. - (for-each delete-file - (find-files "vendor" "\\.(a|dll|exe|lib)$")))))) - (inputs (modify-inputs (package-inputs base-rust) - (replace "llvm" llvm-15)))))) +(define-public rust-target-lexicon-0.12 + (package + (name "rust-target-lexicon") + (version "0.12.16") + (source + (origin + (method url-fetch) + (uri (crate-uri "target-lexicon" version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 "1cg3bnx1gdkdr5hac1hzxy64fhw4g7dqkd0n3dxy5lfngpr1mi31")))) + (build-system cargo-build-system) + (arguments + `(#:cargo-inputs (("rust-serde" ,rust-serde-1)) + #:cargo-development-inputs (("rust-serde-json" ,rust-serde-json-1)))) + (home-page "https://github.com/bytecodealliance/target-lexicon") + (synopsis "Targeting utilities for compilers and related tools") + (description + "This package provides Targeting utilities for compilers and related tools.") + (license license:asl2.0))) -(define rust-1.68 - (rust-bootstrapped-package - rust-1.67 "1.68.2" "15ifyd5jj8rd979dkakp887hgmhndr68pqaqvd2hqkfdywirqcwk")) -(define rust-1.69 - (let ((base-rust - (rust-bootstrapped-package - rust-1.68 "1.69.0" - "03zn7kx5bi5mdfsqfccj4h8gd6abm7spj0kjsfxwlv5dcwc9f1gv"))) - (package - (inherit base-rust) - (source - (origin - (inherit (package-source base-rust)) - (snippet - '(begin - (for-each delete-file-recursively - '("src/llvm-project" - "vendor/openssl-src/openssl" - "vendor/tikv-jemalloc-sys/jemalloc")) - ;; Adjust rustix to always build with cc. - (substitute* '("Cargo.lock" - "src/bootstrap/Cargo.lock") - (("\"errno\",") "\"cc\",\n \"errno\",")) - ;; Add a dependency on the the 'cc' feature of rustix. - (substitute* "vendor/fd-lock/Cargo.toml" - (("\"fs\"") "\"fs\", \"cc\"")) - (substitute* "vendor/is-terminal/Cargo.toml" - (("\"termios\"") "\"termios\", \"cc\"")) - ;; Also remove the bundled (mostly Windows) libraries. - (for-each delete-file - (find-files "vendor" "\\.(a|dll|exe|lib)$"))))))))) - -(define rust-1.70 - (let ((base-rust - (rust-bootstrapped-package - rust-1.69 "1.70.0" - "0z6j7d0ni0rmfznv0w3mrf882m11kyh51g2bxkj40l3s1c0axgxj"))) - (package - (inherit base-rust) - (source - (origin - (inherit (package-source base-rust)) - (snippet - '(begin - (for-each delete-file-recursively - '("src/llvm-project" - "vendor/openssl-src/openssl" - "vendor/tikv-jemalloc-sys/jemalloc")) - ;; Adjust rustix to always build with cc. - (substitute* "Cargo.lock" - (("\"errno\",") "\"cc\",\n \"errno\",")) - ;; Add a dependency on the the 'cc' feature of rustix. - (substitute* '("vendor/is-terminal/Cargo.toml" - "vendor/is-terminal-0.4.4/Cargo.toml") - (("\"termios\"") "\"termios\", \"cc\"")) - ;; Also remove the bundled (mostly Windows) libraries. - (for-each delete-file - (find-files "vendor" "\\.(a|dll|exe|lib)$")))) - ;; Rust 1.70 adds the rustix library which depends on the vendored - ;; fd-lock crate. The fd-lock crate uses Outline assembly which expects - ;; a precompiled static library. Enabling the "cc" feature tells the - ;; build.rs script to compile the assembly files instead of searching - ;; for a precompiled library. - (patches (search-our-patches "rust-1.70-fix-rustix-build.patch"))))))) - -(define rust-1.71 - (let ((base-rust - (rust-bootstrapped-package - rust-1.70 "1.71.1" "0bj79syjap1kgpg9pc0r4jxc0zkxwm6phjf3digsfafms580vabg"))) - (package - (inherit base-rust) - (source - (origin - (inherit (package-source base-rust)) - (snippet - '(begin - (for-each delete-file-recursively - '("src/llvm-project" - "vendor/openssl-src/openssl" - "vendor/tikv-jemalloc-sys/jemalloc")) - ;; Adjust rustix to always build with cc. - (substitute* '("Cargo.lock" - "src/tools/cargo/Cargo.lock") - (("\"errno\",") "\"cc\",\n \"errno\",")) - ;; Add a dependency on the the 'cc' feature of rustix. - (substitute* '("vendor/is-terminal/Cargo.toml" - "vendor/is-terminal-0.4.6/Cargo.toml") - (("\"termios\"") "\"termios\", \"cc\"")) - ;; Also remove the bundled (mostly Windows) libraries. - (for-each delete-file - (find-files "vendor" "\\.(a|dll|exe|lib)$")))))) - (arguments - (substitute-keyword-arguments (package-arguments base-rust) - ((#:phases phases) - `(modify-phases ,phases - (replace 'patch-cargo-checksums - (lambda _ - (substitute* (cons* "Cargo.lock" - "src/bootstrap/Cargo.lock" - (find-files "src/tools" "Cargo.lock")) - (("(checksum = )\".*\"" all name) - (string-append name "\"" ,%cargo-reference-hash "\""))) - (generate-all-checksums "vendor")))))))))) - -;; END of the newer rustc versions copied over from the current Guix master. +(define-public rust-cargo-zigbuild-0.19 + (package + (name "rust-cargo-zigbuild") + (version "0.19.4") + (source + (origin + (method url-fetch) + (uri (crate-uri "cargo-zigbuild" version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 "0r4pdjvj7iz928gyyj569y7rwaz9mwddaz0s0a0cggiybmandpi0")))) + (build-system cargo-build-system) + (arguments + `(#:cargo-inputs (("rust-anyhow" ,rust-anyhow-1) + ("rust-cargo-config2" ,rust-cargo-config2-0.1) + ("rust-cargo-options" ,rust-cargo-options-0.7) + ("rust-cargo-metadata" ,rust-cargo-metadata-0.18) + ("rust-clap" ,rust-clap-4) + ("rust-crc" ,rust-crc-3) + ("rust-dirs" ,rust-dirs-5) + ("rust-fat-macho" ,rust-fat-macho-0.4) + ("rust-fs-err" ,rust-fs-err-2) + ("rust-path-slash" ,rust-path-slash-0.2) + ("rust-rustc-version" ,rust-rustc-version-0.4) + ("rust-rustflags" ,rust-rustflags-0.1) + ("rust-semver" ,rust-semver-1) + ("rust-serde" ,rust-serde-1) + ("rust-serde-json" ,rust-serde-json-1) + ("rust-shlex" ,rust-shlex-1) + ("rust-target-lexicon" ,rust-target-lexicon-0.12) + ("rust-which" ,rust-which-6)))) + (home-page "https://github.com/rust-cross/cargo-zigbuild") + (synopsis "Compile Cargo project with zig as linker") + (description + "This package provides Compile Cargo project with zig as linker.") + (license license:expat))) (concatenate-manifests (list (specifications->manifest (list + "rust" "rust:cargo" + "zig" "coreutils-minimal" "patchelf" - "gcc-toolchain@10.3.0" + "gcc-toolchain" "pkg-config" "eudev" "fontconfig")) @@ -187,4 +227,4 @@ FILE-NAME found in ./patches relative to the current file." ;; resort to backporting the newer rustc releases here. Also have proper Guix packages ;; for the two projects. (packages->manifest - `(,rust-1.71)))) + `(,rust-cargo-zigbuild-0.19)))) From e3acfa7f806a0694d29adb9369984c25760840e3 Mon Sep 17 00:00:00 2001 From: edouardparis Date: Wed, 4 Dec 2024 15:03:24 +0100 Subject: [PATCH 2/2] Update guix commit: rust 1.82 --- contrib/reproducible/guix/guix-build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/reproducible/guix/guix-build.sh b/contrib/reproducible/guix/guix-build.sh index 0217cbbe..6d19b5e2 100755 --- a/contrib/reproducible/guix/guix-build.sh +++ b/contrib/reproducible/guix/guix-build.sh @@ -31,7 +31,7 @@ maybe_create_dir "$BIN_DIR" # That's what Guix comes with. RUST_VERSION="1.60.0" -CARGO_BIN="$BIN_DIR/cargo" +CARGO_BIN="$(which cargo)" # First off get the cargo binary to run on the host to vendor dependencies. # We assume the host is a 64bit Linux system. @@ -41,6 +41,8 @@ if ! [ -f "$CARGO_BIN" ]; then echo "b8a4c3959367d053825e31f90a5eb86418eb0d80cacda52bfa80b078e18150d5 $ARCHIVE_PATH" | $SHASUM_BIN -c # Path of the cargo binary within the archive CARGO_BIN_PATH="rust-$RUST_VERSION-x86_64-unknown-linux-gnu/cargo/bin/cargo" + CARGO_BIN="$BIN/cargo" + ( cd $BIN_DIR && tar -xzf $ARCHIVE_PATH $CARGO_BIN_PATH && mv $CARGO_BIN_PATH $CARGO_BIN ) fi @@ -48,7 +50,7 @@ fi # across time. time_machine() { guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=3ef5e20bcdb6caed49e5db46a135ee4c17d69b5f \ + --commit=1c4a00820a1ba6265d2d96f4f7804d0807d69dcc \ --cores="$JOBS" \ --keep-failed \ --fallback \