From 2c70aba2d573d80a9b4ed78c0914e3996a6352ca Mon Sep 17 00:00:00 2001 From: edouardparis Date: Fri, 29 Nov 2024 17:21:50 +0100 Subject: [PATCH 1/5] nix build and nix shells for release Cross compiling the crates for windows and macos --- contrib/reproducible/README.md | 56 ++++++++++++- doc/RELEASE.md | 7 ++ flake.lock | 55 +++++++++++++ flake.nix | 146 ++++++++++++++++++++++++++++++++- 4 files changed, 258 insertions(+), 6 deletions(-) diff --git a/contrib/reproducible/README.md b/contrib/reproducible/README.md index 0ca04d6b..76696304 100644 --- a/contrib/reproducible/README.md +++ b/contrib/reproducible/README.md @@ -1,5 +1,7 @@ # Liana reproducible builds +## Reproducible build of linux binaries with GUIX + Releases of Liana are built in a reproducible manner, providing an assurance the binary a user is going to run corresponds to the sources published. It enables the possibility for the user, or a third party, to audit the code being ran. @@ -13,5 +15,55 @@ builds [here](https://bootstrappable.org/). For instructions on bootstrappable builds of Linux releases, see the [`guix`](./guix) folder. -For instructions on reproducible builds of Windows and MacOS releases, see the [`docker`](./docker) -folder. + +## Reproducible build of windows and macos binaries with NIX + +You will have to install [Nix](https://nixos.org/download/#download-nix), a package manager. +We rely on nix flakes, so you may need to activate this feature by setting in your `~/.config/nix/nix.conf`: + +``` +experimental-features = nix-command flakes +``` + +### Windows + +Simply run: + +``` +nix build .#x86_64-pc-windows-gnu +``` + +Binary will be present in the `./result` folder. + + +### MACOS + +First you need get the MacOS SDK. It is required to be able to build the MacOS binaries. The `12_2` version is +required (`Xcode_12.2.xip`). You need to download it from Apple's website. An Apple ID and cookies +enabled for the hostname are required. You can create one for free. (Note it is illegal to +distribute the archive.) Once logged in you can use the [direct +link](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip) to download +the archive. Alternatively, go to 'Downloads', then 'More' and search for [`Xcode +12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2). +The `sha256sum` of the downloaded XIP archive should be +`28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`. + +Then you have to extract the SDK and add it to the nix store: + +``` +nix run github:edouardparis/unxip#unxip -- Xcode_12.2.xip Xcode_12.2 +cd Xcode_12.2 +nix-store --add-fixed --recursive sha256 Xcode.app +``` +It may take a long time. + +Then to compile binaries for new apple CPUs: +``` +nix build .#aarch64-apple-darwin +``` +Or for legacy CPUs: +``` +nix build .#x86_64-apple-darwin +``` + +Binaries will be present in the `./result` folder. diff --git a/doc/RELEASE.md b/doc/RELEASE.md index 71febd3e..fdea5385 100644 --- a/doc/RELEASE.md +++ b/doc/RELEASE.md @@ -44,3 +44,10 @@ This document details the release process. - Update the package managers with the new version. As of this writing we only update the [AUR package](https://aur.archlinux.org/packages/liana-bin) ourselves. - Celebrate. + +In order to build the release assets: + +``` +nix develop .#release +./contrib/release/release.sh +``` diff --git a/flake.lock b/flake.lock index 5ba6ab84..ad945ebb 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,41 @@ { "nodes": { + "crane": { + "locked": { + "lastModified": 1732407143, + "narHash": "sha256-qJOGDT6PACoX+GbNH2PPx2ievlmtT1NVeTB80EkRLys=", + "owner": "ipetkov", + "repo": "crane", + "rev": "f2b4b472983817021d9ffb60838b2b36b9376b20", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1732689334, + "narHash": "sha256-yKI1KiZ0+bvDvfPTQ1ZT3oP/nIu3jPYm4dnbRd6hYg4=", + "owner": "nix-community", + "repo": "fenix", + "rev": "a8a983027ca02b363dfc82fbe3f7d9548a8d3dce", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -36,10 +72,29 @@ }, "root": { "inputs": { + "crane": "crane", + "fenix": "fenix", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs" } }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1732633904, + "narHash": "sha256-7VKcoLug9nbAN2txqVksWHHJplqK9Ou8dXjIZAIYSGc=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "8d5e91c94f80c257ce6dbdfba7bd63a5e8a03fa6", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, "systems": { "locked": { "lastModified": 1681028828, diff --git a/flake.nix b/flake.nix index 09e5f470..9877d135 100644 --- a/flake.nix +++ b/flake.nix @@ -4,13 +4,124 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; flake-utils.url = "github:numtide/flake-utils"; + crane.url = "github:ipetkov/crane"; + fenix = { + url = "github:nix-community/fenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; - outputs = { self, nixpkgs, flake-utils, ... }: + outputs = { self, nixpkgs, flake-utils, crane, fenix, ... }: flake-utils.lib.eachDefaultSystem (system: let - pkgs = import nixpkgs { inherit system; }; - in { + pkgs = import nixpkgs { inherit system; config = { allowUnfree = true; };}; + + inherit (pkgs) lib; + + toolchain = with fenix.packages.${system}; + combine [ + minimal.rustc + minimal.cargo + targets.x86_64-pc-windows-gnu.latest.rust-std + targets.aarch64-apple-darwin.latest.rust-std + targets.x86_64-apple-darwin.latest.rust-std + ]; + + craneLib = (crane.mkLib pkgs).overrideToolchain toolchain; + commonBuildSettings = { + src = lib.fileset.toSource { + root = ./.; + fileset = lib.fileset.unions [ + (craneLib.fileset.commonCargoSources ./.) + (lib.fileset.maybeMissing ./liana-ui/static) + ]; + }; + strictDeps = true; + doCheck = false; + }; + + x86_64-pc-windows-gnu = craneLib.buildPackage { + inherit (commonBuildSettings) src strictDeps doCheck; + + CARGO_BUILD_TARGET = "x86_64-pc-windows-gnu"; + CARGO_BUILD_RUSTFLAGS = "-C link-arg=-Wl,--no-insert-timestamp"; + TARGET_CC = "${pkgs.pkgsCross.mingwW64.stdenv.cc}/bin/${pkgs.pkgsCross.mingwW64.stdenv.cc.targetPrefix}cc"; + + pname = "liana-gui"; + cargoExtraArgs = "-p liana-gui"; + depsBuildBuild = with pkgs; [ + pkgsCross.mingwW64.stdenv.cc + pkgsCross.mingwW64.windows.pthreads + ]; + + installPhaseCommand = '' + mkdir -p $out/x86_64-pc-windows-gnu + cp target/x86_64-pc-windows-gnu/release/liana-gui.exe $out/x86_64-pc-windows-gnu + ''; + }; + + x86_64-apple-darwin = craneLib.buildPackage { + inherit (commonBuildSettings) src strictDeps doCheck; + + CARGO_BUILD_TARGET = "x86_64-apple-darwin"; + buildPhaseCargoCommand = "cargo zigbuild --release --message-format json-render-diagnostics"; + + depsBuildBuild = [ + pkgs.zig + pkgs.cargo-zigbuild + pkgs.darwin.xcode_12_2 + ]; + + preBuild = '' + export SDKROOT=${pkgs.darwin.xcode_12_2}/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk + + export XDG_CACHE_HOME=$TMPDIR/xdg_cache + mkdir -p $XDG_CACHE_HOME + export CARGO_ZIGBUILD_CACHE_DIR=$TMPDIR/cargo-zigbuild-cache + mkdir -p $CARGO_ZIGBUILD_CACHE_DIR + export CC=zigcc + export CXX=zigc++ + ''; + + installPhaseCommand = '' + mkdir -p $out/x86_64-apple-darwin + cp target/x86_64-apple-darwin/release/liana-gui $out/x86_64-apple-darwin + cp target/x86_64-apple-darwin/release/lianad $out/x86_64-apple-darwin + cp target/x86_64-apple-darwin/release/liana-cli $out/x86_64-apple-darwin + ''; + }; + + aarch64-apple-darwin = craneLib.buildPackage { + inherit (commonBuildSettings) src strictDeps doCheck; + + CARGO_BUILD_TARGET = "aarch64-apple-darwin"; + buildPhaseCargoCommand = "cargo zigbuild --release --message-format json-render-diagnostics"; + + depsBuildBuild = [ + pkgs.zig + pkgs.cargo-zigbuild + pkgs.darwin.xcode_12_2 + ]; + + preBuild = '' + export SDKROOT=${pkgs.darwin.xcode_12_2}/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk + + export XDG_CACHE_HOME=$TMPDIR/xdg_cache + mkdir -p $XDG_CACHE_HOME + export CARGO_ZIGBUILD_CACHE_DIR=$TMPDIR/cargo-zigbuild-cache + mkdir -p $CARGO_ZIGBUILD_CACHE_DIR + export CC=zigcc + export CXX=zigc++ + ''; + + installPhaseCommand = '' + mkdir -p $out/aarch64-apple-darwin + cp target/aarch64-apple-darwin/release/liana-gui $out/aarch64-apple-darwin + cp target/aarch64-apple-darwin/release/lianad $out/aarch64-apple-darwin + cp target/aarch64-apple-darwin/release/liana-cli $out/aarch64-apple-darwin + ''; + }; + devShell = pkgs.mkShell rec { buildInputs = with pkgs; [ expat @@ -31,7 +142,34 @@ LD_LIBRARY_PATH = builtins.foldl' (a: b: "${a}:${b}/lib") "${pkgs.vulkan-loader}/lib" buildInputs; }; + + releaseShell = pkgs.mkShell { + buildInputs = with pkgs; [ + zip + unzip + gnutar + dpkg + rcodesign + ]; + }; + + in { + packages = { + x86_64-pc-windows-gnu = x86_64-pc-windows-gnu; + x86_64-apple-darwin = x86_64-apple-darwin; + aarch64-apple-darwin = aarch64-apple-darwin; + release = pkgs.buildEnv { + name = "release"; + paths = [ x86_64-pc-windows-gnu x86_64-apple-darwin aarch64-apple-darwin ]; + }; + }; + + + devShells = { + dev = devShell; + release = releaseShell; + default = devShell; + }; } ); } - From a2917c5bd73e230d9b2cb13ec25250bd36b6d5ad Mon Sep 17 00:00:00 2001 From: edouardparis Date: Wed, 4 Dec 2024 17:21:26 +0100 Subject: [PATCH 2/5] Add SOURCE_DATE_EPOCH and TAR_OPTIONS for reproducibility close #1474 close #1479 --- contrib/release/release.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/release/release.sh b/contrib/release/release.sh index 00ee6dde..fc7e52e0 100755 --- a/contrib/release/release.sh +++ b/contrib/release/release.sh @@ -25,6 +25,10 @@ create_dir() { test -d "$1" || mkdir "$1" } +# Determine the reference time used for determinism (overridable by environment) +export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" + # We'll use a folder for the builds output and another one for the final assets. RELEASE_DIR="$PWD/release_assets" BUILD_DIR="$PWD/release_build" From 34e6afd54351178fdfae6eca01ee24e744d21fa1 Mon Sep 17 00:00:00 2001 From: edouardparis Date: Tue, 3 Dec 2024 18:41:12 +0100 Subject: [PATCH 3/5] Add script contrib/release/sign.sh for signing binaries --- .gitignore | 3 ++ contrib/release/release.sh | 56 +++++++++------------- contrib/release/sign.sh | 98 ++++++++++++++++++++++++++++++++++++++ flake.nix | 15 +++--- 4 files changed, 132 insertions(+), 40 deletions(-) create mode 100755 contrib/release/sign.sh diff --git a/.gitignore b/.gitignore index 5d499192..b984a1f8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ TODO Xcode_12.2.xip .idea/ fuzz/corpus +result +release_build +release_assets diff --git a/contrib/release/release.sh b/contrib/release/release.sh index fc7e52e0..6d23d247 100755 --- a/contrib/release/release.sh +++ b/contrib/release/release.sh @@ -15,19 +15,17 @@ WINDOWS_DIR_NAME="$LIANA_PREFIX-x86_64-windows-gnu" WINDOWS_ARCHIVE="$WINDOWS_DIR_NAME.zip" MAC_DIR_NAME="$LIANA_PREFIX-x86_64-apple-darwin" MAC_ARCHIVE="$MAC_DIR_NAME.tar.gz" -MAC_CODESIGN="${MAC_CODESIGN:-"0"}" -RCODESIGN_BIN="${RCODESIGN_BIN:-"$PWD/../../macos_codesigning/apple-codesign-0.22.0-x86_64-unknown-linux-musl/rcodesign"}" -CODESIGN_KEY="${CODESIGN_KEY:-"$PWD/../../macos_codesigning/wizardsardine_liana.key"}" -CODESIGN_CERT="${CODESIGN_CERT:-"$PWD/../../macos_codesigning/antoine_devid_liana_codesigning.cer"}" -NOTARY_API_CREDS_FILE="${NOTARY_API_CREDS_FILE:-"$PWD/../../macos_codesigning/encoded_appstore_api_key.json"}" create_dir() { - test -d "$1" || mkdir "$1" + if [ -d "$1" ]; then + rm -rf "$1" + fi + mkdir "$1" } # Determine the reference time used for determinism (overridable by environment) export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" -export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --sort=name" # We'll use a folder for the builds output and another one for the final assets. RELEASE_DIR="$PWD/release_assets" @@ -36,15 +34,17 @@ create_dir "$RELEASE_DIR" create_dir "$BUILD_DIR" OUT_DIR="$BUILD_DIR" ./contrib/reproducible/guix/guix-build.sh -TARGET_DIR="$BUILD_DIR" ./contrib/reproducible/docker/docker-build.sh -# Create the Linux archive and Debian binary package. +nix build .#release +NIX_BUILD_DIR="$(nix path-info .#release)" + +#Create the Linux archive and Debian binary package. ( cd "$BUILD_DIR" create_dir "$LINUX_DIR_NAME" cp "$BUILD_DIR/x86_64-unknown-linux-gnu/release/lianad" "$BUILD_DIR/x86_64-unknown-linux-gnu/release/liana-cli" "$BUILD_DIR/x86_64-unknown-linux-gnu/release/liana-gui" ../README.md "$LINUX_DIR_NAME" - tar -czf "$LINUX_ARCHIVE" "$LINUX_DIR_NAME" - cp "$LINUX_ARCHIVE" "$RELEASE_DIR" + tar --mtime="@${SOURCE_DATE_EPOCH}" -czf "$LINUX_ARCHIVE" "$LINUX_DIR_NAME" + mv "$LINUX_ARCHIVE" "$RELEASE_DIR" unzip ../contrib/release/debian/package.zip sed -i "s/VERSION_PLACEHOLDER/$VERSION/g" ./package/DEBIAN/control @@ -59,40 +59,28 @@ TARGET_DIR="$BUILD_DIR" ./contrib/reproducible/docker/docker-build.sh ( cd "$BUILD_DIR" create_dir "$WINDOWS_DIR_NAME" - cp "$BUILD_DIR/x86_64-pc-windows-gnu/release/liana-gui.exe" ../README.md "$WINDOWS_DIR_NAME" + cp "$NIX_BUILD_DIR/x86_64-pc-windows-gnu/liana-gui.exe" ../README.md "$WINDOWS_DIR_NAME" zip -r "$WINDOWS_ARCHIVE" "$WINDOWS_DIR_NAME" - cp "$WINDOWS_ARCHIVE" "$RELEASE_DIR" - cp "$BUILD_DIR/x86_64-pc-windows-gnu/release/liana-gui.exe" "$RELEASE_DIR/$LIANA_PREFIX.exe" + mv "$WINDOWS_ARCHIVE" "$RELEASE_DIR" + cp "$NIX_BUILD_DIR/x86_64-pc-windows-gnu/liana-gui.exe" "$RELEASE_DIR/$LIANA_PREFIX.exe" ) # Create the MacOS archive and a zipped application bundle of liana-gui. ( cd "$BUILD_DIR" create_dir "$MAC_DIR_NAME" - cp "$BUILD_DIR/x86_64-apple-darwin/release/lianad" "$BUILD_DIR/x86_64-apple-darwin/release/liana-cli" "$BUILD_DIR/x86_64-apple-darwin/release/liana-gui" ../README.md "$MAC_DIR_NAME" - tar -czf "$MAC_ARCHIVE" "$MAC_DIR_NAME" - cp "$MAC_ARCHIVE" "$RELEASE_DIR" + cp "$NIX_BUILD_DIR/x86_64-apple-darwin/lianad" "$NIX_BUILD_DIR/x86_64-apple-darwin/liana-cli" "$NIX_BUILD_DIR/x86_64-apple-darwin/liana-gui" ../README.md "$MAC_DIR_NAME" + tar --mtime="@${SOURCE_DATE_EPOCH}" -czf "$MAC_ARCHIVE" "$MAC_DIR_NAME" + mv "$MAC_ARCHIVE" "$RELEASE_DIR" unzip ../contrib/release/macos/Liana.app.zip sed -i "s/VERSION_PLACEHOLDER/$VERSION/g" ./Liana.app/Contents/Info.plist - cp "$BUILD_DIR/x86_64-apple-darwin/release/liana-gui" ./Liana.app/Contents/MacOS/Liana - zip -ry Liana-noncodesigned.zip Liana.app - cp ./Liana-noncodesigned.zip "$RELEASE_DIR/" - - if [ "$MAC_CODESIGN" = "1" ]; then - $RCODESIGN_BIN sign --digest sha256 --code-signature-flags runtime --pem-source "$CODESIGN_KEY" --der-source "$CODESIGN_CERT" Liana.app/ - $RCODESIGN_BIN notary-submit --max-wait-seconds 600 --api-key-path "$NOTARY_API_CREDS_FILE" --staple Liana.app - zip -ry Liana.zip Liana.app - cp ./Liana.zip "$RELEASE_DIR/" - fi + cp "$NIX_BUILD_DIR/x86_64-apple-darwin/liana-gui" ./Liana.app/Contents/MacOS/Liana + chmod u+w ./Liana.app/Contents/MacOS/Liana + zip -ry "Liana-$VERSION-noncodesigned.zip" Liana.app + mv "Liana-$VERSION-noncodesigned.zip" "$RELEASE_DIR/" ) -# Finally, sign all the assets -( - cd "$RELEASE_DIR" - for asset in $(ls); do - gpg --detach-sign --armor "$asset" - done -) +find "$RELEASE_DIR" -type f -exec sha256sum {} + | tee "$RELEASE_DIR/shasums.txt" set +ex diff --git a/contrib/release/sign.sh b/contrib/release/sign.sh new file mode 100755 index 00000000..055bef8b --- /dev/null +++ b/contrib/release/sign.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env sh + +set -e # Exit immediately if a command exits with a non-zero status +set -x # Print commands and their arguments as they are executed + +VERSION="${VERSION:-"8.0"}" +# Define the release directory +RELEASE_DIR="$PWD/release_assets" +RELEASE_BUILD_DIR="$PWD/release_build" + +# Function to perform GPG signing +sign_with_gpg() { + ( + cd "$RELEASE_DIR" + gpg --detach-sign --armor "shasums.txt" + ) +} + +# Function to convert a path to an absolute path +absolute_path() { + local path="$1" + if [[ "$path" = /* ]]; then + echo "$path" + else + echo "$PWD/$path" + fi +} + +# Function to perform rcodesign signing +sign_with_rcodesign() { + # Ensure the correct number of arguments are provided + if [ "$#" -ne 3 ]; then + echo "Usage: $0 rcodesign " + exit 1 + fi + + # Assign arguments to variables + CODESIGN_CERT="$(absolute_path $1)" + CODESIGN_KEY="$(absolute_path $2)" + NOTARY_API_CREDS_FILE="$(absolute_path $3)" + + # Verify that the provided files exist + if [ ! -f "$CODESIGN_CERT" ]; then + echo "Certificate file not found: $CODESIGN_CERT" + exit 1 + fi + + if [ ! -f "$CODESIGN_KEY" ]; then + echo "Key file not found: $CODESIGN_KEY" + exit 1 + fi + + if [ ! -f "$NOTARY_API_CREDS_FILE" ]; then + echo "API credentials file not found: $NOTARY_API_CREDS_FILE" + exit 1 + fi + + cd "$RELEASE_BUILD_DIR" + rcodesign sign \ + --digest sha256 \ + --code-signature-flags runtime \ + --pem-source "$CODESIGN_KEY" \ + --der-source "$CODESIGN_CERT" \ + Liana.app/ + + rcodesign notary-submit \ + --max-wait-seconds 600 \ + --api-key-path "$NOTARY_API_CREDS_FILE" \ + --staple Liana.app + + zip -ry "Liana-$VERSION.zip" Liana.app + mv "Liana-$VERSION.zip" "$RELEASE_DIR/" +} + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 [args...]" + exit 1 +fi + +COMMAND="$1" +shift # Shift the arguments to access any additional parameters + +case "$COMMAND" in + gpg) + sign_with_gpg + ;; + rcodesign) + sign_with_rcodesign "$@" + ;; + *) + echo "Invalid command: $COMMAND" + echo "Usage: $0 [args...]" + exit 1 + ;; +esac + +# Disable debugging and exit on success +set +ex diff --git a/flake.nix b/flake.nix index 9877d135..48f98a77 100644 --- a/flake.nix +++ b/flake.nix @@ -81,6 +81,9 @@ mkdir -p $CARGO_ZIGBUILD_CACHE_DIR export CC=zigcc export CXX=zigc++ + + # rcodesign needs place to sign binary + export RUSTFLAGS="-C link-arg=-Wl,-headerpad_max_install_names" ''; installPhaseCommand = '' @@ -144,12 +147,12 @@ }; releaseShell = pkgs.mkShell { - buildInputs = with pkgs; [ - zip - unzip - gnutar - dpkg - rcodesign + buildInputs = [ + pkgs.zip + pkgs.unzip + pkgs.gnutar + pkgs.dpkg + pkgs.rcodesign ]; }; From cc083f9be6f6dd2f50a60572d09872913ece5dd0 Mon Sep 17 00:00:00 2001 From: edouardparis Date: Fri, 6 Dec 2024 17:10:17 +0100 Subject: [PATCH 4/5] Remove docker reproducible build --- .dockerignore | 4 -- contrib/reproducible/docker/README.md | 33 ----------- contrib/reproducible/docker/docker-build.sh | 58 ------------------- contrib/reproducible/docker/macos.Dockerfile | 50 ---------------- contrib/reproducible/docker/macos_cmd.sh | 46 --------------- .../reproducible/docker/windows.Dockerfile | 36 ------------ contrib/reproducible/docker/windows_cmd.sh | 21 ------- 7 files changed, 248 deletions(-) delete mode 100644 .dockerignore delete mode 100644 contrib/reproducible/docker/README.md delete mode 100755 contrib/reproducible/docker/docker-build.sh delete mode 100644 contrib/reproducible/docker/macos.Dockerfile delete mode 100755 contrib/reproducible/docker/macos_cmd.sh delete mode 100644 contrib/reproducible/docker/windows.Dockerfile delete mode 100755 contrib/reproducible/docker/windows_cmd.sh diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index f55d05ee..00000000 --- a/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!Cargo.* -!gui/Cargo.* -!contrib/reproducible/docker/cargo_config.toml diff --git a/contrib/reproducible/docker/README.md b/contrib/reproducible/docker/README.md deleted file mode 100644 index 7d939cbe..00000000 --- a/contrib/reproducible/docker/README.md +++ /dev/null @@ -1,33 +0,0 @@ -## Reproducible Liana builds - -This repository contains the scripts to [reproducibly build](https://reproducible-builds.org/) Liana -on Windows and MacOS, for which we are missing [bootstrapable Guix builds](../guix/). - -In order to perform the builds you will need [Docker](https://www.docker.com/). - -The [`docker-build.sh`](./docker-build.sh) script will create a Docker image containing the build -environment for both Mac and Windows (see the respective `Dockerfile`s). It will then build the GUI -on Windows (the daemon isn't supported there) and both the daemon and the GUI on MacOS. The output -will be placed in a given `TARGET_DIR` (whose default value is `deter_build_target`). - -### Build instructions - -First of all, get [Docker](https://www.docker.com/). - -Then get the MacOS SDK. It is required to be able to build the MacOS binaries. The `12_2` version is -required (`Xcode_12.2.xip`). You need to download it from Apple's website. An Apple ID and cookies -enabled for the hostname are required. You can create one for free. (Note it is illegal to -distribute the archive.) Once logged in you can use the [direct -link](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip) to download -the archive. Alternatively, go to 'Downloads', then 'More' and search for [`Xcode -12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2). -The `sha256sum` of the downloaded XIP archive should be -`28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`. - -Copy the downloaded `Xcode_12.2.xip` archive at the root of this repository (or provide a custom -path to the script by setting the `XCODE_PATH` env var). - -Finally, run the script from the root of the repository: -``` -./contrib/reproducible/docker/docker-build.sh -``` diff --git a/contrib/reproducible/docker/docker-build.sh b/contrib/reproducible/docker/docker-build.sh deleted file mode 100755 index 55127d88..00000000 --- a/contrib/reproducible/docker/docker-build.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env sh - -set -ex - -TARGET_DIR="${TARGET_DIR:-"$PWD/deter_build_target"}" - -XCODE_PATH="${XCODE_PATH:-"$PWD/Xcode_12.2.xip"}" -XCODE_FILENAME="$(basename $XCODE_PATH)" -XCODE_SHASUM="28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0" - -# Build (only) the Liana GUI on Windows. -docker build . -t liana_cross_win -f contrib/reproducible/docker/windows.Dockerfile -docker run --rm -ti \ - -v "$TARGET_DIR":/liana/target \ - -v "$PWD/contrib/reproducible/docker":/liana/docker \ - -v "$PWD/Cargo.toml":/liana/Cargo.toml \ - -v "$PWD/Cargo.lock":/liana/Cargo.lock \ - -v "$PWD/liana/Cargo.toml":/liana/liana/Cargo.toml \ - -v "$PWD/liana/src":/liana/liana/src \ - -v "$PWD/lianad/Cargo.toml":/liana/lianad/Cargo.toml \ - -v "$PWD/lianad/src":/liana/lianad/src \ - -v "$PWD/liana-gui/Cargo.toml":/liana/liana-gui/Cargo.toml \ - -v "$PWD/liana-gui/src":/liana/liana-gui/src \ - -v "$PWD/liana-ui/Cargo.toml":/liana/liana-ui/Cargo.toml \ - -v "$PWD/liana-ui/src":/liana/liana-ui/src \ - -v "$PWD/liana-ui/static":/liana/liana-ui/static \ - -v "$PWD/fuzz/Cargo.toml":/liana/fuzz/Cargo.toml \ - liana_cross_win - - -# Sanity check the given MacOS SDK is the expected one. -if ! $(echo "$XCODE_SHASUM $XCODE_PATH" | sha256sum -c --status); then - echo "No or invalid Xcode SDK found. Need an Xcode_X.Y.xip archive whose hash is $XCODE_SHASUM. You can configure the path using \$XCODE_PATH."; - exit 1; -fi - -# Build both the Liana daemon and GUI on MacOS. -docker build . -t liana_cross_mac -f contrib/reproducible/docker/macos.Dockerfile -docker run -ti \ - -v "$TARGET_DIR":/liana/target \ - -v "$PWD/contrib/reproducible/docker":/liana/docker \ - -v "$PWD/Cargo.toml":/liana/Cargo.toml \ - -v "$PWD/Cargo.lock":/liana/Cargo.lock \ - -v "$PWD/liana/Cargo.toml":/liana/liana/Cargo.toml \ - -v "$PWD/liana/src":/liana/liana/src \ - -v "$PWD/lianad/Cargo.toml":/liana/lianad/Cargo.toml \ - -v "$PWD/lianad/src":/liana/lianad/src \ - -v "$PWD/liana-gui/Cargo.toml":/liana/liana-gui/Cargo.toml \ - -v "$PWD/liana-gui/src":/liana/liana-gui/src \ - -v "$PWD/liana-ui/Cargo.toml":/liana/liana-ui/Cargo.toml \ - -v "$PWD/liana-ui/src":/liana/liana-ui/src \ - -v "$PWD/liana-ui/static":/liana/liana-ui/static \ - -v "$PWD/fuzz/Cargo.toml":/liana/fuzz/Cargo.toml \ - -v "$XCODE_PATH":"/liana/$XCODE_FILENAME" \ - -e XCODE_PATH="/liana/$XCODE_FILENAME" \ - liana_cross_mac - -set +ex diff --git a/contrib/reproducible/docker/macos.Dockerfile b/contrib/reproducible/docker/macos.Dockerfile deleted file mode 100644 index 6296515e..00000000 --- a/contrib/reproducible/docker/macos.Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -FROM debian:bullseye - -WORKDIR /liana - -# We try to pin our dependencies to avoid potential sources of non-determinism, but we don't go -# out of our way to pin the whole tree of deps. Instead invest time in getting Guix cross-compilation. -RUN apt update && apt satisfy -y \ - "clang (>=11.0, <=11.0)" \ - "make (<=4.3-*)" \ - "libssl-dev (>=1.1, <=1.1)" \ - "liblzma-dev (>=5.2, <=5.2)" \ - "libxml2 (>=2.9, <= 2.9)" \ - "libxml2-dev (>=2.9, <=2.9)" \ - "cmake (>=3.18, <=3.18)" \ - "git (>=2.30, <=2.30)" \ - "patch (>=2.7, <=2.7)" \ - "python3 (>=3.9, <=3.9)" \ - "llvm-dev (>=11.0, <=11.0)" \ - "cpio (<=2.13+*)" \ - "zlib1g-dev (>=1.2, <=1.2)" \ - "libbz2-dev (>=1.0, <=1.0)" \ - "xz-utils (>=5.2, <=5.2)" \ - "bzip2 (>=1.0, <=1.0)" \ - "curl (>=7.74, <=7.74)" - -# Download the cargo binary and compiled stdlib from the distributed releases to make sure to build with -# the very same toolchain. We use 1.71.1 because it is unfortunately the MSRV of the GUI. -RUN curl -O "https://static.rust-lang.org/dist/rust-1.71.1-x86_64-unknown-linux-gnu.tar.gz" && \ - echo "34778d1cda674990dfc0537bc600066046ae9cb5d65a07809f7e7da31d4689c4 rust-1.71.1-x86_64-unknown-linux-gnu.tar.gz" | sha256sum -c && \ - tar -xzf rust-1.71.1-x86_64-unknown-linux-gnu.tar.gz && \ - curl -O "https://static.rust-lang.org/dist/rust-1.71.1-x86_64-apple-darwin.tar.gz" && \ - echo "916056603da88336aba68bbeab49711cc8fdb9cfb46a49b04850c0c09761f58c rust-1.71.1-x86_64-apple-darwin.tar.gz" | sha256sum -c && \ - tar -xzf rust-1.71.1-x86_64-apple-darwin.tar.gz && \ - rm -r *.tar.gz - -# NOTE: we were previously caching dependencies here (through `cargo vendor`). It's a tradeoff between the image size -# and not needing internet access when running the image to build the software. - -# For some reason, we can't just set the RUSTFLAGS environment variable to add `-L` for compiling dependencies. -# This doesn't work: RUSTFLAGS="-L/liana/rust-1.71.1-x86_64-apple-darwin/rust-std-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/" -# As a workaround, we use a wrapped `rustc` binary that always links against the macOS stdlib we just downloaded. -# Some issues that seem to be related: -# https://github.com/rust-lang/rust/issues/40717 -# https://github.com/rust-lang/rust/issues/48409 -RUN echo "#!/bin/sh" > rustc_wrapper.sh && \ - echo "/liana/rust-1.71.1-x86_64-unknown-linux-gnu/rustc/bin/rustc \"\$@\" -L/liana/rust-1.71.1-x86_64-apple-darwin/rust-std-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/ -L/liana/rust-1.71.1-x86_64-unknown-linux-gnu/rust-std-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib/" >> rustc_wrapper.sh && \ - chmod +x rustc_wrapper.sh -ENV RUSTC="/liana/rustc_wrapper.sh" - -CMD ["./docker/macos_cmd.sh"] diff --git a/contrib/reproducible/docker/macos_cmd.sh b/contrib/reproducible/docker/macos_cmd.sh deleted file mode 100755 index 332c70a2..00000000 --- a/contrib/reproducible/docker/macos_cmd.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env sh - -# ========================================================== -# The script ran within the Docker container to build Liana. -# ========================================================== - -set -xe - -test -f "$XCODE_PATH" || exit 1 - -# Build the SDK and the toolchain using osxcross. It is expected to be located at $XCODE_PATH -# It's not part of the image to be able to share the $XCODE_PATH instead of copying it in the Docker context -# and then to the image. -git clone https://github.com/darosior/osxcross -b dependencies_pinning -cd osxcross -git checkout 50e86ebca7d14372febd0af8cd098705049161b9 -DARLING_DMG_REVISION=241238313a47d3cf6427ac5a75b7a0311a3a4cb4 \ - P7ZIP_REVISION=2f60a51ac3aa2507d36df3c4f58f71a3716b1357 \ - PBZX_REVISION=2a4d7c3300c826d918def713a24d25c237c8ed53 \ - XAR_REVISION=c2111a9a9cabc50d2b9c604aff41a481ae3f1989 ./tools/gen_sdk_package_pbzx.sh "$XCODE_PATH" -mv MacOSX* tarballs/ -DARLING_DMG_REVISION=241238313a47d3cf6427ac5a75b7a0311a3a4cb4 \ - P7ZIP_REVISION=2f60a51ac3aa2507d36df3c4f58f71a3716b1357 \ - PBZX_REVISION=2a4d7c3300c826d918def713a24d25c237c8ed53 \ - XAR_REVISION=c2111a9a9cabc50d2b9c604aff41a481ae3f1989 \ - UNATTENDED=1 ./build.sh -cd .. - -# Finally build the projects using the toolchain just created. -alias cargo="/liana/rust-1.71.1-x86_64-unknown-linux-gnu/cargo/bin/cargo" - -for package_name in "lianad" "liana-gui"; do - PATH="$PATH:$PWD/osxcross/target/bin/" \ - CC=o64-clang \ - CXX=o64-clang++ \ - RUSTFLAGS="$RUSTFLAGS -Clinker=o64-clang" \ - cargo rustc \ - -p "$package_name" \ - --target x86_64-apple-darwin \ - --release -done - -# Avoid having to get root on the host to remove the target dir. -chmod -R a+rw target/ - -set +xe diff --git a/contrib/reproducible/docker/windows.Dockerfile b/contrib/reproducible/docker/windows.Dockerfile deleted file mode 100644 index 129451fd..00000000 --- a/contrib/reproducible/docker/windows.Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM debian:bullseye - -WORKDIR /liana - -# We try to pin our dependencies to avoid potential sources of non-determinism, but we don't go -# out of our way to pin the whole tree of deps. Instead invest time in getting Guix cross-compilation. -RUN apt update && apt satisfy -y \ - "gcc-mingw-w64-x86-64 (>=10.2, <=10.2)" \ - "curl (>=7.74, <=7.74)" \ - "gcc (>=10.2, <=10.2)" - -# Download the cargo binary and compiled stdlib from the distributed releases to make sure to build with -# the very same toolchain. We use 1.71.1 because it is unfortunately the MSRV of the GUI. -RUN curl -O "https://static.rust-lang.org/dist/rust-1.71.1-x86_64-unknown-linux-gnu.tar.gz" && \ - echo "34778d1cda674990dfc0537bc600066046ae9cb5d65a07809f7e7da31d4689c4 rust-1.71.1-x86_64-unknown-linux-gnu.tar.gz" | sha256sum -c && \ - tar -xzf rust-1.71.1-x86_64-unknown-linux-gnu.tar.gz && \ - curl -O "https://static.rust-lang.org/dist/rust-1.71.1-x86_64-pc-windows-gnu.tar.gz" && \ - echo "15289233721ad7c3d697890d9c6079ca3b8a0f6740c080fbec3e8ae3a5ea5c8c rust-1.71.1-x86_64-pc-windows-gnu.tar.gz" | sha256sum -c && \ - tar -xzf rust-1.71.1-x86_64-pc-windows-gnu.tar.gz && \ - rm -r *.tar.gz - -# NOTE: we were previously caching dependencies here (through `cargo vendor`). It's a tradeoff between the image size -# and not needing internet access when running the image to build the software. - -# For some reason, we can't just set the RUSTFLAGS environment variable to add `-L` for compiling dependencies. -# This doesn't work: RUSTFLAGS="-L /liana/rust-1.71.1-x86_64-pc-windows-gnu/rust-std-x86_64-pc-windows-gnu/lib/rustlib/x86_64-pc-windows-gnu/lib/ -L /liana/rust-1.71.1-x86_64-unknown-linux-gnu/rust-std-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib/" -# As a workaround, we use a wrapped `rustc` binary that always links against the windows stdlib we just downloaded. -# Some issues that seem to be related: -# https://github.com/rust-lang/rust/issues/40717 -# https://github.com/rust-lang/rust/issues/48409 -RUN echo "#!/bin/sh" > rustc_wrapper.sh && \ - echo "/liana/rust-1.71.1-x86_64-unknown-linux-gnu/rustc/bin/rustc \"\$@\" -L /liana/rust-1.71.1-x86_64-pc-windows-gnu/rust-std-x86_64-pc-windows-gnu/lib/rustlib/x86_64-pc-windows-gnu/lib/ -L /liana/rust-1.71.1-x86_64-unknown-linux-gnu/rust-std-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib/" >> rustc_wrapper.sh && \ - chmod +x rustc_wrapper.sh -ENV RUSTC="/liana/rustc_wrapper.sh" - -CMD ["./docker/windows_cmd.sh"] diff --git a/contrib/reproducible/docker/windows_cmd.sh b/contrib/reproducible/docker/windows_cmd.sh deleted file mode 100755 index b087a7cb..00000000 --- a/contrib/reproducible/docker/windows_cmd.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env sh - -# ================================================================= -# The script ran within the Docker container to build the Liana GUI. -# ================================================================= - -set -xe - -# Build the GUI for Windows. The Windows Portable Execution (PE) format contains some timestamps. -# Instruct ld to set them to 0. -alias cargo="/liana/rust-1.71.1-x86_64-unknown-linux-gnu/cargo/bin/cargo" -RUSTFLAGS="-Clink-arg=-Wl,--no-insert-timestamp" \ - cargo rustc \ - -p liana-gui \ - --release \ - --target x86_64-pc-windows-gnu - -# Avoid having to get root on the host to remove the target dir. -chmod -R a+rw target/ - -set +xe From 6b4e7b0521c7328647738b46afd67386273c403e Mon Sep 17 00:00:00 2001 From: edouardparis Date: Fri, 13 Dec 2024 11:14:13 +0100 Subject: [PATCH 5/5] fix reproducible dpkg-deb --- contrib/release/release.sh | 22 ++++++++++++++++------ contrib/release/sign.sh | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/contrib/release/release.sh b/contrib/release/release.sh index 6d23d247..6425f9af 100755 --- a/contrib/release/release.sh +++ b/contrib/release/release.sh @@ -23,10 +23,21 @@ create_dir() { mkdir "$1" } + + # Determine the reference time used for determinism (overridable by environment) -export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" +export SOURCE_DATE_EPOCH="$(git -c log.showsignature=false log --format=%at -1)" +export TZ=UTC export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --sort=name" +zip_archive () { + local archive="$1" + shift + touch -d "@$SOURCE_DATE_EPOCH" "$@" + find "$@" -type f -exec touch -d "@$SOURCE_DATE_EPOCH" {} + + find "$@" -type f | sort | zip -oX "$archive" -@ +} + # We'll use a folder for the builds output and another one for the final assets. RELEASE_DIR="$PWD/release_assets" BUILD_DIR="$PWD/release_build" @@ -51,7 +62,7 @@ NIX_BUILD_DIR="$(nix path-info .#release)" cp "$BUILD_DIR/x86_64-unknown-linux-gnu/release/lianad" "$BUILD_DIR/x86_64-unknown-linux-gnu/release/liana-cli" "$BUILD_DIR/x86_64-unknown-linux-gnu/release/liana-gui" ../README.md ./package/usr/bin/ DIRNAME="liana_$VERSION-1_amd64" mv ./package "$DIRNAME" - dpkg-deb -Zxz --build "$DIRNAME" + dpkg-deb -Zxz --build --root-owner-group "$DIRNAME" mv "$DIRNAME.deb" "$RELEASE_DIR" ) @@ -60,7 +71,7 @@ NIX_BUILD_DIR="$(nix path-info .#release)" cd "$BUILD_DIR" create_dir "$WINDOWS_DIR_NAME" cp "$NIX_BUILD_DIR/x86_64-pc-windows-gnu/liana-gui.exe" ../README.md "$WINDOWS_DIR_NAME" - zip -r "$WINDOWS_ARCHIVE" "$WINDOWS_DIR_NAME" + zip_archive "$WINDOWS_ARCHIVE" "$WINDOWS_DIR_NAME" mv "$WINDOWS_ARCHIVE" "$RELEASE_DIR" cp "$NIX_BUILD_DIR/x86_64-pc-windows-gnu/liana-gui.exe" "$RELEASE_DIR/$LIANA_PREFIX.exe" ) @@ -76,11 +87,10 @@ NIX_BUILD_DIR="$(nix path-info .#release)" unzip ../contrib/release/macos/Liana.app.zip sed -i "s/VERSION_PLACEHOLDER/$VERSION/g" ./Liana.app/Contents/Info.plist cp "$NIX_BUILD_DIR/x86_64-apple-darwin/liana-gui" ./Liana.app/Contents/MacOS/Liana - chmod u+w ./Liana.app/Contents/MacOS/Liana - zip -ry "Liana-$VERSION-noncodesigned.zip" Liana.app + zip_archive "Liana-$VERSION-noncodesigned.zip" Liana.app mv "Liana-$VERSION-noncodesigned.zip" "$RELEASE_DIR/" ) -find "$RELEASE_DIR" -type f -exec sha256sum {} + | tee "$RELEASE_DIR/shasums.txt" +find "$RELEASE_DIR" -type f ! -name "shasums.txt" -exec sha256sum {} + | tee "$RELEASE_DIR/shasums.txt" set +ex diff --git a/contrib/release/sign.sh b/contrib/release/sign.sh index 055bef8b..4e876f77 100755 --- a/contrib/release/sign.sh +++ b/contrib/release/sign.sh @@ -26,6 +26,20 @@ absolute_path() { fi } +# Determine the reference time used for determinism (overridable by environment) +export SOURCE_DATE_EPOCH="$(git -c log.showsignature=false log --format=%at -1)" +export TZ=UTC + +zip_archive () { + local archive="$1" + shift + touch -d "@$SOURCE_DATE_EPOCH" "$@" + find "$@" -type f -exec touch -d "@$SOURCE_DATE_EPOCH" {} + + zip -r -oX - "$@" > "$archive" +} + + + # Function to perform rcodesign signing sign_with_rcodesign() { # Ensure the correct number of arguments are provided @@ -56,6 +70,8 @@ sign_with_rcodesign() { fi cd "$RELEASE_BUILD_DIR" + chmod u+w ./Liana.app/Contents/MacOS/Liana + rcodesign sign \ --digest sha256 \ --code-signature-flags runtime \ @@ -68,8 +84,9 @@ sign_with_rcodesign() { --api-key-path "$NOTARY_API_CREDS_FILE" \ --staple Liana.app - zip -ry "Liana-$VERSION.zip" Liana.app + zip_archive "Liana-$VERSION.zip" Liana.app mv "Liana-$VERSION.zip" "$RELEASE_DIR/" + sha256sum "$RELEASE_DIR/Liana-$VERSION.zip" | tee -a "$RELEASE_DIR/shasums.txt" } if [ "$#" -lt 1 ]; then