diff --git a/.dockerignore b/.dockerignore index 3b2df1bb..110a7f3e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ * +!Cargo.* !gui/Cargo.* !contrib/docker/cargo_config.toml diff --git a/contrib/docker/README.md b/contrib/docker/README.md new file mode 100644 index 00000000..f046b340 --- /dev/null +++ b/contrib/docker/README.md @@ -0,0 +1,33 @@ +## Reproducible Liana builds + +This repository contains the scripts to [reproducibly build](https://reproducible-builds.org/) Liana +on Windows and MacOS, for which we are missing [bootstrapable Guix builds](../guix/). + +In order to perform the builds you will need [Docker](https://www.docker.com/). + +The [`docker-build.sh`](./docker-build.sh) script will create a Docker image containing the build +environment for both Mac and Windows (see the respective `Dockerfile`s). It will then build the GUI +on Windows (the daemon isn't supported there) and both the daemon and the GUI on MacOS. The output +will be placed in a given `TARGET_DIR` (whose default value is `deter_build_target`). + +### Build instructions + +First of all, get [Docker](https://www.docker.com/). + +Then get the MacOS SDK. It is required to be able to build the MacOS binaries. The `12_2` version is +required (`Xcode_12.2.xip`). You need to download it from Apple's website. An Apple ID and cookies +enabled for the hostname are required. You can create one for free. (Note it is illegal to +distribute the archive.) Once logged in you can use the [direct +link](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip) to download +the archive. Alternatively, go to 'Downloads', then 'More' and search for [`Xcode +12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2). +The `sha256sum` of the downloaded XIP archive should be +`28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`. + +Copy the downloaded `Xcode_12.2.xip` archive at the root of this repository (or provide a custom +path to the script by setting the `XCODE_PATH` env var). + +Finally, run the script from the root of the repository: +``` +./contrib/docker/docker-build.sh +``` diff --git a/contrib/docker/docker-build.sh b/contrib/docker/docker-build.sh index b687f6a2..5905f132 100755 --- a/contrib/docker/docker-build.sh +++ b/contrib/docker/docker-build.sh @@ -2,14 +2,35 @@ set -ex -TARGET_DIR="$PWD/deter_build_target" +TARGET_DIR="${TARGET_DIR:-"$PWD/deter_build_target"}" -docker build . -t liana_cross_compile -f contrib/docker/Dockerfile +XCODE_PATH="${XCODE_PATH:-"$PWD/Xcode_12.2.xip"}" + +# Build (only) the Liana GUI on Windows. +docker build . -t liana_cross_win -f contrib/docker/windows.Dockerfile docker run --rm -ti \ - -v "$TARGET_DIR":/liana/target \ + -v "$TARGET_DIR/gui":/liana/target \ -v "$PWD/contrib/docker":/liana/docker \ -v "$PWD/gui/src":/liana/src \ -v "$PWD/gui/static":/liana/static \ - liana_cross_compile + liana_cross_win + +# Sanity check the given MacOS SDK is the expected one. +if ! $(echo "28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0 $(basename $XCODE_PATH)" | sha256sum -c --status); then + echo "No or invalid Xcode SDK found. Need an Xcode_12.2.xip. You can configure the path using \$XCODE_PATH."; + exit 1; +fi + +# Build both the Liana daemon and GUI on MacOS. +docker build . -t liana_cross_mac -f contrib/docker/macos.Dockerfile +docker run --rm -ti \ + -v "$TARGET_DIR":/liana/target \ + -v "$TARGET_DIR/gui":/liana/gui/target \ + -v "$PWD/contrib/docker":/liana/docker \ + -v "$PWD/src":/liana/src \ + -v "$PWD/gui/src":/liana/gui/src \ + -v "$PWD/gui/static":/liana/gui/static \ + -v "$XCODE_PATH":/liana/Xcode_12.2.xip \ + liana_cross_mac set +ex diff --git a/contrib/docker/macos.Dockerfile b/contrib/docker/macos.Dockerfile new file mode 100644 index 00000000..e0eff2dd --- /dev/null +++ b/contrib/docker/macos.Dockerfile @@ -0,0 +1,63 @@ +FROM debian:bullseye + +WORKDIR /liana + +# We try to pin our dependencies to avoid potential sources of non-determinism, but we don't go +# out of our way to pin the whole tree of deps. Instead invest time in getting Guix cross-compilation. +RUN apt update && apt install -y \ + clang=1:11.0-51+nmu5 \ + make=4.3-4.1 \ + libssl-dev=1.1.1n-0+deb11u3 \ + liblzma-dev=5.2.5-2.1~deb11u1 \ + libxml2=2.9.10+dfsg-6.7+deb11u2 \ + libxml2-dev=2.9.10+dfsg-6.7+deb11u2 \ + cmake=3.18.4-2+deb11u1 \ + git=1:2.30.2-1 \ + patch=2.7.6-7 \ + python3=3.9.2-3 \ + llvm-dev=1:11.0-51+nmu5 \ + cpio=2.13+dfsg-4 \ + zlib1g-dev=1:1.2.11.dfsg-2+deb11u2 \ + libbz2-dev=1.0.8-4 \ + xz-utils=5.2.5-2.1~deb11u1 \ + bzip2=1.0.8-4 \ + curl=7.74.0-1.3+deb11u3 + +# Download the cargo binary and compiled stdlib from the distributed releases to make sure to build with +# the very same toolchain. We use 1.65.0 because it is unfortunately the MSRV of the GUI. +RUN curl -O "https://static.rust-lang.org/dist/rust-1.65.0-x86_64-unknown-linux-gnu.tar.gz" && \ + echo "8f754fdd5af783fe9020978c64e414cb45f3ad0a6f44d045219bbf2210ca3cb9 rust-1.65.0-x86_64-unknown-linux-gnu.tar.gz" | sha256sum -c && \ + tar -xzf rust-1.65.0-x86_64-unknown-linux-gnu.tar.gz && \ + curl -O "https://static.rust-lang.org/dist/rust-1.65.0-x86_64-apple-darwin.tar.gz" && \ + echo "139087a3937799415fd829e5a88162a69a32c23725a44457f9c96b98e4d64a7c rust-1.65.0-x86_64-apple-darwin.tar.gz" | sha256sum -c && \ + tar -xzf rust-1.65.0-x86_64-apple-darwin.tar.gz && \ + rm -r *.tar.gz + +# Copy the Cargo files for both the daemon and the GUI to vendor the dependencies. +COPY Cargo.toml Cargo.lock /liana/ +COPY gui/Cargo.toml gui/Cargo.lock /liana/gui/ + +# We cache the dependencies sources in the image to avoid re-indexing everything from scratch +# at every run. It was useful when debugging the build, it could be removed eventually if we +# think the tradeoff vs the image size isn't worth it anymore. +RUN /liana/rust-1.65.0-x86_64-unknown-linux-gnu/cargo/bin/cargo vendor && \ + cd gui && \ + /liana/rust-1.65.0-x86_64-unknown-linux-gnu/cargo/bin/cargo vendor && \ + cd .. + +# Cargo configuration for using the vendored dependencies during the builds. +COPY contrib/docker/cargo_config.toml /liana/.cargo/cargo_config.toml +COPY contrib/docker/cargo_config.toml /liana/gui/.cargo/cargo_config.toml + +# For some reason, we can't just set the RUSTFLAGS environment variable to add `-L` for compiling dependencies. +# This doesn't work: RUSTFLAGS="-L/liana/rust-1.65.0-x86_64-apple-darwin/rust-std-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/" +# As a workaround, we use a wrapped `rustc` binary that always links against the macOS stdlib we just downloaded. +# Some issues that seem to be related: +# https://github.com/rust-lang/rust/issues/40717 +# https://github.com/rust-lang/rust/issues/48409 +RUN echo "#!/bin/sh" > rustc_wrapper.sh && \ + echo "/liana/rust-1.65.0-x86_64-unknown-linux-gnu/rustc/bin/rustc \"\$@\" -L/liana/rust-1.65.0-x86_64-apple-darwin/rust-std-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/ -L/liana/rust-1.65.0-x86_64-unknown-linux-gnu/rust-std-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib/" >> rustc_wrapper.sh && \ + chmod +x rustc_wrapper.sh +ENV RUSTC="/liana/rustc_wrapper.sh" + +CMD ["./docker/macos_cmd.sh"] diff --git a/contrib/docker/macos_cmd.sh b/contrib/docker/macos_cmd.sh new file mode 100755 index 00000000..d5935a11 --- /dev/null +++ b/contrib/docker/macos_cmd.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env sh + +# ========================================================== +# The script ran within the Docker container to build Liana. +# ========================================================== + +set -xe + +# Build the SDK and the toolchain using osxcross. It is expected to be located at /liana/Xcode_12.2.xip. +# It's not part of the image to be able to share the Xcode_12.2.xip instead of copying it in the Docker context +# and then to the image. +git clone https://github.com/darosior/osxcross -b dependencies_pinning +cd osxcross +git checkout 50e86ebca7d14372febd0af8cd098705049161b9 +DARLING_DMG_REVISION=241238313a47d3cf6427ac5a75b7a0311a3a4cb4 \ + P7ZIP_REVISION=2f60a51ac3aa2507d36df3c4f58f71a3716b1357 \ + PBZX_REVISION=2a4d7c3300c826d918def713a24d25c237c8ed53 \ + XAR_REVISION=c2111a9a9cabc50d2b9c604aff41a481ae3f1989 ./tools/gen_sdk_package_pbzx.sh ../Xcode_12.2.xip +mv MacOSX11.0.sdk.tar.xz tarballs/ +DARLING_DMG_REVISION=241238313a47d3cf6427ac5a75b7a0311a3a4cb4 \ + P7ZIP_REVISION=2f60a51ac3aa2507d36df3c4f58f71a3716b1357 \ + PBZX_REVISION=2a4d7c3300c826d918def713a24d25c237c8ed53 \ + XAR_REVISION=c2111a9a9cabc50d2b9c604aff41a481ae3f1989 \ + UNATTENDED=1 ./build.sh +cd .. + +# Finally build the projects using the toolchain just created. +alias cargo="/liana/rust-1.65.0-x86_64-unknown-linux-gnu/cargo/bin/cargo" + +PATH="$PATH:$PWD/osxcross/target/bin/" \ + CC=o64-clang \ + CXX=o64-clang++ \ + RUSTFLAGS="$RUSTFLAGS -Clinker=o64-clang" \ + cargo rustc \ + --target x86_64-apple-darwin \ + --release + +cd gui/ +PATH="$PATH:$PWD/../osxcross/target/bin/" \ + CC=o64-clang \ + CXX=o64-clang++ \ + RUSTFLAGS="$RUSTFLAGS -Clinker=o64-clang" \ + cargo rustc \ + --target x86_64-apple-darwin \ + --release +cd .. + +# Avoid having to get root on the host to remove the target dir. +chmod -R a+rw target/ gui/target + +set +xe diff --git a/contrib/docker/Dockerfile b/contrib/docker/windows.Dockerfile similarity index 96% rename from contrib/docker/Dockerfile rename to contrib/docker/windows.Dockerfile index ba3a41f1..057e596d 100644 --- a/contrib/docker/Dockerfile +++ b/contrib/docker/windows.Dockerfile @@ -38,9 +38,7 @@ COPY contrib/docker/cargo_config.toml /liana/.cargo/cargo_config.toml # https://github.com/rust-lang/rust/issues/48409 RUN echo "#!/bin/sh" > rustc_wrapper.sh && \ echo "/liana/rust-1.65.0-x86_64-unknown-linux-gnu/rustc/bin/rustc \"\$@\" -L /liana/rust-1.65.0-x86_64-pc-windows-gnu/rust-std-x86_64-pc-windows-gnu/lib/rustlib/x86_64-pc-windows-gnu/lib/ -L /liana/rust-1.65.0-x86_64-unknown-linux-gnu/rust-std-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib/" >> rustc_wrapper.sh && \ - chmod +x rustc_wrapper.sh && \ - cat rustc_wrapper.sh && \ - ./rustc_wrapper.sh -vV + chmod +x rustc_wrapper.sh ENV RUSTC="/liana/rustc_wrapper.sh" -CMD ["./docker/build.sh"] +CMD ["./docker/windows_cmd.sh"] diff --git a/contrib/docker/build.sh b/contrib/docker/windows_cmd.sh similarity index 66% rename from contrib/docker/build.sh rename to contrib/docker/windows_cmd.sh index 40a88dfa..414a3352 100755 --- a/contrib/docker/build.sh +++ b/contrib/docker/windows_cmd.sh @@ -4,10 +4,17 @@ # The script ran within the Docker container to build the Liana GUI. # ================================================================= +set -xe # Build the GUI for Windows. The Windows Portable Execution (PE) format contains some timestamps. # Instruct ld to set them to 0. -RUSTFLAGS="-Clink-arg=-Wl,--no-insert-timestamp" /liana/rust-1.65.0-x86_64-unknown-linux-gnu/cargo/bin/cargo rustc --release --target x86_64-pc-windows-gnu +alias cargo="/liana/rust-1.65.0-x86_64-unknown-linux-gnu/cargo/bin/cargo" +RUSTFLAGS="-Clink-arg=-Wl,--no-insert-timestamp" \ + cargo rustc \ + --release \ + --target x86_64-pc-windows-gnu # Avoid having to get root on the host to remove the target dir. chmod -R a+rw target/ + +set +xe