From 052f10fd68984a01cc2c87eee9edcc16dbfa6ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Thu, 9 Jul 2026 06:30:44 -0400 Subject: [PATCH] fix(build): prevent 32-bit startup crash (segfault/SIGILL) in downloads binaries (#5739) * fix(build): force nodynamic webp tag on 32-bit standalone binaries gen2brain/webp's native libwebp backend links ebitengine/purego, whose reverse callbacks are unsupported on 32-bit ARM and x86. purego registers its callback in package init(), so the binary crashes at startup (SIGSEGV or SIGILL) before any Navidrome code runs. The nodynamic build tag from #5606 forces the safe WASM path, but it was only applied to the Docker-image build stage. The standalone build stage, which produces the downloads-page tarballs and the deb/rpm packages, still linked purego, so the armv7/v6/v5 and 386 downloads crashed on launch (#5738, #5735). Move the tag decision into release/build-tags.sh, shared by both build stages so they can no longer drift, and add release/verify-binary.sh as a build-time guard that fails if a 32-bit binary links purego. * fix(build): harden webp build-tag scripts per review - verify-binary.sh: fail loudly when the target binary is missing (e.g. an unmatched glob) instead of letting `go version -m` fail inside a pipeline and silently pass, which would bypass the guard. - build-tags.sh / verify-binary.sh: fall back to `go env GOARCH` when xx-info is unavailable, so the scripts stay correct outside the xx build image. (Not `uname -m`, which reports the build host, not the cross target.) - Dockerfile: use `set -e` in the standalone build block and drop the redundant `|| exit 1` suffixes; keep the debug GOENV dump non-fatal. * chore(build): quote -tags argument in both build stages Defensive quoting per review; the value comes from release/build-tags.sh and contains no whitespace today, but quoting prevents word-splitting if it ever does. * fix(build): link 32-bit arm binaries with LLD to fix startup crash The standalone armv7/v6/v5 binaries of 0.63.0 crash before main() with SIGSEGV/SIGILL (issues #5738, #5735). Root cause, established from a core dump of the crashing binary under qemu: GNU ld emits corrupt R_ARM_IRELATIVE addends for libatomic's ifunc resolvers (wrong address and missing Thumb bit) once .text outgrows the 16MB Thumb branch range. glibc's static-init ifunc resolution then does `blx` into ARM-mode garbage and the process dies before any log output. v0.62.0 was unaffected only because its .text was still under 16MB (15.1MB); v0.63.0 crossed the line (17.5MB), so every 0.63.0 32-bit arm build crashes regardless of Go or dependency versions. Link 32-bit arm with LLD (already installed in the build stage), which emits correct IRELATIVE addends. Verified under qemu: the armv7 artifact built by the unchanged pipeline now boots to "Navidrome server is ready" with SQLite migrations working, where the previous binary segfaulted at startup. Also add a CI smoke test that runs each cross-compiled linux binary under binfmt/qemu right after building it, so any future crashes-at-startup-on-some-arch regression fails the pipeline instead of shipping in a release. --- .github/workflows/pipeline.yml | 15 +++++++++++++++ Dockerfile | 29 ++++++++++++++++++----------- release/build-tags.sh | 22 ++++++++++++++++++++++ release/verify-binary.sh | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 11 deletions(-) create mode 100755 release/build-tags.sh create mode 100755 release/verify-binary.sh diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 228bac9e7..8c714945e 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -300,6 +300,21 @@ jobs: GIT_SHA=${{ env.GIT_SHA }} GIT_TAG=${{ env.GIT_TAG }} + - name: Set up QEMU for smoke test + if: env.IS_LINUX == 'true' + uses: docker/setup-qemu-action@v3 + + # The binary is static, so binfmt+qemu runs it directly on the runner. + # Catches startup crashes in cross-compiled binaries before they ship, + # e.g. the broken ifunc relocations on 32-bit arm from issue #5738. + - name: Smoke-test binary + if: env.IS_LINUX == 'true' + run: | + BIN=./output/${{ env.PLATFORM }}/navidrome + chmod +x "$BIN" + "$BIN" --help >/dev/null + echo "OK: ${{ matrix.platform }} binary starts" + - name: Upload Binaries uses: actions/upload-artifact@v7 with: diff --git a/Dockerfile b/Dockerfile index e8a00f470..df5df52ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,20 +69,15 @@ RUN --mount=type=bind,source=. \ set -e xx-go --wrap export CGO_ENABLED=1 - # Native libwebp (gen2brain/webp) uses ebitengine/purego reverse callbacks, - # which purego does not support on 32-bit ARM or x86 and crash with a SIGSEGV - # (issue #5597). Build those arches with the "nodynamic" tag so gen2brain/webp - # is WASM-only and never links the purego path. 64-bit arches keep native libwebp. - BUILD_TAGS=netgo,sqlite_fts5 - if [ "$(xx-info arch)" = "arm" ] || [ "$(xx-info arch)" = "386" ]; then - BUILD_TAGS=${BUILD_TAGS},nodynamic - fi + BUILD_TAGS=$(./release/build-tags.sh) # -latomic is required on 32-bit arm (arm/v6, arm/v7) so SQLite's 64-bit atomics resolve. - go build -tags=${BUILD_TAGS} -ldflags="-w -s \ + go build -tags="${BUILD_TAGS}" -ldflags="-w -s \ -linkmode=external -extldflags '-latomic' \ -X github.com/navidrome/navidrome/consts.gitSha=${GIT_SHA} \ -X github.com/navidrome/navidrome/consts.gitTag=${GIT_TAG}" \ -o /out/navidrome . + # Fail the build if native libwebp (purego) leaked into a 32-bit binary (issue #5738). + ./release/verify-binary.sh /out/navidrome # Fail the build if the binary is accidentally statically linked: dlopen (and # therefore native libwebp detection) only works with a dynamic interpreter. file /out/navidrome | grep -q "dynamically linked" || { echo "ERROR: /out/navidrome is not dynamically linked"; file /out/navidrome; exit 1; } @@ -116,11 +111,12 @@ RUN --mount=type=bind,source=. \ --mount=from=osxcross,src=/osxcross/SDK,target=/xx-sdk,ro \ --mount=type=cache,target=/root/.cache \ --mount=type=cache,target=/go/pkg/mod </dev/null || true # Only Darwin (macOS) requires clang (default), Windows requires gcc, everything else can use any compiler. # So let's use gcc for everything except Darwin. @@ -129,14 +125,25 @@ RUN --mount=type=bind,source=. \ export CXX=$(xx-info)-g++ export LD_EXTRA="-extldflags '-static -latomic'" fi + # GNU ld corrupts the R_ARM_IRELATIVE addends of libatomic's ifunc resolvers + # (wrong address, Thumb bit lost) once .text outgrows the 16MB Thumb branch + # range, making static arm binaries jump to garbage inside glibc's ifunc + # resolution and crash before main() (issue #5738). Link 32-bit arm with LLD, + # which emits correct addends. + if [ "$(xx-info arch)" = "arm" ]; then + export LD_EXTRA="-extldflags '-static -latomic -fuse-ld=lld'" + fi if [ "$(xx-info os)" = "windows" ]; then export EXT=".exe" fi - go build -tags=netgo,sqlite_fts5 -ldflags="${LD_EXTRA} -w -s \ + BUILD_TAGS=$(./release/build-tags.sh) + go build -tags="${BUILD_TAGS}" -ldflags="${LD_EXTRA} -w -s \ -X github.com/navidrome/navidrome/consts.gitSha=${GIT_SHA} \ -X github.com/navidrome/navidrome/consts.gitTag=${GIT_TAG}" \ -o /out/navidrome${EXT} . + # Fail the build if native libwebp (purego) leaked into a 32-bit binary (issue #5738). + ./release/verify-binary.sh /out/navidrome* EOT # Verify if the binary was built for the correct platform and it is statically linked diff --git a/release/build-tags.sh b/release/build-tags.sh new file mode 100755 index 000000000..f719117ff --- /dev/null +++ b/release/build-tags.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Print the Go build tags for the xx-cc target platform (used by the Dockerfile). +# +# gen2brain/webp's native libwebp backend links ebitengine/purego, whose reverse +# callbacks are unsupported on 32-bit ARM and x86 and SIGSEGV at package-init time, +# taking the whole process down at startup (issues #5597 / #5606 / #5738). Force the +# WASM-only path there with the "nodynamic" tag; 64-bit arches keep native libwebp. +# +# This is the single source of truth for the tag decision: both Dockerfile build +# stages (Docker-image and standalone downloads) call it so they cannot drift apart. +set -e + +# Prefer xx-info (the cross-build target arch); fall back to `go env GOARCH` so the +# script is still correct when run outside the xx environment. Both report the +# cross-compilation target, unlike `uname -m`, which would report the build host. +arch=$(xx-info arch 2>/dev/null || go env GOARCH) + +tags="netgo,sqlite_fts5" +case "${arch}" in + arm | 386) tags="${tags},nodynamic" ;; +esac +printf '%s' "${tags}" diff --git a/release/verify-binary.sh b/release/verify-binary.sh new file mode 100755 index 000000000..cde775992 --- /dev/null +++ b/release/verify-binary.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Fail the build if a 32-bit ARM/x86 binary links ebitengine/purego, which would +# SIGSEGV at startup on those arches (issue #5738). +# +# Independent safety net for build-tags.sh: it inspects the actual build metadata +# recorded in the binary (survives stripping) instead of trusting the requested +# tags, so it still fires if the tag decision is wrong or gen2brain/webp changes +# its build-tag semantics. Runs in the Dockerfile, where xx-info and go are present. +# +# Usage: verify-binary.sh [...] +set -e + +# Prefer xx-info (the cross-build target arch); fall back to `go env GOARCH` so the +# check is still correct when run outside the xx environment. +arch=$(xx-info arch 2>/dev/null || go env GOARCH) + +case "${arch}" in + arm | 386) ;; + *) exit 0 ;; # 64-bit arches legitimately link purego for native libwebp +esac + +for bin in "$@"; do + # Fail loudly if the expected binary is missing (e.g. an unmatched glob), rather + # than letting `go version -m` fail inside the pipeline and silently pass. + if [ ! -f "${bin}" ]; then + echo "ERROR: expected binary '${bin}' not found; purego verification did not run." + exit 1 + fi + if go version -m "${bin}" | grep -q "ebitengine/purego"; then + echo "ERROR: 32-bit binary '${bin}' links ebitengine/purego; it will SIGSEGV at startup (issue #5738)." + echo " Ensure the 'nodynamic' build tag is applied (see release/build-tags.sh)." + exit 1 + fi +done