From 052f10fd68984a01cc2c87eee9edcc16dbfa6ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Thu, 9 Jul 2026 06:30:44 -0400 Subject: [PATCH 1/2] fix(build): prevent 32-bit startup crash (segfault/SIGILL) in downloads binaries (#5739) * fix(build): force nodynamic webp tag on 32-bit standalone binaries gen2brain/webp's native libwebp backend links ebitengine/purego, whose reverse callbacks are unsupported on 32-bit ARM and x86. purego registers its callback in package init(), so the binary crashes at startup (SIGSEGV or SIGILL) before any Navidrome code runs. The nodynamic build tag from #5606 forces the safe WASM path, but it was only applied to the Docker-image build stage. The standalone build stage, which produces the downloads-page tarballs and the deb/rpm packages, still linked purego, so the armv7/v6/v5 and 386 downloads crashed on launch (#5738, #5735). Move the tag decision into release/build-tags.sh, shared by both build stages so they can no longer drift, and add release/verify-binary.sh as a build-time guard that fails if a 32-bit binary links purego. * fix(build): harden webp build-tag scripts per review - verify-binary.sh: fail loudly when the target binary is missing (e.g. an unmatched glob) instead of letting `go version -m` fail inside a pipeline and silently pass, which would bypass the guard. - build-tags.sh / verify-binary.sh: fall back to `go env GOARCH` when xx-info is unavailable, so the scripts stay correct outside the xx build image. (Not `uname -m`, which reports the build host, not the cross target.) - Dockerfile: use `set -e` in the standalone build block and drop the redundant `|| exit 1` suffixes; keep the debug GOENV dump non-fatal. * chore(build): quote -tags argument in both build stages Defensive quoting per review; the value comes from release/build-tags.sh and contains no whitespace today, but quoting prevents word-splitting if it ever does. * fix(build): link 32-bit arm binaries with LLD to fix startup crash The standalone armv7/v6/v5 binaries of 0.63.0 crash before main() with SIGSEGV/SIGILL (issues #5738, #5735). Root cause, established from a core dump of the crashing binary under qemu: GNU ld emits corrupt R_ARM_IRELATIVE addends for libatomic's ifunc resolvers (wrong address and missing Thumb bit) once .text outgrows the 16MB Thumb branch range. glibc's static-init ifunc resolution then does `blx` into ARM-mode garbage and the process dies before any log output. v0.62.0 was unaffected only because its .text was still under 16MB (15.1MB); v0.63.0 crossed the line (17.5MB), so every 0.63.0 32-bit arm build crashes regardless of Go or dependency versions. Link 32-bit arm with LLD (already installed in the build stage), which emits correct IRELATIVE addends. Verified under qemu: the armv7 artifact built by the unchanged pipeline now boots to "Navidrome server is ready" with SQLite migrations working, where the previous binary segfaulted at startup. Also add a CI smoke test that runs each cross-compiled linux binary under binfmt/qemu right after building it, so any future crashes-at-startup-on-some-arch regression fails the pipeline instead of shipping in a release. --- .github/workflows/pipeline.yml | 15 +++++++++++++++ Dockerfile | 29 ++++++++++++++++++----------- release/build-tags.sh | 22 ++++++++++++++++++++++ release/verify-binary.sh | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 11 deletions(-) create mode 100755 release/build-tags.sh create mode 100755 release/verify-binary.sh diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 228bac9e7..8c714945e 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -300,6 +300,21 @@ jobs: GIT_SHA=${{ env.GIT_SHA }} GIT_TAG=${{ env.GIT_TAG }} + - name: Set up QEMU for smoke test + if: env.IS_LINUX == 'true' + uses: docker/setup-qemu-action@v3 + + # The binary is static, so binfmt+qemu runs it directly on the runner. + # Catches startup crashes in cross-compiled binaries before they ship, + # e.g. the broken ifunc relocations on 32-bit arm from issue #5738. + - name: Smoke-test binary + if: env.IS_LINUX == 'true' + run: | + BIN=./output/${{ env.PLATFORM }}/navidrome + chmod +x "$BIN" + "$BIN" --help >/dev/null + echo "OK: ${{ matrix.platform }} binary starts" + - name: Upload Binaries uses: actions/upload-artifact@v7 with: diff --git a/Dockerfile b/Dockerfile index e8a00f470..df5df52ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,20 +69,15 @@ RUN --mount=type=bind,source=. \ set -e xx-go --wrap export CGO_ENABLED=1 - # Native libwebp (gen2brain/webp) uses ebitengine/purego reverse callbacks, - # which purego does not support on 32-bit ARM or x86 and crash with a SIGSEGV - # (issue #5597). Build those arches with the "nodynamic" tag so gen2brain/webp - # is WASM-only and never links the purego path. 64-bit arches keep native libwebp. - BUILD_TAGS=netgo,sqlite_fts5 - if [ "$(xx-info arch)" = "arm" ] || [ "$(xx-info arch)" = "386" ]; then - BUILD_TAGS=${BUILD_TAGS},nodynamic - fi + BUILD_TAGS=$(./release/build-tags.sh) # -latomic is required on 32-bit arm (arm/v6, arm/v7) so SQLite's 64-bit atomics resolve. - go build -tags=${BUILD_TAGS} -ldflags="-w -s \ + go build -tags="${BUILD_TAGS}" -ldflags="-w -s \ -linkmode=external -extldflags '-latomic' \ -X github.com/navidrome/navidrome/consts.gitSha=${GIT_SHA} \ -X github.com/navidrome/navidrome/consts.gitTag=${GIT_TAG}" \ -o /out/navidrome . + # Fail the build if native libwebp (purego) leaked into a 32-bit binary (issue #5738). + ./release/verify-binary.sh /out/navidrome # Fail the build if the binary is accidentally statically linked: dlopen (and # therefore native libwebp detection) only works with a dynamic interpreter. file /out/navidrome | grep -q "dynamically linked" || { echo "ERROR: /out/navidrome is not dynamically linked"; file /out/navidrome; exit 1; } @@ -116,11 +111,12 @@ RUN --mount=type=bind,source=. \ --mount=from=osxcross,src=/osxcross/SDK,target=/xx-sdk,ro \ --mount=type=cache,target=/root/.cache \ --mount=type=cache,target=/go/pkg/mod </dev/null || true # Only Darwin (macOS) requires clang (default), Windows requires gcc, everything else can use any compiler. # So let's use gcc for everything except Darwin. @@ -129,14 +125,25 @@ RUN --mount=type=bind,source=. \ export CXX=$(xx-info)-g++ export LD_EXTRA="-extldflags '-static -latomic'" fi + # GNU ld corrupts the R_ARM_IRELATIVE addends of libatomic's ifunc resolvers + # (wrong address, Thumb bit lost) once .text outgrows the 16MB Thumb branch + # range, making static arm binaries jump to garbage inside glibc's ifunc + # resolution and crash before main() (issue #5738). Link 32-bit arm with LLD, + # which emits correct addends. + if [ "$(xx-info arch)" = "arm" ]; then + export LD_EXTRA="-extldflags '-static -latomic -fuse-ld=lld'" + fi if [ "$(xx-info os)" = "windows" ]; then export EXT=".exe" fi - go build -tags=netgo,sqlite_fts5 -ldflags="${LD_EXTRA} -w -s \ + BUILD_TAGS=$(./release/build-tags.sh) + go build -tags="${BUILD_TAGS}" -ldflags="${LD_EXTRA} -w -s \ -X github.com/navidrome/navidrome/consts.gitSha=${GIT_SHA} \ -X github.com/navidrome/navidrome/consts.gitTag=${GIT_TAG}" \ -o /out/navidrome${EXT} . + # Fail the build if native libwebp (purego) leaked into a 32-bit binary (issue #5738). + ./release/verify-binary.sh /out/navidrome* EOT # Verify if the binary was built for the correct platform and it is statically linked diff --git a/release/build-tags.sh b/release/build-tags.sh new file mode 100755 index 000000000..f719117ff --- /dev/null +++ b/release/build-tags.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Print the Go build tags for the xx-cc target platform (used by the Dockerfile). +# +# gen2brain/webp's native libwebp backend links ebitengine/purego, whose reverse +# callbacks are unsupported on 32-bit ARM and x86 and SIGSEGV at package-init time, +# taking the whole process down at startup (issues #5597 / #5606 / #5738). Force the +# WASM-only path there with the "nodynamic" tag; 64-bit arches keep native libwebp. +# +# This is the single source of truth for the tag decision: both Dockerfile build +# stages (Docker-image and standalone downloads) call it so they cannot drift apart. +set -e + +# Prefer xx-info (the cross-build target arch); fall back to `go env GOARCH` so the +# script is still correct when run outside the xx environment. Both report the +# cross-compilation target, unlike `uname -m`, which would report the build host. +arch=$(xx-info arch 2>/dev/null || go env GOARCH) + +tags="netgo,sqlite_fts5" +case "${arch}" in + arm | 386) tags="${tags},nodynamic" ;; +esac +printf '%s' "${tags}" diff --git a/release/verify-binary.sh b/release/verify-binary.sh new file mode 100755 index 000000000..cde775992 --- /dev/null +++ b/release/verify-binary.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Fail the build if a 32-bit ARM/x86 binary links ebitengine/purego, which would +# SIGSEGV at startup on those arches (issue #5738). +# +# Independent safety net for build-tags.sh: it inspects the actual build metadata +# recorded in the binary (survives stripping) instead of trusting the requested +# tags, so it still fires if the tag decision is wrong or gen2brain/webp changes +# its build-tag semantics. Runs in the Dockerfile, where xx-info and go are present. +# +# Usage: verify-binary.sh [...] +set -e + +# Prefer xx-info (the cross-build target arch); fall back to `go env GOARCH` so the +# check is still correct when run outside the xx environment. +arch=$(xx-info arch 2>/dev/null || go env GOARCH) + +case "${arch}" in + arm | 386) ;; + *) exit 0 ;; # 64-bit arches legitimately link purego for native libwebp +esac + +for bin in "$@"; do + # Fail loudly if the expected binary is missing (e.g. an unmatched glob), rather + # than letting `go version -m` fail inside the pipeline and silently pass. + if [ ! -f "${bin}" ]; then + echo "ERROR: expected binary '${bin}' not found; purego verification did not run." + exit 1 + fi + if go version -m "${bin}" | grep -q "ebitengine/purego"; then + echo "ERROR: 32-bit binary '${bin}' links ebitengine/purego; it will SIGSEGV at startup (issue #5738)." + echo " Ensure the 'nodynamic' build tag is applied (see release/build-tags.sh)." + exit 1 + fi +done From 4381366e663b4647cfec1eb5597c39c3c42e9d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Thu, 9 Jul 2026 19:43:04 -0400 Subject: [PATCH 2/2] ci: validate DB migration order on pull requests (#5750) * ci: add DB migration ordering/naming validation script * ci: run DB migration validation on pull requests * ci: don't reject non-migration .go files * ci: fetch latest master before validating migration order * ci: annotate the offending migration file on validation failure * ci: gate Build on the migration check so a bad migration fails fast * ci: reject migrations placed in a subdirectory of db/migrations * ci: reword fetch-step comment to not hard-code the base branch name --- .github/workflows/pipeline.yml | 23 +++- .github/workflows/validate-migrations.sh | 150 +++++++++++++++++++++++ 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100755 .github/workflows/validate-migrations.sh diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 8c714945e..d21d0a681 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -96,6 +96,23 @@ jobs: exit 1 fi + validate-migrations: + name: Validate DB migrations + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + # Refresh the base branch so the check compares against its CURRENT tip, + # not the (possibly stale) commit the PR was opened against. + - name: Fetch latest base branch + run: git fetch --no-tags origin "+refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}" + - name: Validate migration ordering and naming + env: + BASE_REF: origin/${{ github.event.pull_request.base.ref }} + run: ./.github/workflows/validate-migrations.sh + go: name: Test Go code runs-on: ubuntu-latest @@ -257,7 +274,11 @@ jobs: build: name: Build - needs: [js, go, go-windows, go-lint, i18n-lint, git-version, check-push-enabled] + needs: [js, go, go-windows, go-lint, i18n-lint, git-version, check-push-enabled, validate-migrations] + # validate-migrations only runs on pull_request, so it is "skipped" on push/tag + # builds. Run Build unless a dependency actually failed — a *skipped* dependency + # (the migration check on non-PR events) must not block release builds. + if: ${{ !cancelled() && !failure() }} strategy: matrix: platform: [ linux/amd64, linux/arm64, linux/arm/v5, linux/arm/v6, linux/arm/v7, linux/386, linux/riscv64, darwin/amd64, darwin/arm64, windows/amd64, windows/386 ] diff --git a/.github/workflows/validate-migrations.sh b/.github/workflows/validate-migrations.sh new file mode 100755 index 000000000..07d05c3a6 --- /dev/null +++ b/.github/workflows/validate-migrations.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# +# Validates DB migrations added by a pull request: +# 1. Ordering - an added migration must be NEWER than the latest migration +# already on the base branch. Goose applies migrations in +# timestamp order, so an older-timestamped migration would be +# silently skipped on databases already upgraded past it. +# 2. Uniqueness - no two migration files may share a timestamp. +# 3. Naming - files must match YYYYMMDDHHMMSS_lower_snake_name.(sql|go). +# +# On failure it prints a human-readable message and, when running in GitHub +# Actions, emits an error annotation bound to the offending file so the message +# also renders inline in the PR "Files changed" tab. +# +# Compares HEAD against $BASE_REF (default origin/master). Requires full history +# (fetch-depth: 0 in CI). +# -e is intentionally omitted: the script accumulates violations into $status +# and must not exit on the first non-zero command (grep no-match, a false [[ ]] +# in an if, `is_migration || continue`). +set -uo pipefail +export LC_ALL=C + +MIGRATIONS_DIR="db/migrations" +BASE_REF="${BASE_REF:-origin/master}" +NAME_RE='^[0-9]{14}_[a-z0-9_]+\.(sql|go)$' + +status=0 + +# Log a message to stderr and mark the run as failed. +fail() { + printf '%s\n' "$1" >&2 + status=1 +} + +# Emit a GitHub Actions error annotation bound to a file, so the message renders +# inline on the offending migration in the PR "Files changed" tab. No-op outside +# CI. `%`, newline and CR are encoded as required by the workflow-command syntax +# (the `%` replacement must run first so the encodings we add aren't re-escaped). +annotate() { # $1=file $2=message + [ "${GITHUB_ACTIONS:-}" = "true" ] || return 0 + local msg="$2" + msg="${msg//'%'/%25}" + msg="${msg//$'\n'/%0A}" + msg="${msg//$'\r'/%0D}" + printf '::error file=%s,line=1::%s\n' "$1" "$msg" +} + +# Report a migration problem: log it, annotate the offending file, mark failed. +report() { # $1=file $2=message + fail "$2" + printf '\n' >&2 + annotate "$1" "$2" +} + +human_ts() { + local t="$1" + printf '%s-%s-%s %s:%s:%s' "${t:0:4}" "${t:4:2}" "${t:6:2}" "${t:8:2}" "${t:10:2}" "${t:12:2}" +} + +is_migration() { # $1=basename -> 0 if a .sql/.go file with a 14-digit prefix + local b="$1" + case "$b" in + *.sql | *.go) ;; + *) return 1 ;; + esac + [[ "${b%%_*}" =~ ^[0-9]{14}$ ]] +} + +if ! git rev-parse --verify --quiet "$BASE_REF" >/dev/null; then + printf '❌ Cannot resolve base ref "%s". In CI, check out with fetch-depth: 0.\n' "$BASE_REF" >&2 + exit 1 +fi + +# --- Newest timestamp already on the base branch --- +base_max="" +base_max_file="" +while IFS= read -r f; do + [ -z "$f" ] && continue + b="$(basename "$f")" + is_migration "$b" || continue + ts="${b%%_*}" + if [[ "$ts" > "$base_max" ]]; then + base_max="$ts" + base_max_file="$f" + fi +done < <(git ls-tree -r --name-only "$BASE_REF" -- "$MIGRATIONS_DIR" 2>/dev/null) + +# --- Ordering + naming on files added by this PR --- +while IFS= read -r f; do + [ -z "$f" ] && continue + b="$(basename "$f")" + case "$b" in + *.sql) ;; # any .sql in this dir must be a migration + *.go) [[ "$b" == [0-9]* ]] || continue ;; # non-timestamped .go = helper (e.g. migration.go), skip + *) continue ;; + esac + if [ "${f%/*}" != "$MIGRATIONS_DIR" ]; then + report "$f" "❌ Migration file in a subdirectory: $f + Migrations must live directly in $MIGRATIONS_DIR/ — only $MIGRATIONS_DIR/*.sql (and + top-level .go migrations) are embedded, so a nested file would be SILENTLY SKIPPED. + Move it to $MIGRATIONS_DIR/$b." + continue + fi + if ! [[ "$b" =~ $NAME_RE ]]; then + report "$f" "❌ Malformed migration filename: $f + Expected YYYYMMDDHHMMSS_lower_snake_name.(sql|go); the name segment must be lowercase. + Regenerate with: make migration-sql name= (or make migration-go name=)" + continue + fi + ts="${b%%_*}" + if [[ -n "$base_max" ]] && ! [[ "$ts" > "$base_max" ]]; then + report "$f" "❌ Migration ordering error: $f ($(human_ts "$ts")) + is older than (or equal to) the newest migration already on ${BASE_REF#origin/}: + $base_max_file ($(human_ts "$base_max")) + + Goose applies migrations in timestamp order, so databases already upgraded + past that point would SILENTLY SKIP your migration. + + Fix: regenerate it with a current timestamp: + make migration-sql name= (or make migration-go name=) + then move your SQL/Go body into the new file and delete the old one." + fi +done < <(git diff --diff-filter=A --name-only "$BASE_REF"...HEAD -- "$MIGRATIONS_DIR" 2>/dev/null) + +# --- Duplicate timestamps across the merged set (HEAD) --- +all_migs="$(git ls-tree -r --name-only HEAD -- "$MIGRATIONS_DIR" 2>/dev/null)" +dups="$(printf '%s\n' "$all_migs" | while IFS= read -r f; do + b="$(basename "$f")" + is_migration "$b" || continue + printf '%s\n' "${b%%_*}" +done | sort | uniq -d)" +if [ -n "$dups" ]; then + while IFS= read -r ts; do + [ -z "$ts" ] && continue + colliding="$(printf '%s\n' "$all_migs" | grep "/${ts}_" || true)" + printf '❌ Duplicate migration timestamp %s used by multiple files:\n' "$ts" >&2 + while IFS= read -r cf; do + [ -z "$cf" ] && continue + printf ' %s\n' "$cf" >&2 + annotate "$cf" "Duplicate migration timestamp $ts — shared by another migration. Timestamps must be unique; regenerate one with make migration-*." + done <<< "$colliding" + printf ' Every migration needs a unique timestamp. Regenerate one with make migration-*.\n' >&2 + status=1 + done <<< "$dups" +fi + +if [ "$status" -eq 0 ]; then + echo "✅ DB migrations OK (ordering, uniqueness, naming)." +fi +exit "$status"