From 243172fbf47494235b873140186a037a4742e108 Mon Sep 17 00:00:00 2001 From: Jorge Ramirez Date: Sat, 27 Jun 2026 17:14:01 -0700 Subject: [PATCH] build(tensorrt): JetPack 7 / L4T R39 image with GPU-accelerated ONNX detector Adds a `-tensorrt-jp7` Frigate image for JetPack 7.2 / L4T R39.2 Jetson hosts, built on nvcr.io/nvidia/tensorrt:26.02-py3-igpu (TensorRT 10.11, CUDA 13, py3.12), keeping the existing JP6 path unchanged. ONNX Runtime GPU is built from source (no public aarch64 onnxruntime-gpu wheel), TensorRT-Python branch is selected by the base image, and TensorRT runtime library checks are major-version aware. VALIDATED on a real AGX Orin (L4T R39.2 / nv_tegra_release R39 rev 2.0): - image frigate:test-tensorrt-jp7 builds (rc=0, 16.3GB, sha256:c9e4d382f1603ee130ee4a7315b4f71f9461405e3785707251505e2d6d088df3) - ONNX Runtime 1.25.1 exposes CUDAExecutionProvider, and a real Add-model inference RAN on the iGPU CUDA EP (functional, not just listed) - frigate.util.model.get_ort_providers(False,"AUTO") = [CUDA, CPU] (CUDA first, CPU last, TensorRT EP excluded) -> the ONNX detector GPU-accelerates on JP7 - /etc/TENSORRT_VER = 10.11.0 Native `type: tensorrt` (.trt gen) + the ORT TensorRT EP stay DRAFT-GATED: the L4T R39 host ships NO libnvdla_compiler.so (absent from host AND base image), so `import tensorrt` and libonnxruntime_providers_tensorrt.so fail to load. ONNX detector GPU acceleration is the supported JP7 path; native trt is deferred. Build fixes the new noble/CUDA-13 base surfaced (beyond the plan): - build_nginx.sh: enable deb-src for the deb822 ubuntu.sources (Ubuntu 24.04) - tensorrt_libyolo.sh: strip -lnvToolsExt (removed in CUDA 13) + -lnvparsers (dropped in TensorRT 10) when those libs are absent - docker/main: noble/py3.12 build adjustments (Dockerfile, build_sqlite_vec.sh) Reproducible: `make -C docker/tensorrt ... local-trt-jp7` (JETPACK7_ARGS) on any arm64 builder; built on-device only because the GPU smoke test needs the iGPU. --- .github/workflows/ci.yml | 35 ++++++ .github/workflows/release.yml | 6 +- docker/main/Dockerfile | 18 ++- docker/main/build_nginx.sh | 10 +- docker/main/build_sqlite_vec.sh | 11 +- docker/tensorrt/Dockerfile.arm64 | 78 +++++++++--- docker/tensorrt/build_jetson_ffmpeg.sh | 119 ++++++++++++++++-- .../detector/build_onnxruntime_arm64.sh | 72 +++++++++++ .../detector/build_python_tensorrt.sh | 48 ++++++- .../s6-overlay/s6-rc.d/trt-model-prepare/run | 63 ++++++---- docker/tensorrt/detector/tensorrt_libyolo.sh | 15 +++ docker/tensorrt/trt.hcl | 24 ++++ docker/tensorrt/trt.mk | 12 ++ .../hardware_acceleration_enrichments.md | 2 +- .../hardware_acceleration_video.md | 8 +- docs/docs/configuration/object_detectors.md | 10 +- docs/docs/frigate/hardware.md | 6 +- docs/docs/frigate/installation.md | 3 +- .../DockerComposeGenerator/config/config.yaml | 19 ++- 19 files changed, 486 insertions(+), 73 deletions(-) create mode 100755 docker/tensorrt/detector/build_onnxruntime_arm64.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41080be5d9..139e23bc3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,41 @@ jobs: tensorrt.tags=${{ steps.setup.outputs.image-name }}-tensorrt-jp6 *.cache-from=type=registry,ref=${{ steps.setup.outputs.cache-name }}-jp6 *.cache-to=type=registry,ref=${{ steps.setup.outputs.cache-name }}-jp6,mode=max + jetson_jp7_build: + runs-on: ubuntu-22.04-arm + name: Jetson Jetpack 7 + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Set up QEMU and Buildx + id: setup + uses: ./.github/actions/setup + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push TensorRT (Jetson, Jetpack 7) + env: + ARCH: arm64 + BASE_IMAGE: nvcr.io/nvidia/tensorrt:26.02-py3-igpu + SLIM_BASE: nvcr.io/nvidia/tensorrt:26.02-py3-igpu + TRT_BASE: nvcr.io/nvidia/tensorrt:26.02-py3-igpu + BUILD_ONNXRUNTIME_FROM_SOURCE: "1" + ONNXRUNTIME_VERSION: "1.25.1" + ONNXRUNTIME_BRANCH: rel-1.25.1 + TENSORRT_PYTHON_BRANCH: auto + L4T_APT_RELEASE: r39.2 + JETSON_SOC_REPO: som + uses: docker/bake-action@v7 + with: + source: . + push: true + targets: tensorrt + files: docker/tensorrt/trt.hcl + set: | + tensorrt.tags=${{ steps.setup.outputs.image-name }}-tensorrt-jp7 + *.cache-from=type=registry,ref=${{ steps.setup.outputs.cache-name }}-jp7 + *.cache-to=type=registry,ref=${{ steps.setup.outputs.cache-name }}-jp7,mode=max amd64_extra_builds: runs-on: ubuntu-22.04 name: AMD64 Extra Build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1fbf58f6e3..5c2bce89ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,14 +39,16 @@ jobs: STABLE_TAG=${BASE}:stable PULL_TAG=${BASE}:${BUILD_TAG} docker run --rm -v $HOME/.docker/config.json:/config.json quay.io/skopeo/stable:latest copy --authfile /config.json --multi-arch all docker://${PULL_TAG} docker://${VERSION_TAG} - for variant in standard-arm64 tensorrt tensorrt-jp6 rk rocm synaptics; do + for variant in standard-arm64 tensorrt tensorrt-jp6 tensorrt-jp7 rk rocm synaptics; do + docker manifest inspect "${PULL_TAG}-${variant}" >/dev/null docker run --rm -v $HOME/.docker/config.json:/config.json quay.io/skopeo/stable:latest copy --authfile /config.json --multi-arch all docker://${PULL_TAG}-${variant} docker://${VERSION_TAG}-${variant} done # stable tag if [[ "${BUILD_TYPE}" == "stable" ]]; then docker run --rm -v $HOME/.docker/config.json:/config.json quay.io/skopeo/stable:latest copy --authfile /config.json --multi-arch all docker://${PULL_TAG} docker://${STABLE_TAG} - for variant in standard-arm64 tensorrt tensorrt-jp6 rk rocm synaptics; do + for variant in standard-arm64 tensorrt tensorrt-jp6 tensorrt-jp7 rk rocm synaptics; do + docker manifest inspect "${PULL_TAG}-${variant}" >/dev/null docker run --rm -v $HOME/.docker/config.json:/config.json quay.io/skopeo/stable:latest copy --authfile /config.json --multi-arch all docker://${PULL_TAG}-${variant} docker://${STABLE_TAG}-${variant} done fi diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile index 1a475a650c..5c34b3c3f4 100644 --- a/docker/main/Dockerfile +++ b/docker/main/Dockerfile @@ -87,7 +87,9 @@ RUN apt-get -qq update \ && apt-get -qq install -y wget python3 python3-dev python3-distutils gcc pkg-config libhdf5-dev \ && wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ && sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \ - && python3 get-pip.py "pip" \ + # --ignore-installed: the Ubuntu 24.04 (noble) JP7 base ships a distro pip with no RECORD + # file, so get-pip.py's default reinstall dies with "uninstall-no-record-file". Harmless on JP6. + && python3 get-pip.py --ignore-installed "pip" \ && pip3 install -r /requirements-ov.txt # Get OpenVino Model @@ -181,9 +183,11 @@ RUN apt-get -qq update \ RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 +# --ignore-installed: the Ubuntu 24.04 (noble) JP7 base ships a distro pip with no RECORD file, +# so get-pip.py's default reinstall dies with "uninstall-no-record-file". Harmless on JP6. RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ && sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \ - && python3 get-pip.py "pip" + && python3 get-pip.py --ignore-installed "pip" COPY docker/main/requirements.txt /requirements.txt COPY docker/main/requirements-dev.txt /requirements-dev.txt @@ -268,12 +272,18 @@ RUN --mount=type=bind,source=docker/main/install_deps.sh,target=/deps/install_de ENV DEFAULT_FFMPEG_VERSION="8.0" ENV INCLUDED_FFMPEG_VERSIONS="${DEFAULT_FFMPEG_VERSION}:7.0:5.0" +# --ignore-installed: the Ubuntu 24.04 (noble) JP7 base ships a distro pip with no RECORD file, +# so get-pip.py's default reinstall dies with "uninstall-no-record-file". Harmless on JP6. RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ && sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \ - && python3 get-pip.py "pip" + && python3 get-pip.py --ignore-installed "pip" +# --ignore-installed: on the Ubuntu 24.04 (noble) JP7 base, `pip3 install -U` tries to upgrade +# distro-managed packages (e.g. wheel 0.42.0) that ship without a RECORD file, which fails with +# "uninstall-no-record-file". Installing over them into /usr/local without uninstalling avoids +# that; harmless on JP6 where these come from pip and have RECORD files. RUN --mount=type=bind,from=wheels,source=/wheels,target=/deps/wheels \ - pip3 install -U /deps/wheels/*.whl + pip3 install -U --ignore-installed /deps/wheels/*.whl # Install Axera Engine RUN pip3 install https://github.com/AXERA-TECH/pyaxengine/releases/download/0.1.3-frigate/axengine-0.1.3-py3-none-any.whl diff --git a/docker/main/build_nginx.sh b/docker/main/build_nginx.sh index 708a4cb45c..97c4bf905d 100755 --- a/docker/main/build_nginx.sh +++ b/docker/main/build_nginx.sh @@ -10,9 +10,15 @@ NGX_DEVEL_KIT_VERSION="v0.3.3" source /etc/os-release -if [[ "$VERSION_ID" == "12" ]]; then +# Enable deb-src so `apt-get build-dep nginx` can find the source package. Detect the apt +# source FORMAT rather than guessing by distro: Debian 12 + Ubuntu 24.04 (noble — the JP7 +# TensorRT igpu base) use the deb822 *.sources format with a `Types:` line; older Ubuntu +# (e.g. the JP6 22.04 jammy base) uses the legacy one-line /etc/apt/sources.list. +if [[ -f /etc/apt/sources.list.d/debian.sources ]]; then sed -i '/^Types:/s/deb/& deb-src/' /etc/apt/sources.list.d/debian.sources -else +elif [[ -f /etc/apt/sources.list.d/ubuntu.sources ]]; then + sed -i '/^Types:/s/deb/& deb-src/' /etc/apt/sources.list.d/ubuntu.sources +elif [[ -f /etc/apt/sources.list ]]; then cp /etc/apt/sources.list /etc/apt/sources.list.d/sources-src.list sed -i 's|deb http|deb-src http|g' /etc/apt/sources.list.d/sources-src.list fi diff --git a/docker/main/build_sqlite_vec.sh b/docker/main/build_sqlite_vec.sh index b41f3383d9..2111693375 100755 --- a/docker/main/build_sqlite_vec.sh +++ b/docker/main/build_sqlite_vec.sh @@ -6,9 +6,16 @@ SQLITE_VEC_VERSION="0.1.3" source /etc/os-release -if [[ "$VERSION_ID" == "12" ]]; then +# Enable deb-src so `apt-get build-dep sqlite3` can find the source package. Detect the apt +# source FORMAT rather than guessing by distro: Debian 12 + Ubuntu 24.04 (noble — the JP7 +# TensorRT igpu base) use the deb822 *.sources format with a `Types:` line; older Ubuntu +# (e.g. the JP6 22.04 jammy base) uses the legacy one-line /etc/apt/sources.list. On noble the +# legacy /etc/apt/sources.list is effectively empty, so the old copy+sed path enabled nothing. +if [[ -f /etc/apt/sources.list.d/debian.sources ]]; then sed -i '/^Types:/s/deb/& deb-src/' /etc/apt/sources.list.d/debian.sources -else +elif [[ -f /etc/apt/sources.list.d/ubuntu.sources ]]; then + sed -i '/^Types:/s/deb/& deb-src/' /etc/apt/sources.list.d/ubuntu.sources +elif [[ -f /etc/apt/sources.list ]]; then cp /etc/apt/sources.list /etc/apt/sources.list.d/sources-src.list sed -i 's|deb http|deb-src http|g' /etc/apt/sources.list.d/sources-src.list fi diff --git a/docker/tensorrt/Dockerfile.arm64 b/docker/tensorrt/Dockerfile.arm64 index dd3c5de5e3..f5ab0bd9ed 100644 --- a/docker/tensorrt/Dockerfile.arm64 +++ b/docker/tensorrt/Dockerfile.arm64 @@ -4,6 +4,12 @@ ARG DEBIAN_FRONTEND=noninteractive ARG BASE_IMAGE ARG TRT_BASE=nvcr.io/nvidia/tensorrt:23.12-py3 +ARG BUILD_ONNXRUNTIME_FROM_SOURCE=0 +ARG ONNXRUNTIME_VERSION=1.25.1 +ARG ONNXRUNTIME_BRANCH=rel-1.25.1 +ARG TENSORRT_PYTHON_BRANCH=release/8.6 +ARG L4T_APT_RELEASE= +ARG JETSON_SOC_REPO= # Build TensorRT-specific library FROM ${TRT_BASE} AS trt-deps @@ -12,7 +18,9 @@ ARG TARGETARCH ARG COMPUTE_LEVEL RUN apt-get update \ - && apt-get install -y git build-essential cuda-nvcc-* cuda-nvtx-* libnvinfer-dev libnvinfer-plugin-dev libnvparsers-dev libnvonnxparsers-dev \ + && TRT_DEV_PACKAGES="git build-essential cuda-nvcc-* cuda-nvtx-* libnvinfer-dev libnvinfer-plugin-dev libnvonnxparsers-dev" \ + && if apt-cache show libnvparsers-dev > /dev/null 2>&1; then TRT_DEV_PACKAGES="${TRT_DEV_PACKAGES} libnvparsers-dev"; fi \ + && apt-get install -y ${TRT_DEV_PACKAGES} \ && rm -rf /var/lib/apt/lists/* RUN --mount=type=bind,source=docker/tensorrt/detector/tensorrt_libyolo.sh,target=/tensorrt_libyolo.sh \ /tensorrt_libyolo.sh @@ -58,6 +66,12 @@ HEALTHCHECK --start-period=600s --start-interval=5s --interval=15s --timeout=5s FROM ${BASE_IMAGE} AS build-wheels ARG DEBIAN_FRONTEND +ARG BUILD_ONNXRUNTIME_FROM_SOURCE +ARG ONNXRUNTIME_VERSION +ARG ONNXRUNTIME_BRANCH +ARG TENSORRT_PYTHON_BRANCH +ARG L4T_APT_RELEASE +ARG JETSON_SOC_REPO # Add deadsnakes PPA for python3.11 RUN apt-get -qq update && \ @@ -69,41 +83,66 @@ RUN apt-get -qq update && \ RUN apt-get -qq update \ && apt-get -qq install -y --no-install-recommends \ python3.11 python3.11-dev \ - wget build-essential cmake git \ + wget curl build-essential cmake git ninja-build \ && rm -rf /var/lib/apt/lists/* # Ensure python3 defaults to python3.11 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 +# --ignore-installed: on the JP7 Ubuntu 24.04 (noble) base the deadsnakes python3.11 ships a +# distro-packaged pip 24.0 with no RECORD file, so get-pip.py's default reinstall fails with +# "uninstall-no-record-file". Installing over it without uninstalling avoids that; harmless on JP6. RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ && sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \ - && python3 get-pip.py "pip" + && python3 get-pip.py --ignore-installed "pip" FROM build-wheels AS trt-wheels ARG DEBIAN_FRONTEND ARG TARGETARCH +ARG BUILD_ONNXRUNTIME_FROM_SOURCE +ARG ONNXRUNTIME_VERSION +ARG ONNXRUNTIME_BRANCH +ARG TENSORRT_PYTHON_BRANCH # python-tensorrt build deps are 3.4 GB! RUN apt-get update \ - && apt-get install -y ccache cuda-cudart-dev-* cuda-nvcc-* libnvonnxparsers-dev libnvparsers-dev libnvinfer-plugin-dev \ + && TRT_DEV_PACKAGES="ccache cuda-cudart-dev-* cuda-nvcc-* libnvonnxparsers-dev libnvinfer-plugin-dev" \ + && if apt-cache show libnvparsers-dev > /dev/null 2>&1; then TRT_DEV_PACKAGES="${TRT_DEV_PACKAGES} libnvparsers-dev"; fi \ + && apt-get install -y ${TRT_DEV_PACKAGES} \ && ([ -e /usr/local/cuda ] || ln -s /usr/local/cuda-* /usr/local/cuda) \ && rm -rf /var/lib/apt/lists/*; +# The JP7 base image ships a stale cmake 3.24 at /usr/local/bin/cmake that shadows apt's 3.28. +# TensorRT 10.11's python CMakeLists (and the ONNX Runtime source build) require cmake >= 3.27, +# so install a current cmake from pip into /usr/local/bin ahead of the stale one. No-op risk on +# JP6 where the bundled cmake already satisfies the minimum. +RUN pip3 install --no-cache-dir --upgrade "cmake>=3.27" + # Determine version of tensorrt already installed in base image, e.g. "Version: 8.4.1-1+cuda11.4" -RUN NVINFER_VER=$(dpkg -s libnvinfer8 | grep -Po "Version: \K.*") \ - && echo $NVINFER_VER | grep -Po "^\d+\.\d+\.\d+" > /etc/TENSORRT_VER +RUN NVINFER_VER="$(dpkg-query -W -f='${Version}\n' 'libnvinfer*' 2>/dev/null | grep -E '^[0-9]+[.][0-9]+[.][0-9]+' | sort -V | tail -n1)" \ + && test -n "$NVINFER_VER" \ + && TENSORRT_VER="$(echo "$NVINFER_VER" | grep -Eo '^[0-9]+[.][0-9]+[.][0-9]+')" \ + && TENSORRT_MAJOR="$(echo "$TENSORRT_VER" | cut -d. -f1)" \ + && echo "$TENSORRT_VER" > /etc/TENSORRT_VER \ + && echo "$TENSORRT_MAJOR" > /etc/TENSORRT_MAJOR RUN --mount=type=bind,source=docker/tensorrt/detector/build_python_tensorrt.sh,target=/deps/build_python_tensorrt.sh \ --mount=type=cache,target=/root/.ccache \ export PATH="/usr/lib/ccache:$PATH" CCACHE_DIR=/root/.ccache CCACHE_MAXSIZE=2G \ - && TENSORRT_VER=$(cat /etc/TENSORRT_VER) /deps/build_python_tensorrt.sh + && TENSORRT_VER=$(cat /etc/TENSORRT_VER) TENSORRT_PYTHON_BRANCH="${TENSORRT_PYTHON_BRANCH}" /deps/build_python_tensorrt.sh COPY docker/tensorrt/requirements-arm64.txt /requirements-tensorrt.txt RUN pip3 wheel --wheel-dir=/trt-wheels -r /requirements-tensorrt.txt # See https://elinux.org/Jetson_Zoo#ONNX_Runtime -ADD https://nvidia.box.com/shared/static/9yvw05k6u343qfnkhdv2x6xhygze0aq1.whl /trt-wheels/onnxruntime_gpu-1.19.0-cp311-cp311-linux_aarch64.whl +RUN --mount=type=bind,source=docker/tensorrt/detector/build_onnxruntime_arm64.sh,target=/deps/build_onnxruntime_arm64.sh \ + mkdir -p /trt-wheels \ + && if [ "${BUILD_ONNXRUNTIME_FROM_SOURCE}" = "1" ]; then \ + ONNXRUNTIME_VERSION="${ONNXRUNTIME_VERSION}" ONNXRUNTIME_BRANCH="${ONNXRUNTIME_BRANCH}" /deps/build_onnxruntime_arm64.sh; \ + else \ + wget -q https://nvidia.box.com/shared/static/9yvw05k6u343qfnkhdv2x6xhygze0aq1.whl -O /trt-wheels/onnxruntime_gpu-1.19.0-cp311-cp311-linux_aarch64.whl; \ + fi FROM build-wheels AS trt-model-wheels ARG DEBIAN_FRONTEND @@ -116,16 +155,20 @@ RUN --mount=type=bind,source=docker/tensorrt/requirements-models-arm64.txt,targe FROM wget AS jetson-ffmpeg ARG DEBIAN_FRONTEND +ARG L4T_APT_RELEASE +ARG JETSON_SOC_REPO ENV CCACHE_DIR /root/.ccache ENV CCACHE_MAXSIZE 2G RUN --mount=type=bind,source=docker/tensorrt/build_jetson_ffmpeg.sh,target=/deps/build_jetson_ffmpeg.sh \ --mount=type=cache,target=/root/.ccache \ - /deps/build_jetson_ffmpeg.sh + L4T_APT_RELEASE="${L4T_APT_RELEASE}" JETSON_SOC_REPO="${JETSON_SOC_REPO}" /deps/build_jetson_ffmpeg.sh # Frigate w/ TensorRT for NVIDIA Jetson platforms FROM tensorrt-base AS frigate-tensorrt RUN apt-get update \ - && apt-get install -y python-is-python3 libprotobuf23 \ + && pick_package() { for package in "$@"; do if apt-cache show "$package" > /dev/null 2>&1; then echo "$package"; return 0; fi; done; return 1; } \ + && PROTOBUF_RUNTIME="$(pick_package libprotobuf23 libprotobuf32t64 libprotobuf32)" \ + && apt-get install -y python-is-python3 "${PROTOBUF_RUNTIME}" \ && rm -rf /var/lib/apt/lists/* COPY --from=jetson-ffmpeg /rootfs / @@ -134,23 +177,30 @@ ENV INCLUDED_FFMPEG_VERSIONS="${DEFAULT_FFMPEG_VERSION}:${INCLUDED_FFMPEG_VERSIO # ffmpeg runtime dependencies RUN apt-get -qq update \ + && pick_package() { for package in "$@"; do if apt-cache show "$package" > /dev/null 2>&1; then echo "$package"; return 0; fi; done; return 1; } \ + && X264_RUNTIME="$(pick_package libx264-163 libx264-164)" \ + && X265_RUNTIME="$(pick_package libx265-199 libx265-209)" \ && apt-get -qq install -y --no-install-recommends \ - libx264-163 libx265-199 libegl1 \ + "${X264_RUNTIME}" "${X265_RUNTIME}" libegl1 \ && rm -rf /var/lib/apt/lists/* # Fixes "Error loading shared libs" RUN mkdir -p /etc/ld.so.conf.d && echo /usr/lib/ffmpeg/jetson/lib/ > /etc/ld.so.conf.d/ffmpeg.conf COPY --from=trt-wheels /etc/TENSORRT_VER /etc/TENSORRT_VER +COPY --from=trt-wheels /etc/TENSORRT_MAJOR /etc/TENSORRT_MAJOR RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \ --mount=type=bind,from=trt-model-wheels,source=/trt-model-wheels,target=/deps/trt-model-wheels \ pip3 uninstall -y onnxruntime \ - && pip3 install -U /deps/trt-wheels/*.whl \ - && pip3 install -U /deps/trt-model-wheels/*.whl \ + # --ignore-installed: on the noble JP7 base, `-U` would try to upgrade distro-managed deps that + # lack a RECORD file (e.g. wheel/setuptools), failing with "uninstall-no-record-file". Install + # the TRT/ORT GPU wheels over them into /usr/local instead. Harmless on JP6. + && pip3 install -U --ignore-installed /deps/trt-wheels/*.whl \ + && pip3 install -U --ignore-installed /deps/trt-model-wheels/*.whl \ && ldconfig WORKDIR /opt/frigate/ COPY --from=rootfs / / # Fixes "Error importing detector runtime: /usr/lib/aarch64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block" -ENV LD_PRELOAD /usr/lib/aarch64-linux-gnu/libstdc++.so.6 \ No newline at end of file +ENV LD_PRELOAD /usr/lib/aarch64-linux-gnu/libstdc++.so.6 diff --git a/docker/tensorrt/build_jetson_ffmpeg.sh b/docker/tensorrt/build_jetson_ffmpeg.sh index fb29eb2141..b617efe852 100755 --- a/docker/tensorrt/build_jetson_ffmpeg.sh +++ b/docker/tensorrt/build_jetson_ffmpeg.sh @@ -8,18 +8,81 @@ set -euxo pipefail INSTALL_PREFIX=/rootfs/usr/lib/ffmpeg/jetson apt-get -qq update -apt-get -qq install -y --no-install-recommends build-essential ccache clang cmake pkg-config +apt-get -qq install -y --no-install-recommends build-essential ccache clang cmake pkg-config unzip apt-get -qq install -y --no-install-recommends libx264-dev libx265-dev pushd /tmp +CUDA_MAJOR="" +if [ -f /usr/local/cuda/version.json ]; then + CUDA_MAJOR="$(grep -m1 -oE '"version"[[:space:]]*:[[:space:]]*"[^"]+"' /usr/local/cuda/version.json | sed -E 's/.*"([0-9]+)\..*/\1/' || true)" +fi +if [ -z "${CUDA_MAJOR}" ] && compgen -G "/usr/local/cuda-13*" > /dev/null; then + CUDA_MAJOR=13 +fi +if [ -z "${CUDA_MAJOR}" ] && command -v nvcc > /dev/null 2>&1; then + CUDA_MAJOR="$(nvcc --version | sed -nE 's/.*release ([0-9]+)\..*/\1/p' | head -n1 || true)" +fi + +# Tracks whether this is the JP7/R39 path, so the ffmpeg build below can link the real R39 +# tegra multimedia libs instead of the stale jetson-ffmpeg stubs (see TEGRA_LIB_DIR usage). +JETSON_R39=0 +TEGRA_LIB_DIR=/usr/lib/aarch64-linux-gnu/nvidia + # Install libnvmpi to enable nvmpi decoders (h264_nvmpi, hevc_nvmpi) -if [ -e /usr/local/cuda-12 ]; then +if [[ "${CUDA_MAJOR}" = "13" || -n "${L4T_APT_RELEASE:-}" || -n "${JETSON_SOC_REPO:-}" ]]; then + JETSON_R39=1 + L4T_APT_RELEASE=${L4T_APT_RELEASE:-r39.2} + JETSON_SOC_REPO=${JETSON_SOC_REPO:-som} + + apt-key adv --fetch-key https://repo.download.nvidia.com/jetson/jetson-ota-public.asc + { + echo "deb https://repo.download.nvidia.com/jetson/common ${L4T_APT_RELEASE} main" + echo "deb https://repo.download.nvidia.com/jetson/${JETSON_SOC_REPO} ${L4T_APT_RELEASE} main" + echo "deb https://repo.download.nvidia.com/jetson/ffmpeg ${L4T_APT_RELEASE} main" + } >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list + + mkdir -p /opt/nvidia/l4t-packages/ + touch /opt/nvidia/l4t-packages/.nv-l4t-disable-boot-fw-update-in-preinstall + + apt-get update + apt-get -qq install -y --no-install-recommends -o Dpkg::Options::="--force-confold" nvidia-l4t-jetson-multimedia-api + + require_jetson_file() { + local description="$1" + local pattern="$2" + shift 2 + local directory + local found + + # Match case-insensitively: R39.2 (JP7) renamed the multimedia headers to all-lowercase + # (e.g. nvbufsurftransform.h) while older L4T shipped CamelCase (NvBufSurfTransform.h). The + # jetson-ffmpeg build's nvUtils2NvBuf.h includes the lowercase names, so the lowercase + # headers (present on R39.2) satisfy the build; only this validation gate was casing-strict. + for directory in "$@"; do + found="$(find "${directory}" -type f -iname "${pattern}" -print -quit 2> /dev/null || true)" + if [ -n "${found}" ]; then + echo "${description}: ${found}" + return 0 + fi + done + + echo "Missing ${description}" >&2 + return 1 + } + + require_jetson_file "Jetson multimedia header NvBufSurface.h" "NvBufSurface.h" /usr/src/jetson_multimedia_api /usr/include + require_jetson_file "Jetson multimedia header NvBufSurfTransform.h" "NvBufSurfTransform.h" /usr/src/jetson_multimedia_api /usr/include + require_jetson_file "Jetson multimedia library libnvbufsurface" "libnvbufsurface.so*" /usr/lib /usr/local/lib + require_jetson_file "Jetson multimedia library libnvbufsurftransform" "libnvbufsurftransform.so*" /usr/lib /usr/local/lib +elif [ -e /usr/local/cuda-12 ]; then # assume Jetpack 6.2 apt-key adv --fetch-key https://repo.download.nvidia.com/jetson/jetson-ota-public.asc - echo "deb https://repo.download.nvidia.com/jetson/common r36.4 main" >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list - echo "deb https://repo.download.nvidia.com/jetson/t234 r36.4 main" >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list - echo "deb https://repo.download.nvidia.com/jetson/ffmpeg r36.4 main" >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list + { + echo "deb https://repo.download.nvidia.com/jetson/common r36.4 main" + echo "deb https://repo.download.nvidia.com/jetson/t234 r36.4 main" + echo "deb https://repo.download.nvidia.com/jetson/ffmpeg r36.4 main" + } >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list mkdir -p /opt/nvidia/l4t-packages/ touch /opt/nvidia/l4t-packages/.nv-l4t-disable-boot-fw-update-in-preinstall @@ -38,11 +101,39 @@ fi wget -q https://github.com/AndBobsYourUncle/jetson-ffmpeg/archive/9c17b09.zip -O jetson-ffmpeg.zip unzip jetson-ffmpeg.zip && rm jetson-ffmpeg.zip && mv jetson-ffmpeg-* jetson-ffmpeg && cd jetson-ffmpeg -LD_LIBRARY_PATH=$(pwd)/stubs:$LD_LIBRARY_PATH # tegra multimedia libs aren't available in image, so use stubs for ffmpeg build +# On R39/JP7 the real tegra multimedia libs ARE installed (by nvidia-l4t-jetson-multimedia-api), +# living in /usr/lib/aarch64-linux-gnu/nvidia. The 2023-era jetson-ffmpeg stubs predate R39 and +# lack newer symbols (e.g. NvBufSurfaceGetDeviceInfo), so libnvmpi.so links with an unresolved +# reference and ffmpeg's later `-lnvmpi` configure test fails. Put the real lib dir ahead of the +# stubs so libnvmpi resolves against R39's libnvbufsurface. Those real libs in turn need the CUDA +# driver (libcuda.so.1) and the CUDA runtime libs, which are only present at container RUNTIME via +# the nvidia runtime — so for the build-time link we point -rpath-link at the CUDA *stubs* dir +# (libcuda.so stub) and the CUDA lib dir. Pre-R39 keeps the stubs-only path. +if [ "${JETSON_R39}" = "1" ]; then + CUDA_STUBS_DIR="$(dirname "$(find /usr/local/cuda*/targets/*/lib/stubs -name libcuda.so 2>/dev/null | head -n1)")" + CUDA_LIB_DIR="$(dirname "$(find /usr/local/cuda*/targets/*/lib -maxdepth 1 -name 'libcudart.so*' 2>/dev/null | head -n1)")" + # The CUDA stub ships only unversioned libcuda.so, but R39's libnvbufsurface.so.1.0.0 records a + # NEEDED entry for the versioned soname libcuda.so.1. -rpath-link looks up that exact soname, so + # without a libcuda.so.1 the transitive cu* driver symbols stay unresolved at link time. Provide + # a build-only versioned symlink to the stub (real libcuda.so.1 is injected at container runtime). + CUDA_SONAME_DIR=/tmp/cuda-soname-compat + mkdir -p "${CUDA_SONAME_DIR}" + if [ -n "${CUDA_STUBS_DIR}" ]; then + ln -sf "${CUDA_STUBS_DIR}/libcuda.so" "${CUDA_SONAME_DIR}/libcuda.so.1" + ln -sf "${CUDA_STUBS_DIR}/libcuda.so" "${CUDA_SONAME_DIR}/libcuda.so" + fi + export LD_LIBRARY_PATH=${TEGRA_LIB_DIR}:${CUDA_SONAME_DIR}:${CUDA_STUBS_DIR}:${CUDA_LIB_DIR}:$(pwd)/stubs:${LD_LIBRARY_PATH:-} + EXTRA_NVMPI_LDFLAGS="-L${TEGRA_LIB_DIR} -Wl,-rpath-link,${TEGRA_LIB_DIR}" + [ -n "${CUDA_STUBS_DIR}" ] && EXTRA_NVMPI_LDFLAGS="${EXTRA_NVMPI_LDFLAGS} -L${CUDA_STUBS_DIR} -Wl,-rpath-link,${CUDA_STUBS_DIR} -L${CUDA_SONAME_DIR} -Wl,-rpath-link,${CUDA_SONAME_DIR}" + [ -n "${CUDA_LIB_DIR}" ] && EXTRA_NVMPI_LDFLAGS="${EXTRA_NVMPI_LDFLAGS} -Wl,-rpath-link,${CUDA_LIB_DIR}" +else + export LD_LIBRARY_PATH=$(pwd)/stubs:${LD_LIBRARY_PATH:-} # tegra multimedia libs aren't available in image, so use stubs for ffmpeg build + EXTRA_NVMPI_LDFLAGS="" +fi mkdir build cd build -cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX -make -j$(nproc) +cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX ${EXTRA_NVMPI_LDFLAGS:+-DCMAKE_SHARED_LINKER_FLAGS="${EXTRA_NVMPI_LDFLAGS}"} +make -j"$(nproc)" make install cd ../../ @@ -57,14 +148,24 @@ wget -q https://ffmpeg.org/releases/ffmpeg-6.0.tar.xz tar xaf ffmpeg-*.tar.xz && rm ffmpeg-*.tar.xz && cd ffmpeg-* patch -p1 < ../jetson-ffmpeg/ffmpeg_patches/ffmpeg6.0_nvmpi.patch export PKG_CONFIG_PATH=$INSTALL_PREFIX/lib/pkgconfig +# On R39/JP7, ffmpeg's `-lnvmpi` configure probe (and final link) must see the real tegra libs in +# /usr/lib/aarch64-linux-gnu/nvidia AND the CUDA driver/runtime stubs to resolve everything libnvmpi +# transitively pulls in (NvBufSurfaceGetDeviceInfo from libnvbufsurface, and that lib's own +# libcuda.so.1 / cu* driver-API deps). Reuse the same link flags assembled for the libnvmpi build +# (TEGRA + CUDA stubs + CUDA lib dirs). Pre-R39 leaves configure flags unchanged. +FFMPEG_EXTRA_LDFLAGS="" +if [ "${JETSON_R39}" = "1" ]; then + FFMPEG_EXTRA_LDFLAGS="${EXTRA_NVMPI_LDFLAGS}" +fi # enable Jetson codecs but disable dGPU codecs ./configure --cc='ccache gcc' --cxx='ccache g++' \ --enable-shared --disable-static --prefix=$INSTALL_PREFIX \ --enable-gpl --enable-libx264 --enable-libx265 \ --enable-nvmpi --enable-ffnvcodec --enable-cuda-llvm \ --disable-cuvid --disable-nvenc --disable-nvdec \ + ${FFMPEG_EXTRA_LDFLAGS:+--extra-ldflags="${FFMPEG_EXTRA_LDFLAGS}"} \ || { cat ffbuild/config.log && false; } -make -j$(nproc) +make -j"$(nproc)" make install cd ../ diff --git a/docker/tensorrt/detector/build_onnxruntime_arm64.sh b/docker/tensorrt/detector/build_onnxruntime_arm64.sh new file mode 100755 index 0000000000..fc9a5698d6 --- /dev/null +++ b/docker/tensorrt/detector/build_onnxruntime_arm64.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -euxo pipefail + +ONNXRUNTIME_VERSION=${ONNXRUNTIME_VERSION:-1.25.1} +ONNXRUNTIME_BRANCH=${ONNXRUNTIME_BRANCH:-rel-1.25.1} +ORT_PARALLEL=${ORT_PARALLEL:-4} +ORT_NVCC_THREADS=${ORT_NVCC_THREADS:-1} + +mkdir -p /trt-wheels /workspace + +pip3 install --upgrade pip setuptools wheel packaging numpy + +command -v nvcc +test -f /usr/local/cuda/include/cuda.h + +require_header() { + local pattern="$1" + local header + + for header in /usr/include/${pattern} /usr/include/aarch64-linux-gnu/${pattern}; do + if [[ -e "${header}" ]]; then + echo "${header}" + return 0 + fi + done + + echo "Missing required header matching ${pattern}" >&2 + return 1 +} + +require_ldconfig_entry() { + local lib_name="$1" + + if ! ldconfig -p | grep -q "${lib_name}\\.so"; then + echo "Missing ldconfig entry for ${lib_name}" >&2 + return 1 + fi +} + +require_header "cudnn.h" +require_header "cudnn_version*.h" +require_header "NvInfer.h" +require_header "NvOnnxParser.h" + +require_ldconfig_entry "libnvinfer" +require_ldconfig_entry "libnvinfer_plugin" +require_ldconfig_entry "libnvonnxparser" + +cd /workspace +rm -rf onnxruntime +git clone --recursive --branch "${ONNXRUNTIME_BRANCH}" https://github.com/microsoft/onnxruntime.git onnxruntime + +cd /workspace/onnxruntime +./build.sh \ + --config Release \ + --update \ + --build \ + --build_wheel \ + --parallel "${ORT_PARALLEL}" \ + --nvcc_threads "${ORT_NVCC_THREADS}" \ + --allow_running_as_root \ + --compile_no_warning_as_error \ + --skip_tests \ + --use_cuda \ + --cuda_home /usr/local/cuda \ + --use_tensorrt \ + --cudnn_home /usr/lib/aarch64-linux-gnu \ + --tensorrt_home /usr/lib/aarch64-linux-gnu \ + --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF + +cp build/Linux/Release/dist/onnxruntime_gpu-"${ONNXRUNTIME_VERSION}"-*.whl /trt-wheels/ diff --git a/docker/tensorrt/detector/build_python_tensorrt.sh b/docker/tensorrt/detector/build_python_tensorrt.sh index 325103485b..c40648def2 100755 --- a/docker/tensorrt/detector/build_python_tensorrt.sh +++ b/docker/tensorrt/detector/build_python_tensorrt.sh @@ -2,6 +2,19 @@ set -euxo pipefail +TENSORRT_PYTHON_BRANCH=${TENSORRT_PYTHON_BRANCH:-release/8.6} + +if [[ -n "${TENSORRT_VER:-}" ]]; then + TENSORRT_MAJOR_MINOR="${TENSORRT_VER%.*}" + EXPECTED_TENSORRT_PYTHON_BRANCH="release/${TENSORRT_MAJOR_MINOR}" + + if [[ "${TENSORRT_PYTHON_BRANCH}" == "auto" ]]; then + TENSORRT_PYTHON_BRANCH="${EXPECTED_TENSORRT_PYTHON_BRANCH}" + elif [[ "${TENSORRT_PYTHON_BRANCH}" != "${EXPECTED_TENSORRT_PYTHON_BRANCH}" ]]; then + echo "WARNING: TENSORRT_PYTHON_BRANCH=${TENSORRT_PYTHON_BRANCH} does not match TENSORRT_VER=${TENSORRT_VER}; auto would select ${EXPECTED_TENSORRT_PYTHON_BRANCH}" >&2 + fi +fi + mkdir -p /trt-wheels if [[ "${TARGETARCH}" == "arm64" ]]; then @@ -12,13 +25,44 @@ if [[ "${TARGETARCH}" == "arm64" ]]; then # Get python-tensorrt source mkdir -p /workspace cd /workspace - git clone -b release/8.6 https://github.com/NVIDIA/TensorRT.git --depth=1 + git clone -b "${TENSORRT_PYTHON_BRANCH}" https://github.com/NVIDIA/TensorRT.git --depth=1 + + # The TensorRT python build runs the legacy `setup.py bdist_wheel`, which on the JP7 base + # otherwise resolves the deadsnakes/Debian system setuptools in /usr/lib/python3/dist-packages. + # That copy's wheel.bdist_wheel routes through distutils `install` -> `install_lib`, whose + # finalize_options reads a Debian-only `install_layout` option that the install command lacks + # under this setuptools, crashing with "AttributeError: install_layout". Install a clean, + # non-Debian setuptools+wheel into /usr/local (which python3.11 imports ahead of /usr/lib) + # so bdist_wheel uses a consistent toolchain. JP6 already had a compatible setuptools. + pip3 install --upgrade --ignore-installed 'setuptools>=70.1,<81' 'wheel>=0.43' # Collect dependencies EXT_PATH=/workspace/external && mkdir -p $EXT_PATH pip3 install pybind11 && ln -s /usr/local/lib/python3.11/dist-packages/pybind11 $EXT_PATH/pybind11 ln -s /usr/include/python3.11 $EXT_PATH/python3.11 - ln -s /usr/include/aarch64-linux-gnu/NvOnnxParser.h /workspace/TensorRT/parsers/onnx/ + + # TensorRT 10's python onnx bindings (pyOnnx.cpp) #include onnx-tensorrt headers such as + # errorHelpers.hpp that live ONLY in the parsers/onnx submodule (onnx-tensorrt), not in the + # base image's system include or the shallow TensorRT clone. TRT 8.6 bindings did not need + # these, so a lone NvOnnxParser.h symlink sufficed there. Populate the submodule at its pinned + # ref so CMake's ONNX_INC_DIR=${TENSORRT_ROOT}/parsers/onnx resolves every header it needs. + cd /workspace/TensorRT + if git submodule update --init --depth=1 parsers/onnx 2>/dev/null && [[ -e parsers/onnx/errorHelpers.hpp ]]; then + : + else + # Fallback: clone onnx-tensorrt at the branch recorded in .gitmodules for this TRT release. + ONNX_TRT_BRANCH="$(git config -f .gitmodules submodule.parsers/onnx.branch || echo '')" + rm -rf parsers/onnx + git clone --depth=1 ${ONNX_TRT_BRANCH:+-b "${ONNX_TRT_BRANCH}"} https://github.com/onnx/onnx-tensorrt.git parsers/onnx + fi + # Prefer the system NvOnnxParser.h (matches the installed libnvonnxparser) when present. + for header in /usr/include/aarch64-linux-gnu/NvOnnxParser.h /usr/include/NvOnnxParser.h; do + if [[ -e "$header" ]]; then + ln -sf "$header" /workspace/TensorRT/parsers/onnx/NvOnnxParser.h + break + fi + done + cd /workspace # Build wheel cd /workspace/TensorRT/python diff --git a/docker/tensorrt/detector/rootfs/etc/s6-overlay/s6-rc.d/trt-model-prepare/run b/docker/tensorrt/detector/rootfs/etc/s6-overlay/s6-rc.d/trt-model-prepare/run index e3440e7ac9..559c4d42c1 100755 --- a/docker/tensorrt/detector/rootfs/etc/s6-overlay/s6-rc.d/trt-model-prepare/run +++ b/docker/tensorrt/detector/rootfs/etc/s6-overlay/s6-rc.d/trt-model-prepare/run @@ -10,11 +10,12 @@ set -o errexit -o nounset -o pipefail MODEL_CACHE_DIR=${MODEL_CACHE_DIR:-"/config/model_cache/tensorrt"} TRT_VER=${TRT_VER:-$(cat /etc/TENSORRT_VER)} +TRT_MAJOR=${TRT_VER%%.*} OUTPUT_FOLDER="${MODEL_CACHE_DIR}/${TRT_VER}" YOLO_MODELS=${YOLO_MODELS:-""} # Create output folder -mkdir -p ${OUTPUT_FOLDER} +mkdir -p "${OUTPUT_FOLDER}" FIRST_MODEL=true MODEL_DOWNLOAD="" @@ -28,9 +29,9 @@ fi for model in ${YOLO_MODELS//,/ } do # Remove old link in case path/version changed - rm -f ${MODEL_CACHE_DIR}/${model}.trt + rm -f "${MODEL_CACHE_DIR}/${model}.trt" - if [[ ! -f ${OUTPUT_FOLDER}/${model}.trt ]]; then + if [[ ! -f "${OUTPUT_FOLDER}/${model}.trt" ]]; then if [[ ${FIRST_MODEL} = true ]]; then MODEL_DOWNLOAD="${model%-dla}"; MODEL_CONVERT="${model}" @@ -40,7 +41,7 @@ do MODEL_CONVERT+=",${model}"; fi else - ln -s ${OUTPUT_FOLDER}/${model}.trt ${MODEL_CACHE_DIR}/${model}.trt + ln -s "${OUTPUT_FOLDER}/${model}.trt" "${MODEL_CACHE_DIR}/${model}.trt" fi done @@ -50,8 +51,8 @@ if [[ -z ${MODEL_CONVERT} ]]; then fi # Setup ENV to select GPU for conversion -if [ ! -z ${TRT_MODEL_PREP_DEVICE+x} ]; then - if [ ! -z ${CUDA_VISIBLE_DEVICES+x} ]; then +if [ -n "${TRT_MODEL_PREP_DEVICE+x}" ]; then + if [ -n "${CUDA_VISIBLE_DEVICES+x}" ]; then PREVIOUS_CVD="$CUDA_VISIBLE_DEVICES" unset CUDA_VISIBLE_DEVICES fi @@ -67,12 +68,30 @@ if [[ "$(arch)" == "aarch64" ]]; then if [[ ! -e /usr/lib/aarch64-linux-gnu/tegra && ! -e /usr/lib/aarch64-linux-gnu/tegra-egl ]]; then echo "ERROR: Container must be launched with nvidia runtime" exit 1 - elif [[ ! -e /usr/lib/aarch64-linux-gnu/libnvinfer.so.8 || - ! -e /usr/lib/aarch64-linux-gnu/libnvinfer_plugin.so.8 || - ! -e /usr/lib/aarch64-linux-gnu/libnvparsers.so.8 || - ! -e /usr/lib/aarch64-linux-gnu/libnvonnxparser.so.8 ]]; then - echo "ERROR: Please run the following on the HOST:" - echo " sudo apt install libnvinfer8 libnvinfer-plugin8 libnvparsers8 libnvonnxparsers8 nvidia-container" + fi + + TRT_LIB_DIR=/usr/lib/aarch64-linux-gnu + REQUIRED_TRT_LIBS=( + "${TRT_LIB_DIR}/libnvinfer.so.${TRT_MAJOR}" + "${TRT_LIB_DIR}/libnvinfer_plugin.so.${TRT_MAJOR}" + "${TRT_LIB_DIR}/libnvonnxparser.so.${TRT_MAJOR}" + ) + + if (( TRT_MAJOR < 10 )) || [[ -e "${TRT_LIB_DIR}/libnvparsers.so.${TRT_MAJOR}" ]]; then + REQUIRED_TRT_LIBS+=("${TRT_LIB_DIR}/libnvparsers.so.${TRT_MAJOR}") + fi + + MISSING_TRT_LIBS=() + for TRT_LIB in "${REQUIRED_TRT_LIBS[@]}"; do + if [[ ! -e "${TRT_LIB}" ]]; then + MISSING_TRT_LIBS+=("${TRT_LIB}") + fi + done + + if (( ${#MISSING_TRT_LIBS[@]} > 0 )); then + echo "ERROR: Missing TensorRT runtime libraries:" + printf ' %s\n' "${MISSING_TRT_LIBS[@]}" + echo "ERROR: Install the matching TensorRT ${TRT_MAJOR} runtime packages and nvidia-container on the HOST." exit 1 fi fi @@ -83,33 +102,33 @@ echo "Generating the following TRT Models: ${MODEL_CONVERT}" cd /usr/local/src/tensorrt_demos/yolo echo "Downloading yolo weights" -./download_yolo.sh $MODEL_DOWNLOAD 2> /dev/null +./download_yolo.sh "${MODEL_DOWNLOAD}" 2> /dev/null for model in ${MODEL_CONVERT//,/ } do - python3 yolo_to_onnx.py -m ${model%-dla} > /dev/null + python3 yolo_to_onnx.py -m "${model%-dla}" > /dev/null echo -e "\nGenerating ${model}.trt. This may take a few minutes.\n"; start=$(date +%s) if [[ $model == *-dla ]]; then - cmd="python3 onnx_to_tensorrt.py -m ${model%-dla} --dla_core 0" + cmd=(python3 onnx_to_tensorrt.py -m "${model%-dla}" --dla_core 0) else - cmd="python3 onnx_to_tensorrt.py -m ${model}" + cmd=(python3 onnx_to_tensorrt.py -m "${model}") fi - $cmd > /tmp/onnx_to_tensorrt.log || { cat /tmp/onnx_to_tensorrt.log && continue; } + "${cmd[@]}" > /tmp/onnx_to_tensorrt.log || { cat /tmp/onnx_to_tensorrt.log && continue; } - mv ${model%-dla}.trt ${OUTPUT_FOLDER}/${model}.trt; - ln -s ${OUTPUT_FOLDER}/${model}.trt ${MODEL_CACHE_DIR}/${model}.trt + mv "${model%-dla}.trt" "${OUTPUT_FOLDER}/${model}.trt"; + ln -s "${OUTPUT_FOLDER}/${model}.trt" "${MODEL_CACHE_DIR}/${model}.trt" echo "Generated ${model}.trt in $(($(date +%s)-start)) seconds" done # Restore ENV after conversion -if [ ! -z ${TRT_MODEL_PREP_DEVICE+x} ]; then +if [ -n "${TRT_MODEL_PREP_DEVICE+x}" ]; then unset CUDA_VISIBLE_DEVICES - if [ ! -z ${PREVIOUS_CVD+x} ]; then + if [ -n "${PREVIOUS_CVD+x}" ]; then export CUDA_VISIBLE_DEVICES="$PREVIOUS_CVD" fi fi # Print which models exist in output folder echo "Available tensorrt models:" -cd ${OUTPUT_FOLDER} && ls *.trt; +cd "${OUTPUT_FOLDER}" && ls -- *.trt; diff --git a/docker/tensorrt/detector/tensorrt_libyolo.sh b/docker/tensorrt/detector/tensorrt_libyolo.sh index 46e4077fac..a542cb943b 100755 --- a/docker/tensorrt/detector/tensorrt_libyolo.sh +++ b/docker/tensorrt/detector/tensorrt_libyolo.sh @@ -7,6 +7,21 @@ SCRIPT_DIR="/usr/local/src/tensorrt_demos" # Clone tensorrt_demos repo git clone --depth 1 https://github.com/NateMeyer/tensorrt_demos.git -b conditional_download +# CUDA 13 removed libnvToolsExt.so (NVTX is now header-only nvtx3). The plugin Makefile links +# -lnvToolsExt only for optional NVTX profiling annotations, so strip it when the lib is absent — +# the link then succeeds on JP7 / CUDA 13. No-op on CUDA 12 (JP6), where the lib still exists. +if ! ldconfig -p | grep -q 'libnvToolsExt\.so'; then + sed -i 's/-lnvToolsExt//g' ./tensorrt_demos/plugins/Makefile +fi + +# TensorRT 10 (JP7 base) dropped libnvparsers.so (the legacy UFF/Caffe parsers). The plugin +# Makefile links -lnvparsers from an over-broad LIBS list, but the YOLO custom layer never uses +# it, so strip it when the lib is absent — the link then succeeds on TRT 10. No-op on TRT 8 +# (JP5/JP6), where libnvparsers still ships. +if ! ldconfig -p | grep -q 'libnvparsers\.so'; then + sed -i 's/-lnvparsers//g' ./tensorrt_demos/plugins/Makefile +fi + # Build libyolo if [ ! -e /usr/local/cuda ]; then ln -s /usr/local/cuda-* /usr/local/cuda diff --git a/docker/tensorrt/trt.hcl b/docker/tensorrt/trt.hcl index 501e871e96..fb958feead 100644 --- a/docker/tensorrt/trt.hcl +++ b/docker/tensorrt/trt.hcl @@ -13,6 +13,24 @@ variable "TRT_BASE" { variable "COMPUTE_LEVEL" { default = "" } +variable "BUILD_ONNXRUNTIME_FROM_SOURCE" { + default = "0" +} +variable "ONNXRUNTIME_VERSION" { + default = "1.25.1" +} +variable "ONNXRUNTIME_BRANCH" { + default = "rel-1.25.1" +} +variable "TENSORRT_PYTHON_BRANCH" { + default = "release/8.6" +} +variable "L4T_APT_RELEASE" { + default = "" +} +variable "JETSON_SOC_REPO" { + default = "" +} variable "BASE_HOOK" { # Ensure an up-to-date python 3.11 is available in jetson images default = < -- [TensortRT](#nvidia-tensorrt-detector): TensorRT can run on Jetson devices, using one of many default models. -- [ONNX](#onnx): TensorRT will automatically be detected and used as a detector in the `-tensorrt-jp6` Frigate image when a supported ONNX model is configured. +- [TensortRT](#nvidia-tensorrt-detector): TensorRT can run on JetPack 6 / L4T R36 Jetson devices, using one of many default models. +- [ONNX](#onnx): Jetson GPU acceleration will automatically be detected and used by the ONNX detector in the `-tensorrt-jp6` image on JetPack 6 / L4T R36 or the `-tensorrt-jp7` image on JetPack 7.2 / L4T R39.2 when a supported ONNX model is configured. **Rockchip** @@ -437,7 +437,7 @@ If the correct build is used for your GPU then the GPU will be detected and used - **Nvidia** - Nvidia GPUs will automatically be detected and used with the ONNX detector in the `-tensorrt` Frigate image. - - Jetson devices will automatically be detected and used with the ONNX detector in the `-tensorrt-jp6` Frigate image. + - Jetson devices will automatically be detected and used with the ONNX detector in the `-tensorrt-jp6` Frigate image on JetPack 6 / L4T R36 or the `-tensorrt-jp7` Frigate image on JetPack 7.2 / L4T R39.2. ::: @@ -567,7 +567,9 @@ For detailed instructions on compiling models, refer to the [MemryX Compiler](ht ## NVidia TensorRT Detector -Nvidia Jetson devices may be used for object detection using the TensorRT libraries. Due to the size of the additional libraries, this detector is only provided in images with the `-tensorrt-jp6` tag suffix, e.g. `ghcr.io/blakeblackshear/frigate:stable-tensorrt-jp6`. This detector is designed to work with Yolo models for object detection. +Nvidia Jetson devices may be used for object detection using the TensorRT libraries on JetPack 6 / L4T R36. Due to the size of the additional libraries, this detector is only provided in images with the `-tensorrt-jp6` tag suffix, e.g. `ghcr.io/blakeblackshear/frigate:stable-tensorrt-jp6`. This detector is designed to work with Yolo models for object detection. + +On JetPack 7.2 / L4T R39.2, the `-tensorrt-jp7` image initially supports ONNX detector GPU acceleration. Native `type: tensorrt` engine generation on JP7 is pending hardware validation. ### Generate Models diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index abe4630653..7b4942c4ae 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -90,7 +90,7 @@ Frigate supports multiple different detectors that work on different types of ha - [Supports majority of model architectures via ONNX](../../configuration/object_detectors#onnx) - Runs well with any size models including large -- [Jetson](#nvidia-jetson): Jetson devices are supported via the TensorRT or ONNX detectors when running Jetpack 6. +- [Jetson](#nvidia-jetson): Jetson devices are supported via the TensorRT or ONNX detectors when running JetPack 6 / L4T R36. On JetPack 7.2 / L4T R39.2, the initial supported path is ONNX detector GPU acceleration. **Rockchip** @@ -258,9 +258,9 @@ Inference speeds may vary depending on the host platform. The above data was mea ### Nvidia Jetson -Jetson devices are supported via the TensorRT or ONNX detectors when running Jetpack 6. It will [make use of the Jetson's hardware media engine](/configuration/hardware_acceleration_video#nvidia-jetson) when configured with the [appropriate presets](/configuration/ffmpeg_presets#hwaccel-presets), and will make use of the Jetson's GPU and DLA for object detection when configured with the [TensorRT detector](/configuration/object_detectors#nvidia-tensorrt-detector). +Jetson devices running JetPack 6 / L4T R36 are supported via the TensorRT or ONNX detectors. Jetson devices running JetPack 7.2 / L4T R39.2 use the `stable-tensorrt-jp7` image and initially support ONNX detector GPU acceleration; native `type: tensorrt` engine generation on JP7 is pending hardware validation. The Jetson images will [make use of the Jetson's hardware media engine](/configuration/hardware_acceleration_video#nvidia-jetson) when configured with the [appropriate presets](/configuration/ffmpeg_presets#hwaccel-presets). The JetPack 6 image will also make use of the Jetson's GPU and DLA for object detection when configured with the [TensorRT detector](/configuration/object_detectors#nvidia-tensorrt-detector). -Inference speed will vary depending on the YOLO model, jetson platform and jetson nvpmodel (GPU/DLA/EMC clock speed). It is typically 20-40 ms for most models. The DLA is more efficient than the GPU, but not faster, so using the DLA will reduce power consumption but will slightly increase inference time. +For JetPack 6 TensorRT detector usage, inference speed will vary depending on the YOLO model, jetson platform and jetson nvpmodel (GPU/DLA/EMC clock speed). It is typically 20-40 ms for most models. The DLA is more efficient than the GPU, but not faster, so using the DLA will reduce power consumption but will slightly increase inference time. ### Rockchip platform diff --git a/docs/docs/frigate/installation.md b/docs/docs/frigate/installation.md index f69218f780..6c71fc8505 100644 --- a/docs/docs/frigate/installation.md +++ b/docs/docs/frigate/installation.md @@ -556,7 +556,8 @@ The official docker image tags for the current stable version are: The community supported docker image tags for the current stable version are: -- `stable-tensorrt-jp6` - Frigate build optimized for Nvidia Jetson devices running Jetpack 6 +- `stable-tensorrt-jp6` - Frigate build optimized for Nvidia Jetson devices running JetPack 6 / L4T R36 +- `stable-tensorrt-jp7` - Frigate build optimized for Nvidia Jetson devices running JetPack 7.2 / L4T R39.2 - `stable-rk` - Frigate build for SBCs with Rockchip SoC ## Home Assistant App diff --git a/docs/src/components/DockerComposeGenerator/config/config.yaml b/docs/src/components/DockerComposeGenerator/config/config.yaml index 42199ffca1..c13cd3ee38 100644 --- a/docs/src/components/DockerComposeGenerator/config/config.yaml +++ b/docs/src/components/DockerComposeGenerator/config/config.yaml @@ -35,9 +35,10 @@ devices: helpType: "warning" needsNvidiaConfig: true - - id: "stable-tensorrt-jp6" - name: "NVIDIA Jetson" - description: "Jetson development board" + - &jetsonDevice + id: "stable-tensorrt-jp6" + name: "NVIDIA Jetson JP6" + description: "JetPack 6 / L4T R36" icon: '' svgStyle: width: 50px @@ -46,10 +47,17 @@ devices: padding-bottom: 15px imageTag: "stable-tensorrt-jp6" autoHardware: [] - helpText: "NVIDIA Jetson devices automatically configure runtime: nvidia." + helpText: "JetPack 6 / L4T R36 Jetson devices automatically configure runtime: nvidia." helpType: "info" runtime: "nvidia" + - <<: *jetsonDevice + id: "stable-tensorrt-jp7" + name: "NVIDIA Jetson JP7" + description: "JetPack 7.2 / L4T R39.2" + imageTag: "stable-tensorrt-jp7" + helpText: "JetPack 7.2 / L4T R39.2 Jetson devices automatically configure runtime: nvidia." + - id: "stable-rocm" name: "AMD GPU" description: "ROCm acceleration" @@ -176,6 +184,7 @@ hardware: description: "Pass through /dev/dri for GPU hardware acceleration (Intel/AMD)." disabledWhen: - "stable-tensorrt-jp6" + - "stable-tensorrt-jp7" - "apple-silicon" devices: - host: "/dev/dri" @@ -187,6 +196,7 @@ hardware: description: "Pass through /dev/accel for Intel NPU acceleration." disabledWhen: - "stable-tensorrt-jp6" + - "stable-tensorrt-jp7" - "apple-silicon" - "stable-rocm" - "stable-rk" @@ -247,6 +257,7 @@ hardware: disabledWhen: - "stable-tensorrt" - "stable-tensorrt-jp6" + - "stable-tensorrt-jp7" - "stable-rocm" - "stable-rk" - "stable-synaptics"