From 74b3a9fcb6634e86b8809eac5f89402abca6cac2 Mon Sep 17 00:00:00 2001 From: Joseph McQueen Date: Mon, 6 Jul 2026 10:02:43 -0400 Subject: [PATCH] fix(docker): actually cross-build linux/amd64 image on Apple Silicon The default buildx builder (docker driver) is bound to the daemon's native platform, so on an Apple Silicon Mac `--platform linux/amd64 --load` was silently producing an arm64 image. The bundle then failed on the linux/amd64 target host with the "exec format error" that install.sh's arch sanity check now surfaces as "Image architecture (arm64) does not match this host (amd64)". package.sh now: - Creates a dedicated `sha-remote-agent-builder` (docker-container driver) on first run so cross-arch builds actually work. - Best-effort installs tonistiigi/binfmt QEMU handlers when the target platform differs from the host. - Uses `--output type=docker,dest=...` instead of `--load` + `docker save`, bypassing the local daemon's cross-arch storage limits entirely. - Verifies the produced image's Architecture against --platform after the build and aborts if they disagree, so a broken ZIP can never leave the build host. - Only re-tags :latest when the built platform matches the host, to avoid leaving a broken cross-arch :latest in the local daemon. README documents the trap and the mitigations. Co-authored-by: Cursor --- docker/remote-agent/README.md | 26 ++++++++ docker/remote-agent/package.sh | 107 ++++++++++++++++++++++++++++----- 2 files changed, 118 insertions(+), 15 deletions(-) diff --git a/docker/remote-agent/README.md b/docker/remote-agent/README.md index e83f58f..cb323ba 100644 --- a/docker/remote-agent/README.md +++ b/docker/remote-agent/README.md @@ -160,6 +160,32 @@ Cross-building requires `docker buildx` — Docker Desktop ships it by default; on Linux install the `docker-buildx-plugin` package if it isn't already there. +#### How the script avoids the "silent arm64 image" trap + +The default buildx builder on Docker Desktop uses the `docker` driver, which +is bound to the daemon's native platform. Passing `--platform linux/amd64` +to it from an Apple Silicon host can silently produce an `arm64` image (or, +depending on the Desktop version, ignore the flag with only a warning). To +sidestep that, `package.sh`: + +1. Creates a dedicated `sha-remote-agent-builder` with the `docker-container` + driver on first run (isolated BuildKit instance, cross-arch capable). +2. Best-effort installs `tonistiigi/binfmt` QEMU handlers when the target + platform doesn't match the host. +3. Writes the image directly to a tarball via + `--output type=docker,dest=...` instead of `--load` + `docker save`, + so the local daemon's cross-arch storage limits are irrelevant. +4. **Verifies** the produced image's `Architecture` against `--platform` + after the build and aborts the packaging run if they disagree — so a + broken ZIP can never leave the build host. + +If the verification ever fires, install binfmt explicitly and rebuild: + +```bash +docker run --privileged --rm tonistiigi/binfmt --install all +./docker/remote-agent/package.sh --platform linux/amd64 +``` + ### Manual export (without the packaging script) If you'd rather do it by hand: diff --git a/docker/remote-agent/package.sh b/docker/remote-agent/package.sh index 3efe749..b03103e 100755 --- a/docker/remote-agent/package.sh +++ b/docker/remote-agent/package.sh @@ -111,30 +111,107 @@ mkdir -p "$STAGING_ROOT" cleanup() { rm -rf "$STAGING_DIR"; } trap cleanup EXIT -# --- 1. Build the image ----------------------------------------------------- +# Normalize host arch into a linux/* platform string for cross-build detection. +HOST_ARCH_RAW="$(uname -m)" +case "$HOST_ARCH_RAW" in + x86_64|amd64) HOST_LINUX_PLATFORM="linux/amd64" ;; + aarch64|arm64) HOST_LINUX_PLATFORM="linux/arm64" ;; + armv7l) HOST_LINUX_PLATFORM="linux/arm/v7" ;; + *) HOST_LINUX_PLATFORM="linux/${HOST_ARCH_RAW}" ;; +esac -log "Building Docker image ${IMAGE_TAG} for ${PLATFORM} (context = ${REPO_ROOT})..." -# buildx with --load emits the image straight into the local Docker daemon -# so `docker save` in the next step picks it up. --load only supports one -# platform at a time, which matches our "one ZIP per target" workflow. +EXPECTED_ARCH="${PLATFORM##*/}" + +# --- 1. Ensure a cross-arch-capable buildx builder -------------------------- +# The DEFAULT buildx builder on Docker Desktop uses the "docker" driver, which +# is tied to the daemon's native platform. Passing --platform linux/amd64 on +# an arm64 host with that driver can silently produce an arm64 image (which +# is exactly the "image arch does not match host" failure the install.sh +# sanity check catches on the target machine). +# +# We work around it by creating (once) a dedicated builder with the +# "docker-container" driver, which spins up an isolated BuildKit instance +# capable of cross-arch builds when QEMU/binfmt is available. + +BUILDER_NAME="sha-remote-agent-builder" +if ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then + log "Creating dedicated buildx builder '${BUILDER_NAME}' (docker-container driver)..." + docker buildx create \ + --name "$BUILDER_NAME" \ + --driver docker-container \ + --bootstrap >/dev/null +else + # Make sure the builder is up (bootstrap is a no-op if it already is). + docker buildx inspect --bootstrap "$BUILDER_NAME" >/dev/null +fi + +# --- 2. Cross-arch binfmt (only when needed) -------------------------------- +# Docker Desktop ships QEMU/binfmt handlers by default so this usually no-ops, +# but plain Docker Engine, Colima, or rootless setups often don't. When we're +# cross-building, best-effort install binfmt for the target arch. If it fails +# (e.g. no --privileged, no internet, no image), warn but continue — the +# subsequent build step will fail fast with a clearer error if binfmt truly +# is missing. + +if [[ "$PLATFORM" != "$HOST_LINUX_PLATFORM" ]]; then + log "Cross-arch build (${HOST_LINUX_PLATFORM} -> ${PLATFORM}); ensuring binfmt handlers..." + if ! docker run --privileged --rm tonistiigi/binfmt --install "$EXPECTED_ARCH" >/dev/null 2>&1; then + warn "Could not auto-install binfmt for ${EXPECTED_ARCH}. If the build fails, install it manually:" + warn " docker run --privileged --rm tonistiigi/binfmt --install all" + fi +fi + +# --- 3. Build directly to a portable tarball -------------------------------- +# `--output type=docker,dest=...` writes a `docker load`-compatible tarball +# straight to disk. This intentionally bypasses `--load` (and therefore the +# question of whether the local daemon can even store cross-arch images). + +UNZIPPED_TAR="$STAGING_ROOT/${BUNDLE_NAME}.tar" +log "Building ${IMAGE_TAG} for ${PLATFORM} -> $(basename "$UNZIPPED_TAR") (context = ${REPO_ROOT})..." docker buildx build \ + --builder "$BUILDER_NAME" \ --platform "$PLATFORM" \ - --load \ + --output "type=docker,dest=${UNZIPPED_TAR},name=${IMAGE_TAG}" \ -f "$DOCKERFILE" \ - -t "$IMAGE_TAG" \ - -t "sha-remote-agent:latest" \ "$REPO_ROOT" -# --- 2. Save the image to a gzipped tarball -------------------------------- +[[ -s "$UNZIPPED_TAR" ]] || die "buildx produced no output tarball. Aborting." + +# --- 4. Verify the built image actually matches --platform ------------------ +# Regression guard: if buildx (or binfmt) silently ignored the requested +# platform, catch it here instead of shipping a broken bundle that only +# fails on the remote host with an "exec format error". + +log "Verifying built image architecture..." +docker load -i "$UNZIPPED_TAR" >/dev/null +ACTUAL_ARCH="$(docker image inspect --format '{{.Architecture}}' "$IMAGE_TAG")" +if [[ "$ACTUAL_ARCH" != "$EXPECTED_ARCH" ]]; then + die "Built image architecture is '${ACTUAL_ARCH}' but '${EXPECTED_ARCH}' was requested. + This usually means buildx couldn't cross-compile for ${PLATFORM}. + Try installing binfmt handlers explicitly: + docker run --privileged --rm tonistiigi/binfmt --install all + Then re-run: + $0 --platform ${PLATFORM}" +fi +log "Verified: image architecture is ${ACTUAL_ARCH} (matches requested ${EXPECTED_ARCH})." + +# Also tag :latest locally for convenience (only when it matches the host, +# so we don't leave a broken cross-arch :latest sitting in the daemon). +if [[ "$PLATFORM" == "$HOST_LINUX_PLATFORM" ]]; then + docker tag "$IMAGE_TAG" "sha-remote-agent:latest" 2>/dev/null || true +fi + +# --- 5. Compress the tarball ------------------------------------------------ IMAGE_TARBALL="${BUNDLE_NAME}.tar.gz" -log "Saving image to ${IMAGE_TARBALL}..." -docker save "$IMAGE_TAG" | gzip > "$STAGING_ROOT/$IMAGE_TARBALL" +log "Compressing image tarball -> ${IMAGE_TARBALL}..." +gzip -c "$UNZIPPED_TAR" > "$STAGING_ROOT/$IMAGE_TARBALL" +rm -f "$UNZIPPED_TAR" TAR_SIZE_MB="$(du -m "$STAGING_ROOT/$IMAGE_TARBALL" | cut -f1)" log "Image tarball size: ${TAR_SIZE_MB} MB" -# --- 3. Copy deploy assets -------------------------------------------------- +# --- 6. Copy deploy assets -------------------------------------------------- log "Copying deploy assets into bundle..." cp "$SCRIPT_DIR/.env.example" "$STAGING_ROOT/.env.example" @@ -148,7 +225,7 @@ sed "s|__VERSION__|${VERSION}|g" \ chmod +x "$STAGING_ROOT/install.sh" -# --- 4. VERSION + SHA256SUMS ----------------------------------------------- +# --- 7. VERSION + SHA256SUMS ----------------------------------------------- printf '%s\n' "$VERSION" > "$STAGING_ROOT/VERSION" @@ -165,7 +242,7 @@ else fi popd >/dev/null -# --- 5. Zip ----------------------------------------------------------------- +# --- 8. Zip ----------------------------------------------------------------- mkdir -p "$DIST_DIR" ZIP_PATH="$DIST_DIR/${BUNDLE_NAME}.zip" @@ -178,7 +255,7 @@ log "Creating ZIP: ${ZIP_PATH}" ZIP_SIZE_MB="$(du -m "$ZIP_PATH" | cut -f1)" -# --- 6. Done --------------------------------------------------------------- +# --- 9. Done --------------------------------------------------------------- log "" log "=========================================================="