From 92ec00d7c50813139b4a1dff32a681a655431248 Mon Sep 17 00:00:00 2001 From: Amir Zarrinkafsh Date: Thu, 16 Sep 2021 22:39:18 +1000 Subject: [PATCH] feat: builds with gox and buildx (#2381) * feat: builds with gox and buildx This change builds all of Authelia respective binaries in parallel within a single step and distributes as necessary to subsequent steps, we now also build and distribute for the following OS/Architecture: freebsd/amd64. Our CI/CD pipeline now also utilises docker buildx as a default for builds and pushes. * refactor: clean up docker helper * Remove `authelia-scripts docker push-image` command as all pushes will be performed with buildx and manifests * Rename the --arch flag to --container * Add Dockerfile.dev for users that want to build an Authelia container from source without utilising suites * Set Dockerfile.dev as default for `authelia-scripts docker build` command * refactor: variant -> container --- .buildkite/annotations/artifacts | 14 +- .buildkite/deployment.sh | 17 +-- .buildkite/hooks/post-command | 4 +- .buildkite/hooks/pre-artifact | 24 ++-- .buildkite/hooks/pre-command | 12 +- .buildkite/pipeline.sh | 26 ++-- .buildkite/steps/buildimages.sh | 43 ------ .buildkite/steps/debhelper.sh | 4 +- .buildkite/steps/debpackages.sh | 12 +- .buildkite/steps/deployimages.sh | 32 ----- .buildkite/steps/ghartifacts.sh | 5 +- .dockerignore | 29 +--- Dockerfile | 49 +++---- Dockerfile.arm64v8 | 48 ------- Dockerfile.coverage.dockerignore | 22 +++ Dockerfile.arm32v7 => Dockerfile.dev | 43 ++++-- Dockerfile.dev.dockerignore | 22 +++ cmd/authelia-scripts/cmd_build.go | 21 ++- cmd/authelia-scripts/cmd_docker.go | 194 +++++++-------------------- cmd/authelia-scripts/const.go | 3 - cmd/authelia-scripts/docker.go | 34 +---- cmd/authelia-scripts/main.go | 2 +- 22 files changed, 212 insertions(+), 448 deletions(-) delete mode 100755 .buildkite/steps/buildimages.sh delete mode 100755 .buildkite/steps/deployimages.sh delete mode 100644 Dockerfile.arm64v8 create mode 100644 Dockerfile.coverage.dockerignore rename Dockerfile.arm32v7 => Dockerfile.dev (54%) create mode 100644 Dockerfile.dev.dockerignore diff --git a/.buildkite/annotations/artifacts b/.buildkite/annotations/artifacts index c4d3e9ba..42ccb229 100644 --- a/.buildkite/annotations/artifacts +++ b/.buildkite/annotations/artifacts @@ -5,24 +5,26 @@
authelia-linux-amd64.tar.gz
authelia-linux-amd64.tar.gz.sha256
+ authelia-freebsd-amd64.tar.gz
+ authelia-freebsd-amd64.tar.gz.sha256
authelia_amd64.deb
authelia_amd64.deb.sha256
-
arm32v7
+
arm
- authelia-linux-arm32v7.tar.gz
- authelia-linux-arm32v7.tar.gz.sha256
+ authelia-linux-arm.tar.gz
+ authelia-linux-arm.tar.gz.sha256
authelia_armhf.deb
authelia_armhf.deb.sha256
-
arm64v8
+
arm64
- authelia-linux-arm64v8.tar.gz
- authelia-linux-arm64v8.tar.gz.sha256
+ authelia-linux-arm64.tar.gz
+ authelia-linux-arm64.tar.gz.sha256
authelia_arm64.deb
authelia_arm64.deb.sha256
diff --git a/.buildkite/deployment.sh b/.buildkite/deployment.sh index 860bb6bf..43c472bd 100755 --- a/.buildkite/deployment.sh +++ b/.buildkite/deployment.sh @@ -22,20 +22,13 @@ env: CI_BYPASS: ${CI_BYPASS} steps: - - label: ":docker: Image Deployments" - command: ".buildkite/steps/deployimages.sh | buildkite-agent pipeline upload" - if: build.env("CI_BYPASS") != "true" - - - wait: - if: build.env("CI_BYPASS") != "true" - - - label: ":docker: Deploy Manifests" + - label: ":docker: Deploy Manifest" command: "authelia-scripts docker push-manifest" + depends_on: + - "unit-test" retry: manual: permit_on_passed: true - env: - DOCKER_CLI_EXPERIMENTAL: "enabled" agents: upload: "fast" if: build.env("CI_BYPASS") != "true" @@ -43,9 +36,7 @@ steps: - label: ":github: Deploy Artifacts" command: "ghartifacts.sh" depends_on: - - "build-docker-linux-amd64" - - "build-docker-linux-arm32v7" - - "build-docker-linux-arm64v8" + - "unit-test" - "build-deb-package-amd64" - "build-deb-package-armhf" - "build-deb-package-arm64" diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command index 549aadb2..c3905de3 100755 --- a/.buildkite/hooks/post-command +++ b/.buildkite/hooks/post-command @@ -31,7 +31,7 @@ if [[ "${BUILDKITE_LABEL}" =~ ":selenium:" ]] || [[ "${BUILDKITE_LABEL}" =~ ":do fi fi -if [[ "${BUILDKITE_LABEL}" == ":docker: Image Deployments" ]]; then +if [[ "${BUILDKITE_LABEL}" == ":debian: Package Builds" ]]; then cat .buildkite/annotations/artifacts | buildkite-agent annotate --style "success" --context "ctx-success" fi @@ -40,7 +40,7 @@ if [[ "${BUILDKITE_LABEL}" =~ ":docker: Deploy" ]]; then docker logout ghcr.io fi -if [[ "${BUILDKITE_LABEL}" == ":docker: Deploy Manifests" ]] && [[ "${BUILDKITE_BRANCH}" == "master" ]] && [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then +if [[ "${BUILDKITE_LABEL}" == ":docker: Deploy Manifest" ]] && [[ "${BUILDKITE_BRANCH}" == "master" ]] && [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then echo "--- :docker: Removing tags for deleted branches" anontoken=$(curl -fsL --retry 3 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:authelia/authelia:pull' | jq -r .token) authtoken=$(curl -fs --retry 3 -H "Content-Type: application/json" -X "POST" -d '{"username": "'${DOCKER_USERNAME}'", "password": "'${DOCKER_PASSWORD}'"}' https://hub.docker.com/v2/users/login/ | jq -r .token) diff --git a/.buildkite/hooks/pre-artifact b/.buildkite/hooks/pre-artifact index 86c916e6..86ef11ec 100755 --- a/.buildkite/hooks/pre-artifact +++ b/.buildkite/hooks/pre-artifact @@ -2,25 +2,25 @@ set +u +declare -A BUILDS=(["linux"]="amd64 arm arm64" ["freebsd"]="amd64") DOCKER_IMAGE=authelia/authelia if [[ "${BUILDKITE_LABEL}" == ":hammer_and_wrench: Unit Test" ]]; then + echo "--- :docker: Saving artifacts for :buildkite: :docker: :github: releases" + for OS in "${!BUILDS[@]}"; do + for ARCH in ${BUILDS[$OS]}; do + tar -czf "authelia-${OS}-${ARCH}.tar.gz" "authelia-${OS}-${ARCH}" authelia.service config.template.yml + sha256sum "authelia-${OS}-${ARCH}.tar.gz" > "authelia-${OS}-${ARCH}.tar.gz.sha256" + done + done + tar -czf authelia-public_html.tar.gz -C dist public_html sha256sum authelia-public_html.tar.gz > authelia-public_html.tar.gz.sha256 fi -if [[ "${BUILDKITE_LABEL}" =~ ":docker: Build Image" ]]; then - echo "--- :docker: Saving artifacts for :buildkite: :docker: :github: releases" - # Save binary for buildkite and github artifacts - if [[ "${ARCH}" != "coverage" ]]; then - docker create --name authelia-binary "${DOCKER_IMAGE}:latest" - docker cp authelia-binary:/app/authelia "./authelia-${OS}-${ARCH}" - docker rm -f authelia-binary - tar -czf "authelia-${OS}-${ARCH}.tar.gz" "authelia-${OS}-${ARCH}" authelia.service config.template.yml - sha256sum "authelia-${OS}-${ARCH}.tar.gz" > "authelia-${OS}-${ARCH}.tar.gz.sha256" - fi - # Saving image for push to docker hub - docker save "${DOCKER_IMAGE}" | zstdmt -T0 -12 > "authelia-image-${ARCH}.tar.zst" +if [[ "${BUILDKITE_LABEL}" == ":docker: Build Image [coverage]" ]]; then + # Saving image for docker push + docker save "${DOCKER_IMAGE}" | zstdmt -T0 -12 > "authelia-image-coverage.tar.zst" fi if [[ "${BUILDKITE_LABEL}" =~ ":debian: Build Package" ]]; then diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 721dde6e..291630d1 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -14,12 +14,6 @@ if [[ "${BUILDKITE_LABEL}" =~ ":selenium:" ]]; then docker tag authelia/authelia authelia:dist fi -if [[ "${BUILDKITE_LABEL}" =~ ":docker: Build Image" ]] && [[ "${ARCH}" != "coverage" ]]; then - echo "--- :react: :swagger: Extract frontend assets" - buildkite-agent artifact download "authelia-public_html.tar.gz" . - tar xzf authelia-public_html.tar.gz -fi - if [[ "${BUILDKITE_LABEL}" =~ ":debian: Build Package" ]]; then buildkite-agent artifact download "authelia-linux-${ARCH}.tar.gz" . fi @@ -29,6 +23,12 @@ if [[ "${BUILDKITE_LABEL}" =~ ":docker: Deploy Image" ]]; then zstdcat "authelia-image-${ARCH}.tar.zst" | docker load fi +if [[ "${BUILDKITE_LABEL}" == ":docker: Deploy Manifest" ]]; then + echo "--- :go: :react: :swagger: Extract pre-built binary" + buildkite-agent artifact download "authelia-linux-*.tar.gz" . + for archive in authelia-linux-*.tar.gz; do tar xzf ${archive} --wildcards "authelia-linux-*"; done +fi + if [[ "${BUILDKITE_LABEL}" == ":github: Deploy Artifacts" ]]; then buildkite-agent artifact download "authelia-*.tar.gz*" . buildkite-agent artifact download "authelia_*.deb*" . diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh index 6aada3d6..c9654905 100755 --- a/.buildkite/pipeline.sh +++ b/.buildkite/pipeline.sh @@ -37,35 +37,29 @@ steps: agents: build: "unit-test" artifact_paths: - - "authelia-public_html.tar.gz" - - "authelia-public_html.tar.gz.sha256" + - "authelia-*.tar.gz" + - "authelia-*.tar.gz.sha256" key: "unit-test" if: build.env("CI_BYPASS") != "true" - wait: if: build.env("CI_BYPASS") != "true" - - label: ":docker: Image Builds" - command: ".buildkite/steps/buildimages.sh | buildkite-agent pipeline upload" - concurrency: 3 - concurrency_group: "builds" + - label: ":docker: Build Image [coverage]" + command: "authelia-scripts docker build --container=coverage" + agents: + build: "linux-coverage" + artifact_paths: + - "authelia-image-coverage.tar.zst" depends_on: ~ - if: build.env("CI_BYPASS") != "true" + key: "build-docker-linux-coverage" + if: build.env("CI_BYPASS") != "true" && build.branch !~ /^(v[0-9]+\.[0-9]+\.[0-9]+)$\$/ && build.message !~ /\[(skip test|test skip)\]/ - label: ":debian: Package Builds" command: ".buildkite/steps/debpackages.sh | buildkite-agent pipeline upload" depends_on: ~ if: build.branch !~ /^(dependabot|renovate)\/.*/ && build.env("CI_BYPASS") != "true" - - wait: - if: build.env("CI_BYPASS") != "true" - - - label: ":vertical_traffic_light: Build Concurrency Gate" - command: "echo End of concurrency gate" - concurrency: 3 - concurrency_group: "builds" - if: build.env("CI_BYPASS") != "true" - - wait: if: build.branch !~ /^(v[0-9]+\.[0-9]+\.[0-9]+)$\$/ && build.env("CI_BYPASS") != "true" && build.message !~ /\[(skip test|test skip)\]/ diff --git a/.buildkite/steps/buildimages.sh b/.buildkite/steps/buildimages.sh deleted file mode 100755 index 8c543d22..00000000 --- a/.buildkite/steps/buildimages.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -eu - -declare -A BUILDS=(["linux"]="amd64 arm32v7 arm64v8 coverage") - -for BUILD_OS in "${!BUILDS[@]}"; do - for BUILD_ARCH in ${BUILDS[$BUILD_OS]}; do -cat << EOF - - label: ":docker: Build Image [${BUILD_ARCH}]" - command: "authelia-scripts docker build --arch=${BUILD_ARCH}" - retry: - manual: - permit_on_passed: true - agents: - build: "${BUILD_OS}-${BUILD_ARCH}" - artifact_paths: - - "authelia-image-${BUILD_ARCH}.tar.zst" -EOF -if [[ "${BUILD_ARCH}" != "coverage" ]]; then -cat << EOF - - "authelia-${BUILD_OS}-${BUILD_ARCH}.tar.gz" - - "authelia-${BUILD_OS}-${BUILD_ARCH}.tar.gz.sha256" - depends_on: - - "unit-test" -EOF -fi -cat << EOF - env: - ARCH: "${BUILD_ARCH}" - OS: "${BUILD_OS}" - key: "build-docker-${BUILD_OS}-${BUILD_ARCH}" -EOF -if [[ "${BUILD_ARCH}" == "coverage" ]]; then -cat << EOF - if: build.branch !~ /^(v[0-9]+\.[0-9]+\.[0-9]+)$\$/ && build.message !~ /\[(skip test|test skip)\]/ -EOF -else -cat << EOF - if: build.branch !~ /^(dependabot|renovate)\/.*/ -EOF -fi - done -done \ No newline at end of file diff --git a/.buildkite/steps/debhelper.sh b/.buildkite/steps/debhelper.sh index a9201b5f..1a28b193 100755 --- a/.buildkite/steps/debhelper.sh +++ b/.buildkite/steps/debhelper.sh @@ -11,8 +11,8 @@ fi wget https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h=authelia-bin -qO PKGBUILD && \ sed -i -e '/^pkgname=/c pkgname=authelia' -e "/pkgver=/c $VERSION" -e '10,14d' \ -e 's/source_x86_64.*/source_x86_64=("authelia-linux-amd64.tar.gz")/' \ --e 's/source_aarch64.*/source_aarch64=("authelia-linux-arm64v8.tar.gz")/' \ --e 's/source_armv7h.*/source_armv7h=("authelia-linux-arm32v7.tar.gz")/' \ +-e 's/source_aarch64.*/source_aarch64=("authelia-linux-arm64.tar.gz")/' \ +-e 's/source_armv7h.*/source_armv7h=("authelia-linux-arm.tar.gz")/' \ -e 's/sha256sums_x86_64.*/sha256sums_x86_64=("SKIP")/' \ -e 's/sha256sums_aarch64.*/sha256sums_aarch64=("SKIP")/' \ -e 's/sha256sums_armv7h.*/sha256sums_armv7h=("SKIP")/' PKGBUILD diff --git a/.buildkite/steps/debpackages.sh b/.buildkite/steps/debpackages.sh index a805f1a8..a4656ffb 100755 --- a/.buildkite/steps/debpackages.sh +++ b/.buildkite/steps/debpackages.sh @@ -14,23 +14,19 @@ EOF if [[ "${DEB_PACKAGE}" == "amd64" ]]; then cat << EOF ARCH: "${DEB_PACKAGE}" - depends_on: - - "build-docker-linux-amd64" EOF elif [[ "${DEB_PACKAGE}" == "armhf" ]]; then cat << EOF - ARCH: "arm32v7" - depends_on: - - "build-docker-linux-arm32v7" + ARCH: "arm" EOF else cat << EOF - ARCH: "arm64v8" - depends_on: - - "build-docker-linux-arm64v8" + ARCH: "arm64" EOF fi cat << EOF + depends_on: + - "unit-test" key: "build-deb-package-${DEB_PACKAGE}" EOF done \ No newline at end of file diff --git a/.buildkite/steps/deployimages.sh b/.buildkite/steps/deployimages.sh deleted file mode 100755 index 757aa9fd..00000000 --- a/.buildkite/steps/deployimages.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -eu - -for BUILD_ARCH in amd64 arm32v7 arm64v8; do -cat << EOF - - label: ":docker: Deploy Image [${BUILD_ARCH}]" - command: "authelia-scripts docker push-image --arch=${BUILD_ARCH}" - retry: - manual: - permit_on_passed: true - depends_on: -EOF -if [[ "${BUILD_ARCH}" == "amd64" ]]; then -cat << EOF - - "build-docker-linux-amd64" -EOF -elif [[ "${BUILD_ARCH}" == "arm32v7" ]]; then -cat << EOF - - "build-docker-linux-arm32v7" -EOF -else -cat << EOF - - "build-docker-linux-arm64v8" -EOF -fi -cat << EOF - agents: - upload: "fast" - env: - ARCH: "${BUILD_ARCH}" -EOF -done \ No newline at end of file diff --git a/.buildkite/steps/ghartifacts.sh b/.buildkite/steps/ghartifacts.sh index 196741c1..b172b901 100755 --- a/.buildkite/steps/ghartifacts.sh +++ b/.buildkite/steps/ghartifacts.sh @@ -5,8 +5,9 @@ artifacts=() for FILE in \ authelia-linux-amd64.tar.gz authelia-linux-amd64.tar.gz.sha256 \ - authelia-linux-arm32v7.tar.gz authelia-linux-arm32v7.tar.gz.sha256 \ - authelia-linux-arm64v8.tar.gz authelia-linux-arm64v8.tar.gz.sha256 \ + authelia-linux-arm.tar.gz authelia-linux-arm.tar.gz.sha256 \ + authelia-linux-arm64.tar.gz authelia-linux-arm64.tar.gz.sha256 \ + authelia-freebsd-amd64.tar.gz authelia-freebsd-amd64.tar.gz.sha256 \ authelia-public_html.tar.gz authelia-public_html.tar.gz.sha256; do # Add the version to the artifact name diff --git a/.dockerignore b/.dockerignore index c14a03ac..a3946b78 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,26 +1,9 @@ -# CI/Git Directories -.git -.cache -.buildkite -.github -coverage.txt -Dockerfile* - -# Node Modules Directories -**/node_modules - -# Documentation -docs -examples - -# Dot Files and Markdown -.* -*.md - -# Other -internal/server/public_html -authelia.service -bootstrap.sh +# Ignore All +* # Overrides +!authelia-linux-* +!LICENSE +!entrypoint.sh +!healthcheck.sh !.healthcheck.env diff --git a/Dockerfile b/Dockerfile index 18289e04..fe1ba6c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,47 +1,32 @@ -# ======================================= -# ===== Build image for the backend ===== -# ======================================= -FROM golang:1.17.1-alpine AS builder-backend - -WORKDIR /go/src/app - -COPY go.mod go.sum ./ - -RUN \ -echo ">> Downloading go modules..." && \ -go mod download - -COPY / ./ - -ARG LDFLAGS_EXTRA - -RUN \ -mv public_html internal/server/public_html && \ -chmod 0666 /go/src/app/.healthcheck.env && \ -echo ">> Starting go build..." && \ -GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -tags netgo \ --ldflags "-s -w ${LDFLAGS_EXTRA}" -trimpath -o authelia ./cmd/authelia - # =================================== # ===== Authelia official image ===== # =================================== FROM alpine:3.14.2 +ARG TARGETOS +ARG TARGETARCH + WORKDIR /app -RUN apk --no-cache add ca-certificates su-exec tzdata - -COPY --from=builder-backend /go/src/app/authelia /go/src/app/LICENSE /go/src/app/entrypoint.sh /go/src/app/healthcheck.sh /go/src/app/.healthcheck.env ./ - -EXPOSE 9091 - -VOLUME /config - # Set environment variables ENV PATH="/app:${PATH}" \ PUID=0 \ PGID=0 +RUN \ +apk --no-cache add ca-certificates su-exec tzdata + +COPY LICENSE .healthcheck.env entrypoint.sh healthcheck.sh ./ + +RUN \ +chmod 0666 /app/.healthcheck.env + +COPY authelia-${TARGETOS}-${TARGETARCH} ./authelia + +EXPOSE 9091 + +VOLUME /config + ENTRYPOINT ["/app/entrypoint.sh"] CMD ["--config", "/config/configuration.yml"] HEALTHCHECK --interval=30s --timeout=3s --start-period=1m CMD /app/healthcheck.sh diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 deleted file mode 100644 index fcbfa0d0..00000000 --- a/Dockerfile.arm64v8 +++ /dev/null @@ -1,48 +0,0 @@ -# ======================================= -# ===== Build image for the backend ===== -# ======================================= -FROM golang:1.17.1-alpine AS builder-backend - -WORKDIR /go/src/app - -COPY go.mod go.sum ./ - -RUN \ -echo ">> Downloading go modules..." && \ -go mod download - -COPY / ./ - -ARG LDFLAGS_EXTRA - -RUN \ -mv public_html internal/server/public_html && \ -chmod 0666 /go/src/app/.healthcheck.env && \ -echo ">> Starting go build..." && \ -GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -tags netgo \ --ldflags "-s -w ${LDFLAGS_EXTRA}" -trimpath -o authelia ./cmd/authelia - -# =================================== -# ===== Authelia official image ===== -# =================================== -FROM arm64v8/alpine:3.14.2 - -WORKDIR /app - -RUN \ -apk --no-cache add ca-certificates su-exec tzdata - -COPY --from=builder-backend /go/src/app/authelia /go/src/app/LICENSE /go/src/app/entrypoint.sh /go/src/app/healthcheck.sh /go/src/app/.healthcheck.env ./ - -EXPOSE 9091 - -VOLUME /config - -# Set environment variables -ENV PATH="/app:${PATH}" \ - PUID=0 \ - PGID=0 - -ENTRYPOINT ["/app/entrypoint.sh"] -CMD ["--config", "/config/configuration.yml"] -HEALTHCHECK --interval=30s --timeout=3s --start-period=1m CMD /app/healthcheck.sh diff --git a/Dockerfile.coverage.dockerignore b/Dockerfile.coverage.dockerignore new file mode 100644 index 00000000..27e482d6 --- /dev/null +++ b/Dockerfile.coverage.dockerignore @@ -0,0 +1,22 @@ +# CI/Git Directories +coverage.txt +Dockerfile* + +# Node Modules Directories +**/node_modules + +# Documentation +docs +examples + +# Dot Files and Markdown +.* +*.md + +# Other +internal/server/public_html +authelia.service +bootstrap.sh + +# Overrides +!.healthcheck.env \ No newline at end of file diff --git a/Dockerfile.arm32v7 b/Dockerfile.dev similarity index 54% rename from Dockerfile.arm32v7 rename to Dockerfile.dev index ba8315e6..62abb43c 100644 --- a/Dockerfile.arm32v7 +++ b/Dockerfile.dev @@ -1,3 +1,15 @@ +# ======================================== +# ===== Build image for the frontend ===== +# ======================================== +FROM node:16-alpine AS builder-frontend + +WORKDIR /node/src/app + +COPY web ./ + +# Install the dependencies and build +RUN yarn install --frozen-lockfile && INLINE_RUNTIME_CHUNK=false yarn coverage + # ======================================= # ===== Build image for the backend ===== # ======================================= @@ -13,36 +25,41 @@ go mod download COPY / ./ -ARG LDFLAGS_EXTRA +# Prepare static files to be embedded in Go binary +COPY --from=builder-frontend /node/src/internal/server/public_html internal/server/public_html +ARG LDFLAGS_EXTRA RUN \ -mv public_html internal/server/public_html && \ +mv api internal/server/public_html/api && \ chmod 0666 /go/src/app/.healthcheck.env && \ echo ">> Starting go build..." && \ -GOOS=linux GOARCH=arm CGO_ENABLED=0 go build -tags netgo \ +CGO_ENABLED=0 go build -tags netgo \ -ldflags "-s -w ${LDFLAGS_EXTRA}" -trimpath -o authelia ./cmd/authelia # =================================== # ===== Authelia official image ===== # =================================== -FROM arm32v7/alpine:3.14.2 +FROM alpine:3.14.2 WORKDIR /app -RUN \ -apk --no-cache add ca-certificates su-exec tzdata - -COPY --from=builder-backend /go/src/app/authelia /go/src/app/LICENSE /go/src/app/entrypoint.sh /go/src/app/healthcheck.sh /go/src/app/.healthcheck.env ./ - -EXPOSE 9091 - -VOLUME /config - # Set environment variables ENV PATH="/app:${PATH}" \ PUID=0 \ PGID=0 +RUN \ +apk --no-cache add ca-certificates su-exec tzdata + +COPY --from=builder-backend /go/src/app/cmd/authelia/authelia /go/src/app/LICENSE /go/src/app/entrypoint.sh /go/src/app/healthcheck.sh /go/src/app/.healthcheck.env ./ + +RUN \ +chmod 0666 /app/.healthcheck.env + +EXPOSE 9091 + +VOLUME /config + ENTRYPOINT ["/app/entrypoint.sh"] CMD ["--config", "/config/configuration.yml"] HEALTHCHECK --interval=30s --timeout=3s --start-period=1m CMD /app/healthcheck.sh diff --git a/Dockerfile.dev.dockerignore b/Dockerfile.dev.dockerignore new file mode 100644 index 00000000..27e482d6 --- /dev/null +++ b/Dockerfile.dev.dockerignore @@ -0,0 +1,22 @@ +# CI/Git Directories +coverage.txt +Dockerfile* + +# Node Modules Directories +**/node_modules + +# Documentation +docs +examples + +# Dot Files and Markdown +.* +*.md + +# Other +internal/server/public_html +authelia.service +bootstrap.sh + +# Overrides +!.healthcheck.env \ No newline at end of file diff --git a/cmd/authelia-scripts/cmd_build.go b/cmd/authelia-scripts/cmd_build.go index 615c0bad..c7bda17c 100644 --- a/cmd/authelia-scripts/cmd_build.go +++ b/cmd/authelia-scripts/cmd_build.go @@ -10,12 +10,18 @@ import ( "github.com/authelia/authelia/v4/internal/utils" ) -func buildAutheliaBinary(xflags []string) { - cmd := utils.CommandWithStdout("go", "build", "-o", "../../"+OutputDir+"/authelia", "-ldflags", strings.Join(xflags, " ")) - cmd.Dir = "cmd/authelia" +func buildAutheliaBinary(xflags []string, buildkite bool) { + cmd := utils.CommandWithStdout("go", "build", "-tags", "netgo", "-trimpath", "-o", OutputDir+"/authelia", "-ldflags", "-s -w "+strings.Join(xflags, " "), "./cmd/authelia/") cmd.Env = append(os.Environ(), - "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0") + "CGO_ENABLED=0") + + if buildkite { + cmd = utils.CommandWithStdout("gox", "-tags=netgo", "-output={{.Dir}}-{{.OS}}-{{.Arch}}", "-ldflags=-s -w "+strings.Join(xflags, " "), "-osarch=linux/amd64 linux/arm linux/arm64 freebsd/amd64", "./cmd/authelia/") + + cmd.Env = append(os.Environ(), + "GOFLAGS=-trimpath", "CGO_ENABLED=0") + } err := cmd.Run() if err != nil { @@ -98,7 +104,7 @@ func Build(cobraCmd *cobra.Command, args []string) { Clean(cobraCmd, args) - xflags, err := getXFlags("", "0", "") + xflags, err := getXFlags(os.Getenv("BUILDKITE_BRANCH"), os.Getenv("BUILDKITE_BUILD_NUMBER"), "") if err != nil { log.Fatal(err) } @@ -118,11 +124,12 @@ func Build(cobraCmd *cobra.Command, args []string) { buildkite, _ := cobraCmd.Flags().GetBool("buildkite") if buildkite { - log.Debug("Buildkite job detected, skipping Authelia Go binary build") + log.Debug("Building Authelia Go binaries with gox...") } else { log.Debug("Building Authelia Go binary...") - buildAutheliaBinary(xflags) } + buildAutheliaBinary(xflags, buildkite) + cleanAssets() } diff --git a/cmd/authelia-scripts/cmd_docker.go b/cmd/authelia-scripts/cmd_docker.go index ddb50f2c..fb90414e 100644 --- a/cmd/authelia-scripts/cmd_docker.go +++ b/cmd/authelia-scripts/cmd_docker.go @@ -6,19 +6,15 @@ import ( "os" "regexp" "strings" - "time" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - - "github.com/authelia/authelia/v4/internal/utils" ) -var arch string +var container string -var supportedArch = []string{"amd64", "arm32v7", "arm64v8", "coverage"} -var defaultArch = "amd64" -var buildkiteQEMU = os.Getenv("BUILDKITE_AGENT_META_DATA_QEMU") +var containers = []string{"dev", "coverage"} +var defaultContainer = "dev" var ciBranch = os.Getenv("BUILDKITE_BRANCH") var ciPullRequest = os.Getenv("BUILDKITE_PULL_REQUEST") var ciTag = os.Getenv("BUILDKITE_TAG") @@ -28,38 +24,23 @@ var publicRepo = regexp.MustCompile(`.*:.*`) var tags = dockerTags.FindStringSubmatch(ciTag) func init() { - DockerBuildCmd.PersistentFlags().StringVar(&arch, "arch", defaultArch, "target architecture among: "+strings.Join(supportedArch, ", ")) - DockerPushCmd.PersistentFlags().StringVar(&arch, "arch", defaultArch, "target architecture among: "+strings.Join(supportedArch, ", ")) + DockerBuildCmd.PersistentFlags().StringVar(&container, "container", defaultContainer, "target container among: "+strings.Join(containers, ", ")) } -func checkArchIsSupported(arch string) { - for _, a := range supportedArch { - if arch == a { +func checkContainerIsSupported(container string) { + for _, v := range containers { + if container == v { return } } - log.Fatal("Architecture is not supported. Please select one of " + strings.Join(supportedArch, ", ") + ".") + log.Fatal("Container is not supported. Please select one of " + strings.Join(containers, ", ") + ".") } func dockerBuildOfficialImage(arch string) error { docker := &Docker{} - // Set default Architecture Dockerfile to amd64. - dockerfile := "Dockerfile" - - // If not the default value. - if arch != defaultArch { - dockerfile = fmt.Sprintf("%s.%s", dockerfile, arch) - } - - if arch == "arm32v7" || arch == "arm64v8" { - if buildkiteQEMU != stringTrue { - err := utils.CommandWithStdout("docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run() - if err != nil { - log.Fatal(err) - } - } - } + filename := "Dockerfile" + dockerfile := fmt.Sprintf("%s.%s", filename, arch) flags, err := getXFlags(ciBranch, os.Getenv("BUILDKITE_BUILD_NUMBER"), "") if err != nil { @@ -76,8 +57,8 @@ var DockerBuildCmd = &cobra.Command{ Short: "Build the docker image of Authelia", Run: func(cmd *cobra.Command, args []string) { log.Infof("Building Docker image %s...", DockerImageName) - checkArchIsSupported(arch) - err := dockerBuildOfficialImage(arch) + checkContainerIsSupported(container) + err := dockerBuildOfficialImage(container) if err != nil { log.Fatal(err) @@ -92,23 +73,11 @@ var DockerBuildCmd = &cobra.Command{ }, } -// DockerPushCmd Command for pushing Authelia docker image to DockerHub. -var DockerPushCmd = &cobra.Command{ - Use: "push-image", - Short: "Publish Authelia docker image to Docker Hub", - Run: func(cmd *cobra.Command, args []string) { - log.Infof("Pushing Docker image %s to Docker Hub...", DockerImageName) - checkArchIsSupported(arch) - publishDockerImage(arch) - }, -} - // DockerManifestCmd Command for pushing Authelia docker manifest to DockerHub. var DockerManifestCmd = &cobra.Command{ Use: "push-manifest", Short: "Publish Authelia docker manifest to Docker Hub", Run: func(cmd *cobra.Command, args []string) { - log.Infof("Pushing Docker manifest of %s to Docker Hub...", DockerImageName) publishDockerManifest() }, } @@ -142,124 +111,53 @@ func login(docker *Docker, registry string) { } } -func deploy(docker *Docker, tag, registry string) { - imageWithTag := registry + "/" + DockerImageName + ":" + tag +func deployManifest(docker *Docker, tag string) { + log.Infof("Docker manifest %s:%s will be deployed on %s and %s", DockerImageName, tag, dockerhub, ghcr) - log.Infof("Docker image %s will be deployed on %s", imageWithTag, registry) + dockerhub := dockerhub + "/" + DockerImageName + ":" + tag + ghcr := ghcr + "/" + DockerImageName + ":" + tag - if err := docker.Tag(DockerImageName, imageWithTag); err != nil { + if err := docker.Manifest(dockerhub, ghcr); err != nil { log.Fatal(err) } - - if err := docker.Push(imageWithTag); err != nil { - log.Fatal(err) - } -} - -func deployManifest(docker *Docker, tag, amd64tag, arm32v7tag, arm64v8tag, registry string) { - dockerImagePrefix := registry + "/" + DockerImageName + ":" - - log.Infof("Docker manifest %s%s will be deployed on %s", dockerImagePrefix, tag, registry) - - err := docker.Manifest(dockerImagePrefix+tag, dockerImagePrefix+amd64tag, dockerImagePrefix+arm32v7tag, dockerImagePrefix+arm64v8tag) - - if err != nil { - log.Fatal(err) - } - - tags := []string{amd64tag, arm32v7tag, arm64v8tag} - - if registry == dockerhub { - for _, t := range tags { - log.Infof("Docker removing tag for %s%s on Docker Hub", dockerImagePrefix, t) - - if err := utils.RunFuncWithRetry(3, 10*time.Second, func() (err error) { - err = docker.CleanTag(t) - return - }); err != nil { - log.Fatal(err) - } - } - } -} - -func publishDockerImage(arch string) { - docker := &Docker{} - - for _, registry := range registries { - switch { - case ciTag != "": - if len(tags) == 4 { - log.Infof("Detected tags: '%s' | '%s' | '%s'", tags[1], tags[2], tags[3]) - login(docker, registry) - deploy(docker, tags[1]+"-"+arch, registry) - - if !ignoredSuffixes.MatchString(ciTag) { - deploy(docker, tags[2]+"-"+arch, registry) - deploy(docker, tags[3]+"-"+arch, registry) - deploy(docker, "latest-"+arch, registry) - } - } else { - log.Fatal("Docker image will not be published, the specified tag does not conform to the standard") - } - case ciBranch != masterTag && !publicRepo.MatchString(ciBranch): - login(docker, registry) - deploy(docker, ciBranch+"-"+arch, registry) - case ciBranch != masterTag && publicRepo.MatchString(ciBranch): - login(docker, registry) - deploy(docker, "PR"+ciPullRequest+"-"+arch, registry) - case ciBranch == masterTag && ciPullRequest == stringFalse: - login(docker, registry) - deploy(docker, "master-"+arch, registry) - default: - log.Info("Docker image will not be published") - } - } } func publishDockerManifest() { docker := &Docker{} - for _, registry := range registries { - switch { - case ciTag != "": - if len(tags) == 4 { - log.Infof("Detected tags: '%s' | '%s' | '%s'", tags[1], tags[2], tags[3]) - login(docker, registry) - deployManifest(docker, tags[1], tags[1]+"-amd64", tags[1]+"-arm32v7", tags[1]+"-arm64v8", registry) + switch { + case ciTag != "": + if len(tags) == 4 { + log.Infof("Detected tags: '%s' | '%s' | '%s'", tags[1], tags[2], tags[3]) + login(docker, dockerhub) + login(docker, ghcr) + deployManifest(docker, tags[1]) + publishDockerReadme(docker) - if registry == dockerhub { - publishDockerReadme(docker) - } - - if !ignoredSuffixes.MatchString(ciTag) { - deployManifest(docker, tags[2], tags[2]+"-amd64", tags[2]+"-arm32v7", tags[2]+"-arm64v8", registry) - deployManifest(docker, tags[3], tags[3]+"-amd64", tags[3]+"-arm32v7", tags[3]+"-arm64v8", registry) - deployManifest(docker, "latest", "latest-amd64", "latest-arm32v7", "latest-arm64v8", registry) - - if registry == dockerhub { - publishDockerReadme(docker) - } - } - } else { - log.Fatal("Docker manifest will not be published, the specified tag does not conform to the standard") - } - case ciBranch != masterTag && !publicRepo.MatchString(ciBranch): - login(docker, registry) - deployManifest(docker, ciBranch, ciBranch+"-amd64", ciBranch+"-arm32v7", ciBranch+"-arm64v8", registry) - case ciBranch != masterTag && publicRepo.MatchString(ciBranch): - login(docker, registry) - deployManifest(docker, "PR"+ciPullRequest, "PR"+ciPullRequest+"-amd64", "PR"+ciPullRequest+"-arm32v7", "PR"+ciPullRequest+"-arm64v8", registry) - case ciBranch == masterTag && ciPullRequest == stringFalse: - login(docker, registry) - deployManifest(docker, "master", "master-amd64", "master-arm32v7", "master-arm64v8", registry) - - if registry == dockerhub { + if !ignoredSuffixes.MatchString(ciTag) { + deployManifest(docker, tags[2]) + deployManifest(docker, tags[3]) + deployManifest(docker, "latest") publishDockerReadme(docker) } - default: - log.Info("Docker manifest will not be published") + } else { + log.Fatal("Docker manifest will not be published, the specified tag does not conform to the standard") } + case ciBranch != masterTag && !publicRepo.MatchString(ciBranch): + login(docker, dockerhub) + login(docker, ghcr) + deployManifest(docker, ciBranch) + case ciBranch != masterTag && publicRepo.MatchString(ciBranch): + login(docker, dockerhub) + login(docker, ghcr) + deployManifest(docker, "PR"+ciPullRequest) + case ciBranch == masterTag && ciPullRequest == stringFalse: + login(docker, dockerhub) + login(docker, ghcr) + deployManifest(docker, "master") + publishDockerReadme(docker) + default: + log.Info("Docker manifest will not be published") } } diff --git a/cmd/authelia-scripts/const.go b/cmd/authelia-scripts/const.go index bfe712d7..0a884f89 100644 --- a/cmd/authelia-scripts/const.go +++ b/cmd/authelia-scripts/const.go @@ -9,14 +9,11 @@ var DockerImageName = "authelia/authelia" // IntermediateDockerImageName local name of the docker image. var IntermediateDockerImageName = "authelia:dist" -var registries = []string{"docker.io", "ghcr.io"} - const dockerhub = "docker.io" const ghcr = "ghcr.io" const masterTag = "master" const stringFalse = "false" -const stringTrue = "true" const webDirectory = "web" const fmtLDFLAGSX = "-X 'github.com/authelia/authelia/v4/internal/utils.%s=%s'" diff --git a/cmd/authelia-scripts/docker.go b/cmd/authelia-scripts/docker.go index 03418fdb..bad42b4f 100644 --- a/cmd/authelia-scripts/docker.go +++ b/cmd/authelia-scripts/docker.go @@ -22,40 +22,12 @@ func (d *Docker) Tag(image, tag string) error { // Login login to the dockerhub registry. func (d *Docker) Login(username, password, registry string) error { - return utils.CommandWithStdout("docker", "login", registry, "-u", username, "-p", password).Run() -} - -// Push push a docker image to dockerhub. -func (d *Docker) Push(tag string) error { - return utils.CommandWithStdout("docker", "push", tag).Run() + return utils.CommandWithStdout("bash", "-c", `echo `+password+` | docker login `+registry+` --password-stdin -u `+username).Run() } // Manifest push a docker manifest to dockerhub. -func (d *Docker) Manifest(tag, amd64tag, arm32v7tag, arm64v8tag string) error { - err := utils.CommandWithStdout("docker", "manifest", "create", tag, amd64tag, arm32v7tag, arm64v8tag).Run() - - if err != nil { - panic(err) - } - - err = utils.CommandWithStdout("docker", "manifest", "annotate", tag, arm32v7tag, "--os", "linux", "--arch", "arm").Run() - - if err != nil { - panic(err) - } - - err = utils.CommandWithStdout("docker", "manifest", "annotate", tag, arm64v8tag, "--os", "linux", "--arch", "arm64", "--variant", "v8").Run() - - if err != nil { - panic(err) - } - - return utils.CommandWithStdout("docker", "manifest", "push", "--purge", tag).Run() -} - -// CleanTag remove a tag from dockerhub. -func (d *Docker) CleanTag(tag string) error { - return utils.CommandWithStdout("bash", "-c", `token=$(curl -fs --retry 3 -H "Content-Type: application/json" -X "POST" -d '{"username": "'$DOCKER_USERNAME'", "password": "'$DOCKER_PASSWORD'"}' https://hub.docker.com/v2/users/login/ | jq -r .token) && curl -fs --retry 3 -o /dev/null -L -X "DELETE" -H "Authorization: JWT $token" https://hub.docker.com/v2/repositories/`+DockerImageName+"/tags/"+tag+"/").Run() +func (d *Docker) Manifest(tag1, tag2 string) error { + return utils.CommandWithStdout("docker", "build", "-t", tag1, "-t", tag2, "--platform", "linux/amd64,linux/arm/v7,linux/arm64", "--builder", "buildx", "--push", ".").Run() } // PublishReadme push README.md to dockerhub. diff --git a/cmd/authelia-scripts/main.go b/cmd/authelia-scripts/main.go index 7a6cb570..cd812d29 100755 --- a/cmd/authelia-scripts/main.go +++ b/cmd/authelia-scripts/main.go @@ -50,7 +50,7 @@ var Commands = []AutheliaCommandDefinition{ { Name: "docker", Short: "Commands related to building and publishing docker image", - SubCommands: CobraCommands{DockerBuildCmd, DockerPushCmd, DockerManifestCmd}, + SubCommands: CobraCommands{DockerBuildCmd, DockerManifestCmd}, }, { Name: "serve [config]",