[Buildkite] Add automatic deployment and removal of Docker images for Branches and PRs (#592)

.buildkite/hooks/post-checkout

@@ -3,16 +3,13 @@
 set +u
 
 if [[ ! $BUILDKITE_COMMAND =~ "buildkite-agent pipeline upload" ]] || \
-[[ $BUILDKITE_COMMAND == ".buildkite/steps/e2etests.sh | buildkite-agent pipeline upload" ]];
-then
+[[ $BUILDKITE_COMMAND == ".buildkite/steps/e2etests.sh | buildkite-agent pipeline upload" ]]; then
   echo "--- :buildkite: Setting up Build environment"
   source bootstrap.sh
-  if [[ $BUILDKITE_COMMAND == "authelia-scripts --log-level debug ci" ]];
-  then
+  if [[ $BUILDKITE_COMMAND == "authelia-scripts --log-level debug ci" ]]; then
     go mod download
   fi
-  if [[ $BUILDKITE_LABEL =~ ":selenium:" ]];
-  then
+  if [[ $BUILDKITE_LABEL =~ ":selenium:" ]]; then
     go mod download
   fi
 fi

.buildkite/hooks/post-command

@@ -2,12 +2,32 @@
 
 set +u
 
-if [[ $BUILDKITE_LABEL =~ ":selenium:" ]] || [[ $BUILDKITE_LABEL =~ ":docker: Build Image" ]];
-then
+if [[ $BUILDKITE_LABEL =~ ":selenium:" ]] || [[ $BUILDKITE_LABEL =~ ":docker: Build Image" ]]; then
   CONTAINERS=$(docker ps -a -q)
-  if [[ ${CONTAINERS} != "" ]];
-  then
+  if [[ ${CONTAINERS} != "" ]]; then
     echo "--- :docker: Clean environment"
     docker rm -f ${CONTAINERS}
   fi
+fi
+
+if [[ $BUILDKITE_BRANCH == "master" ]] || [[ $BUILDKITE_BRANCH =~ ^v.* ]] && [[ $BUILDKITE_PULL_REQUEST == "false" ]]; then
+  if [[ $BUILDKITE_LABEL == ":docker: Deploy Manifests" ]]; then
+    echo "--- :docker: Removing tags for deleted branches"
+    for BRANCH_TAG in $(dockerbranchtags=$(anontoken=$(curl -fsL --retry 3 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:authelia/authelia:pull' | jq -r .token) && \
+    curl -fsL --retry 3 -H "Authorization: Bearer ${anontoken}" https://registry-1.docker.io/v2/authelia/authelia/tags/list | jq -r '.tags[] | select(startswith("PR") | not)' | \
+    sed -r '/^(latest|develop|v.*|([[:digit:]]+)\.?([[:digit:]]+)?\.?([[:digit:]]+)?)/d' | sort) && \
+    githubbranches=$(curl -fs --retry 3 https://api.github.com/repos/authelia/authelia/branches | jq -r '.[].name' | sort) && \
+    comm -23 <(echo "${dockerbranchtags}") <(echo "${githubbranches}")); do
+      echo "Removing tag ${BRANCH_TAG}"
+      authtoken=$(curl -fs --retry 3 -H "Content-Type: application/json" -X "POST" -d '{"username": "'${DOCKER_USERNAME}'", "password": "'${DOCKER_PASSWORD}'"}' https://hub.docker.com/v2/users/login/ | jq -r .token) && \
+      curl -fsL --retry 3 -o /dev/null -X "DELETE" -H "Authorization: JWT ${authtoken}" https://hub.docker.com/v2/repositories/authelia/authelia/tags/${BRANCH_TAG}/
+    done
+    echo "--- :docker: Removing tags for merged or closed pull requests"
+    for PR_TAG in $(dockerprtags=$(curl -fsL --retry 3 -H "Authorization: Bearer ${anontoken}" https://registry-1.docker.io/v2/authelia/authelia/tags/list | jq -r '.tags[] | select(startswith("PR"))' | sort) && \
+    githubprs=$(curl -fs --retry 3 https://api.github.com/repos/authelia/authelia/pulls | jq -r '.[].number' | sed -e 's/^/PR/' | sort) && \
+    comm -23 <(echo "${dockerprtags}") <(echo "${githubprs}")); do
+      echo "Removing tag ${PR_TAG}"
+      curl -fsL --retry 3 -o /dev/null -X "DELETE" -H "Authorization: JWT ${authtoken}" https://hub.docker.com/v2/repositories/authelia/authelia/tags/${PR_TAG}/
+    done
+  fi
 fi

.buildkite/hooks/pre-artifact

@@ -4,16 +4,15 @@ set +u
 
 DOCKER_IMAGE=authelia/authelia
 
-if [[ $BUILDKITE_LABEL =~ ":docker: Build Image" ]];
-then
+if [[ $BUILDKITE_LABEL =~ ":docker: Build Image" ]]; then
   echo "--- :docker: Saving artifacts for :buildkite: :docker: :github: releases"
   # Save binary for buildkite and github artifacts
-  docker create --name authelia-binary $DOCKER_IMAGE:latest
+  docker create --name authelia-binary ${DOCKER_IMAGE}:latest
   docker cp authelia-binary:/usr/app/authelia ./authelia-linux-"${ARCH}"
   docker cp authelia-binary:/usr/app/public_html ./
   docker rm -f authelia-binary
   tar -czf authelia-linux-"${ARCH}".tar.gz authelia-linux-"${ARCH}" authelia.service config.template.yml public_html
   sha256sum authelia-linux-"${ARCH}".tar.gz > authelia-linux-"${ARCH}".tar.gz.sha256
   # Saving image for push to docker hub
-  docker save $DOCKER_IMAGE | zstdmt -T0 -12 > authelia-image-"${ARCH}".tar.zst
+  docker save ${DOCKER_IMAGE} | zstdmt -T0 -12 > authelia-image-"${ARCH}".tar.zst
 fi

.buildkite/hooks/pre-command

@@ -2,8 +2,7 @@
 
 set +u
 
-if [[ $BUILDKITE_LABEL =~ ":selenium:" ]];
-then
+if [[ $BUILDKITE_LABEL =~ ":selenium:" ]]; then
   DEFAULT_ARCH=amd64
   echo "--- :docker: Extract, load and tag build container"
   buildkite-agent artifact download "authelia-image-${DEFAULT_ARCH}*" .
@@ -11,13 +10,11 @@ then
   docker tag authelia/authelia authelia:dist
 fi
 
-if [[ $BUILDKITE_LABEL =~ ":docker: Deploy Image" ]];
-then
+if [[ $BUILDKITE_LABEL =~ ":docker: Deploy Image" ]]; then
   buildkite-agent artifact download "authelia-image-${ARCH}*" .
   zstdcat authelia-image-"${ARCH}".tar.zst | docker load
 fi
 
-if [[ $BUILDKITE_LABEL == ":github: Deploy Artifacts" ]];
-then
+if [[ $BUILDKITE_LABEL == ":github: Deploy Artifacts" ]]; then
   buildkite-agent artifact download "authelia-linux-*" .
 fi

.buildkite/pipeline.yml

@@ -17,21 +17,17 @@ steps:
     depends_on:
       - "build-docker-amd64"
 
-  - wait:
-    if: build.branch == "master" || build.branch =~ /^v/
+  - wait
 
   - label: ":docker: Image Deployments"
     command: ".buildkite/steps/deployimages.sh | buildkite-agent pipeline upload"
-    branches: "master v*"
     concurrency: 1
     concurrency_group: "deployments"
 
-  - wait:
-    if: build.branch == "master" || build.branch =~ /^v/
+  - wait
 
   - label: ":docker: Deploy Manifests"
     command: "authelia-scripts docker push-manifest"
-    branches: "master v*"
     concurrency: 1
     concurrency_group: "deployments"
     env:

.buildkite/steps/buildimages.sh

@@ -10,10 +10,6 @@ do
   echo "      - \"authelia-image-${BUILD_ARCH}.tar.zst\""
   echo "      - \"authelia-linux-${BUILD_ARCH}.tar.gz\""
   echo "      - \"authelia-linux-${BUILD_ARCH}.tar.gz.sha256\""
-  if [[ "${BUILD_ARCH}" != "amd64" ]];
-  then
-    echo "    branches: \"master v*\""
-  fi
   echo "    env:"
   echo "      "ARCH: ${BUILD_ARCH}""
   echo "    key: \"build-docker-${BUILD_ARCH}\""

cmd/authelia-scripts/cmd_docker.go

@@ -23,6 +23,7 @@ var ciPullRequest = os.Getenv("BUILDKITE_PULL_REQUEST")
 var ciTag = os.Getenv("BUILDKITE_TAG")
 var dockerTags = regexp.MustCompile(`v(?P<Patch>(?P<Minor>(?P<Major>\d+)\.\d+)\.\d+.*)`)
 var ignoredSuffixes = regexp.MustCompile("alpha|beta")
+var publicRepo = regexp.MustCompile(`.*\:.*`)
 var tags = dockerTags.FindStringSubmatch(ciTag)
 
 func init() {
@@ -202,7 +203,13 @@ func deployManifest(docker *Docker, tag string, amd64tag string, arm32v7tag stri
 func publishDockerImage(arch string) {
 	docker := &Docker{}
 
-	if ciBranch == "master" && ciPullRequest == "false" {
+	if ciBranch != "master" && !publicRepo.MatchString(ciBranch) {
+		login(docker)
+		deploy(docker, ciBranch+"-"+arch)
+	} else if ciBranch != "master" && publicRepo.MatchString(ciBranch) {
+		login(docker)
+		deploy(docker, "PR"+ciPullRequest+"-"+arch)
+	} else if ciBranch == "master" && ciPullRequest == "false" {
 		login(docker)
 		deploy(docker, "master-"+arch)
 	} else if ciTag != "" {
@@ -227,7 +234,13 @@ func publishDockerImage(arch string) {
 func publishDockerManifest() {
 	docker := &Docker{}
 
-	if ciBranch == "master" && ciPullRequest == "false" {
+	if ciBranch != "master" && !publicRepo.MatchString(ciBranch) {
+		login(docker)
+		deployManifest(docker, ciBranch, ciBranch+"-amd64", ciBranch+"-arm32v7", ciBranch+"-arm64v8")
+	} else if ciBranch != "master" && publicRepo.MatchString(ciBranch) {
+		login(docker)
+		deployManifest(docker, "PR"+ciPullRequest, "PR"+ciPullRequest+"-amd64", "PR"+ciPullRequest+"-arm32v7", "PR"+ciPullRequest+"-arm64v8")
+	} else if ciBranch == "master" && ciPullRequest == "false" {
 		login(docker)
 		deployManifest(docker, "master", "master-amd64", "master-arm32v7", "master-arm64v8")
 		publishDockerReadme(docker)