From 74a7e96409e6b25932196add791d6ebb3eb30d4f Mon Sep 17 00:00:00 2001
From: Amir Zarrinkafsh <nightah@me.com>
Date: Sat, 4 Jun 2022 19:38:13 +1000
Subject: [PATCH] ci: add integration containers for duo and haproxy (#3479)

* ci: add integration containers for duo and haproxy

This change utilises specific integration containers for the DuoPush and HAProxy suites.
In the case of DuoPush suite specifically in dev mode the container will be built on suite startup.

* ci: factorize pre-command hook and unset async on trigger steps
---
 .buildkite/hooks/post-command                 |  6 ++-
 .buildkite/hooks/pre-command                  | 26 +++++++++-
 .buildkite/integration.sh                     | 37 +++++++++++++++
 .buildkite/pipeline.sh                        | 47 +++++++++++++++++--
 .editorconfig                                 |  6 ++-
 ...ker-compose.yml => docker-compose.dev.yml} |  0
 .../compose/duo-api/docker-compose.dist.yml   |  8 ++++
 .../compose/haproxy/docker-compose.yml        |  2 +-
 internal/suites/suite_bypass_all.go           |  1 -
 internal/suites/suite_duo_push.go             |  2 +-
 10 files changed, 126 insertions(+), 9 deletions(-)
 create mode 100755 .buildkite/integration.sh
 rename internal/suites/example/compose/duo-api/{docker-compose.yml => docker-compose.dev.yml} (100%)
 create mode 100644 internal/suites/example/compose/duo-api/docker-compose.dist.yml

diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command
index 15a4110a..92f8c7d9 100755
--- a/.buildkite/hooks/post-command
+++ b/.buildkite/hooks/post-command
@@ -39,6 +39,10 @@ if [[ "${BUILDKITE_LABEL}" == ":debian: Package Builds" ]]; then
   cat .buildkite/annotations/artifacts | buildkite-agent annotate --style "success" --context "ctx-success"
 fi
 
+if [[ "${BUILDKITE_LABEL}" == ":docker: Build and Deploy Image" ]]; then
+  docker logout
+fi
+
 if [[ "${BUILDKITE_LABEL}" =~ ":docker: Deploy" ]]; then
   docker logout
   docker logout ghcr.io
@@ -78,4 +82,4 @@ if [[ "${BUILDKITE_LABEL}" == ":docker: Deploy Manifest" ]] && [[ "${BUILDKITE_B
       done
     done
   done
-fi
\ No newline at end of file
+fi
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 5fe7078c..2b4754e5 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -2,6 +2,16 @@
 
 set +u
 
+INTEGRATION() {
+  if [[ "${BUILDKITE_BRANCH}" =~ ^renovate/ ]]; then
+    sed -i "s/${CONTAINER}/${CONTAINER}:renovate/" "${FILE}"
+  elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ ! "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+    sed -i "s/${CONTAINER}/${CONTAINER}:${BUILDKITE_BRANCH}/" "${FILE}"
+  elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+    sed -i "s/${CONTAINER}/${CONTAINER}:PR${BUILDKITE_PULL_REQUEST}/" "${FILE}"
+  fi
+}
+
 if [[ "${BUILDKITE_LABEL}" == ":service_dog: Linting" ]]; then
   cd web && pnpm install && cd ../
 fi
@@ -24,6 +34,20 @@ if [[ "${BUILDKITE_LABEL}" =~ ":selenium:" ]]; then
   buildkite-agent artifact download "authelia-image-${DEFAULT_ARCH}*" .
   zstdcat "authelia-image-${DEFAULT_ARCH}.tar.zst" | docker load
   docker tag authelia/authelia authelia:dist
+
+  if [[ "${BUILD_DUO}" == "true" ]] && [[ "${SUITE}" == "DuoPush" ]]; then
+    CONTAINER="integration-duo"
+    FILE="internal/suites/example/compose/duo-api/docker-compose.dist.yml"
+    INTEGRATION
+  elif [[ "${BUILD_HAPROXY}" == "true" ]] && [[ "${SUITE}" == "HAProxy" ]]; then
+    CONTAINER="integration-haproxy"
+    FILE="internal/suites/example/compose/haproxy/docker-compose.yml"
+    INTEGRATION
+  fi
+fi
+
+if [[ "${BUILDKITE_LABEL}" == ":docker: Build and Deploy" ]]; then
+  echo ${DOCKER_PASSWORD} | docker login -u ${DOCKER_USERNAME} --password-stdin
 fi
 
 if [[ "${BUILDKITE_LABEL}" == ":docker: Deploy Manifest" ]]; then
@@ -39,4 +63,4 @@ fi
 
 if [[ "${BUILDKITE_LABEL}" == ":debian: :fedora: :ubuntu: Deploy APT" ]]; then
   buildkite-agent artifact download "authelia_*.deb" .
-fi
\ No newline at end of file
+fi
diff --git a/.buildkite/integration.sh b/.buildkite/integration.sh
new file mode 100755
index 00000000..9ee5a95f
--- /dev/null
+++ b/.buildkite/integration.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -u
+
+DIRECTORY="unset"
+GROUP="unset"
+PREFIX="authelia/"
+TAG="unset"
+
+if [[ "${BUILDKITE_BRANCH}" =~ ^renovate/ ]]; then
+  TAG="renovate"
+elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ ! "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+  TAG="${BUILDKITE_BRANCH}"
+elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+  TAG="PR${BUILDKITE_PULL_REQUEST}"
+elif [[ "${BUILDKITE_BRANCH}" == "master" ]] && [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then
+  TAG="latest"
+fi
+
+if [[ "${BUILDKITE_PIPELINE_NAME}" == "integration-duo" ]]; then
+  DIRECTORY="internal/suites/example/compose/duo-api"
+  GROUP="duo-deployments"
+elif [[ "${BUILDKITE_PIPELINE_NAME}" == "integration-haproxy" ]]; then
+  DIRECTORY="internal/suites/example/compose/haproxy"
+  GROUP="haproxy-deployments"
+fi
+
+cat << EOF
+steps:
+  - label: ":docker: Build and Deploy"
+    commands:
+      - "cd ${DIRECTORY}"
+      - "docker build --tag ${PREFIX}${BUILDKITE_PIPELINE_NAME}:${TAG} --platform linux/amd64 --builder buildx --pull --push ."
+    concurrency: 1
+    concurrency_group: "${GROUP}"
+    agents:
+      upload: "fast"
+EOF
diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh
index bc907b04..12f156d4 100755
--- a/.buildkite/pipeline.sh
+++ b/.buildkite/pipeline.sh
@@ -6,23 +6,33 @@ DIVERGED=$(git merge-base --fork-point origin/master > /dev/null; echo $?)
 if [[ "${DIVERGED}" == 0 ]]; then
   if [[ "${BUILDKITE_TAG}" == "" ]]; then
     if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then
+      BUILD_DUO=$(git diff --name-only HEAD~1 | grep -q ^internal/suites/example/compose/duo-api/.* && echo true || echo false)
+      BUILD_HAPROXY=$(git diff --name-only HEAD~1 | grep -q ^internal/suites/example/compose/haproxy/Dockerfile && echo true || echo false)
       CI_BYPASS=$(git diff --name-only HEAD~1 | sed -rn '/^(CODE_OF_CONDUCT\.md|CONTRIBUTING\.md|README\.md|SECURITY\.md|crowdin\.yml|\.all-contributorsrc|\.editorconfig|\.github\/.*|docs\/.*|examples\/.*)/!{q1}' && echo true || echo false)
     else
+      BUILD_DUO=$(git diff --name-only `git merge-base --fork-point origin/master` | grep -q ^internal/suites/example/compose/duo-api/.* && echo true || echo false)
+      BUILD_HAPROXY=$(git diff --name-only `git merge-base --fork-point origin/master` | grep -q ^internal/suites/example/compose/haproxy/Dockerfile && echo true || echo false)
       CI_BYPASS=$(git diff --name-only `git merge-base --fork-point origin/master` | sed -rn '/^(CODE_OF_CONDUCT\.md|CONTRIBUTING\.md|README\.md|SECURITY\.md|crowdin\.yml|\.all-contributorsrc|\.editorconfig|\.github\/.*|docs\/.*|examples\/.*)/!{q1}' && echo true || echo false)
     fi
 
-    if [[ $CI_BYPASS == "true" ]]; then
+    if [[ "${CI_BYPASS}" == "true" ]]; then
       cat .buildkite/annotations/bypass | buildkite-agent annotate --style "info" --context "ctx-info"
     fi
   else
+    BUILD_DUO="false"
+    BUILD_HAPROXY="false"
     CI_BYPASS="false"
   fi
 else
+  BUILD_DUO="false"
+  BUILD_HAPROXY="false"
   CI_BYPASS="false"
 fi
 
 cat << EOF
 env:
+  BUILD_DUO: ${BUILD_DUO}
+  BUILD_HAPROXY: ${BUILD_HAPROXY}
   CI_BYPASS: ${CI_BYPASS}
 
 steps:
@@ -42,7 +52,38 @@ steps:
 
   - wait:
     if: build.env("CI_BYPASS") != "true"
-
+EOF
+if [[ "${BUILD_DUO}" == "true" ]]; then
+cat << EOF
+  - label: ":rocket: Trigger Pipeline [integration-duo]"
+    trigger: "integration-duo"
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}"
+        BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        BUILDKITE_PULL_REQUEST_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
+    depends_on: ~
+EOF
+fi
+if [[ "${BUILD_HAPROXY}" == "true" ]]; then
+cat << EOF
+  - label: ":rocket: Trigger Pipeline [integration-haproxy]"
+    trigger: "integration-haproxy"
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}"
+        BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        BUILDKITE_PULL_REQUEST_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
+    depends_on: ~
+EOF
+fi
+cat << EOF
   - label: ":docker: Build Image [coverage]"
     command: "authelia-scripts docker build --container=coverage"
     retry:
@@ -69,4 +110,4 @@ steps:
     depends_on:
       - "build-docker-linux-coverage"
     if: build.branch !~ /^(v[0-9]+\.[0-9]+\.[0-9]+)$\$/ && build.env("CI_BYPASS") != "true" && build.message !~ /\[(skip test|test skip)\]/
-EOF
\ No newline at end of file
+EOF
diff --git a/.editorconfig b/.editorconfig
index 2766e0c3..8ad12948 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -7,7 +7,11 @@ trim_trailing_whitespace = true
 end_of_line = lf
 insert_final_newline = true
 
-[*.{yml,yaml}]
+[.buildkite/hooks/**]
+indent_style = space
+indent_size = 2
+
+[*.{sh,yml,yaml}]
 indent_style = space
 indent_size = 2
 
diff --git a/internal/suites/example/compose/duo-api/docker-compose.yml b/internal/suites/example/compose/duo-api/docker-compose.dev.yml
similarity index 100%
rename from internal/suites/example/compose/duo-api/docker-compose.yml
rename to internal/suites/example/compose/duo-api/docker-compose.dev.yml
diff --git a/internal/suites/example/compose/duo-api/docker-compose.dist.yml b/internal/suites/example/compose/duo-api/docker-compose.dist.yml
new file mode 100644
index 00000000..79e9aa21
--- /dev/null
+++ b/internal/suites/example/compose/duo-api/docker-compose.dist.yml
@@ -0,0 +1,8 @@
+---
+version: '3'
+services:
+  duo-api:
+    image: authelia/integration-duo
+    networks:
+      - authelianet
+...
diff --git a/internal/suites/example/compose/haproxy/docker-compose.yml b/internal/suites/example/compose/haproxy/docker-compose.yml
index 39b530c6..3681cdcd 100644
--- a/internal/suites/example/compose/haproxy/docker-compose.yml
+++ b/internal/suites/example/compose/haproxy/docker-compose.yml
@@ -2,7 +2,7 @@
 version: '3'
 services:
   haproxy:
-    build: ./example/compose/haproxy/
+    image: authelia/integration-haproxy
     volumes:
       - ./example/compose/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
       - ./example/compose/haproxy/http.lua:/usr/local/etc/haproxy/haproxy-lua-http/http.lua
diff --git a/internal/suites/suite_bypass_all.go b/internal/suites/suite_bypass_all.go
index 556b18d3..fa5cc289 100644
--- a/internal/suites/suite_bypass_all.go
+++ b/internal/suites/suite_bypass_all.go
@@ -17,7 +17,6 @@ func init() {
 		"internal/suites/example/compose/nginx/portal/docker-compose.yml",
 		"internal/suites/example/compose/httpbin/docker-compose.yml",
 		"internal/suites/example/compose/smtp/docker-compose.yml",
-		"internal/suites/example/compose/duo-api/docker-compose.yml",
 	})
 
 	setup := func(suitePath string) error {
diff --git a/internal/suites/suite_duo_push.go b/internal/suites/suite_duo_push.go
index 58665584..4bef9447 100644
--- a/internal/suites/suite_duo_push.go
+++ b/internal/suites/suite_duo_push.go
@@ -16,7 +16,7 @@ func init() {
 		"internal/suites/example/compose/authelia/docker-compose.frontend.{}.yml",
 		"internal/suites/example/compose/nginx/backend/docker-compose.yml",
 		"internal/suites/example/compose/nginx/portal/docker-compose.yml",
-		"internal/suites/example/compose/duo-api/docker-compose.yml",
+		"internal/suites/example/compose/duo-api/docker-compose.{}.yml",
 	})
 
 	setup := func(suitePath string) error {