[CI] Add Codecov support (#1065)

* [CI] Add Codecov support

* [CI] Capture backend coverage from integration tests

* [CI] Remove unnecessary artifacts for coverage build

* [CI] Only run coverage elements where necessary

* [CI] Simplify post-command hook

* Fix yarn dependencies and collect coverage

* [CI] Include cmd/authelia/ path in coverage

* [CI] Exclude internal/suites/ in coverage

Closes #1061.

.buildkite/deployment.sh

@@ -3,7 +3,7 @@ set -u
 
 DIVERGED=$(git merge-base --fork-point origin/master > /dev/null; echo $?)
 
-if [[ $DIVERGED -eq 0 ]]; then
+if [[ $DIVERGED == 0 ]]; then
   if [[ $BUILDKITE_TAG == "" ]]; then
     if [[ $BUILDKITE_BRANCH == "master" ]]; then
       CI_BYPASS=$(git diff --name-only HEAD~1 | sed -rn '/^(BREAKING.md|CONTRIBUTING.md|README.md|docs\/.*)/!{q1}' && echo true || echo false)

.buildkite/hooks/post-command

@@ -9,6 +9,19 @@ if [[ $BUILDKITE_PULL_REQUEST != "false" ]]; then
   fi
 fi
 
+if [[ ! $BUILDKITE_BRANCH =~ ^(master|v.*) ]] && [[ $BUILDKITE_COMMAND_EXIT_STATUS == 0 ]]; then
+  if [[ $BUILDKITE_LABEL == ":hammer_and_wrench: Unit Test" ]]; then
+    echo "--- :codecov: Upload coverage reports"
+    bash <(curl -s https://codecov.io/bash) -c -f "coverage.txt" -F backend
+    bash <(curl -s https://codecov.io/bash) -c -F frontend
+  fi
+
+  if [[ $BUILDKITE_LABEL =~ ":selenium:" ]]; then
+    echo "--- :codecov: Upload coverage reports"
+    bash <(curl -s https://codecov.io/bash) -c -f "coverage.txt" -F backend
+  fi
+fi
+
 if [[ $BUILDKITE_LABEL =~ ":selenium:" ]] || [[ $BUILDKITE_LABEL =~ ":docker: Build Image" ]]; then
   CONTAINERS=$(docker ps -a -q)
   if [[ ${CONTAINERS} != "" ]]; then

.buildkite/hooks/pre-artifact

@@ -12,11 +12,13 @@ fi
 if [[ $BUILDKITE_LABEL =~ ":docker: Build Image" ]]; then
   echo "--- :docker: Saving artifacts for :buildkite: :docker: :github: releases"
   # Save binary for buildkite and github artifacts
-  docker create --name authelia-binary ${DOCKER_IMAGE}:latest
+  if [[ "${ARCH}" != "coverage" ]]; then
-  docker cp authelia-binary:/usr/app/authelia ./authelia-"${OS}"-"${ARCH}"
+    docker create --name authelia-binary ${DOCKER_IMAGE}:latest
-  docker rm -f authelia-binary
+    docker cp authelia-binary:/usr/app/authelia ./authelia-"${OS}"-"${ARCH}"
-  tar -czf authelia-"${OS}"-"${ARCH}".tar.gz authelia-"${OS}"-"${ARCH}" authelia.service config.template.yml
+    docker rm -f authelia-binary
-  sha256sum authelia-"${OS}"-"${ARCH}".tar.gz > authelia-"${OS}"-"${ARCH}".tar.gz.sha256
+    tar -czf authelia-"${OS}"-"${ARCH}".tar.gz authelia-"${OS}"-"${ARCH}" authelia.service config.template.yml
+    sha256sum authelia-"${OS}"-"${ARCH}".tar.gz > authelia-"${OS}"-"${ARCH}".tar.gz.sha256
+  fi
   # Saving image for push to docker hub
   docker save ${DOCKER_IMAGE} | zstdmt -T0 -12 > authelia-image-"${ARCH}".tar.zst
 fi

.buildkite/hooks/pre-command

@@ -3,7 +3,7 @@
 set +u
 
 if [[ $BUILDKITE_LABEL =~ ":selenium:" ]]; then
-  DEFAULT_ARCH=amd64
+  DEFAULT_ARCH=coverage
   echo "--- :docker: Extract, load and tag build container"
   buildkite-agent artifact download "authelia-image-${DEFAULT_ARCH}*" .
   zstdcat authelia-image-${DEFAULT_ARCH}.tar.zst | docker load

.buildkite/pipeline.sh

@@ -3,7 +3,7 @@ set -u
 
 DIVERGED=$(git merge-base --fork-point origin/master > /dev/null; echo $?)
 
-if [[ $DIVERGED -eq 0 ]]; then
+if [[ $DIVERGED == 0 ]]; then
   if [[ $BUILDKITE_TAG == "" ]]; then
     if [[ $BUILDKITE_BRANCH == "master" ]]; then
       CI_BYPASS=$(git diff --name-only HEAD~1 | sed -rn '/^(BREAKING.md|CONTRIBUTING.md|README.md|docs\/.*)/!{q1}' && echo true || echo false)
@@ -53,6 +53,6 @@ steps:
   - label: ":chrome: Integration Tests"
     command: ".buildkite/steps/e2etests.sh | buildkite-agent pipeline upload"
     depends_on:
-      - "build-docker-linux-amd64"
+      - "build-docker-linux-coverage"
     if: build.branch !~ /^(master)|(v[0-9]+\.[0-9]+\.[0-9]+)$\$/ && build.env("CI_BYPASS") != "true"
 EOF

.buildkite/steps/buildimages.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -eu
 
-declare -A BUILDS=(["linux"]="amd64 arm32v7 arm64v8")
+declare -A BUILDS=(["linux"]="amd64 arm32v7 arm64v8 coverage")
 
 for BUILD_OS in "${!BUILDS[@]}"; do
   for BUILD_ARCH in ${BUILDS[$BUILD_OS]}; do
@@ -12,12 +12,23 @@ cat << EOF
       build: "${BUILD_OS}-${BUILD_ARCH}"
     artifact_paths:
       - "authelia-image-${BUILD_ARCH}.tar.zst"
+EOF
+if [[ "${BUILD_ARCH}" != "coverage" ]]; then
+cat << EOF
       - "authelia-${BUILD_OS}-${BUILD_ARCH}.tar.gz"
       - "authelia-${BUILD_OS}-${BUILD_ARCH}.tar.gz.sha256"
+EOF
+fi
+cat << EOF
     env:
       ARCH: "${BUILD_ARCH}"
       OS: "${BUILD_OS}"
     key: "build-docker-${BUILD_OS}-${BUILD_ARCH}"
 EOF
+if [[ "${BUILD_ARCH}" == "coverage" ]]; then
+cat << EOF
+    if: build.branch !~ /^(master)|(v[0-9]+\.[0-9]+\.[0-9]+)$\$/
+EOF
+fi
   done
 done

.buildkite/steps/ghartifacts.sh

@@ -15,7 +15,7 @@ done
 echo "--- :github: Deploy artifacts for release: ${BUILDKITE_TAG}"
 hub release create "${BUILDKITE_TAG}" "${artifacts[@]}" -F <(echo -e "${BUILDKITE_TAG}\n\n$(awk "/${BUILDKITE_TAG}/" RS="## Breaking" BREAKING.md)\n\n## Changelog\n$(git log --oneline --pretty='* %h %s' $(git describe --abbrev=0 --tags $(git rev-list --tags --skip=1 --max-count=1))...$(git describe --abbrev=0 --tags))\n\n## Docker Container\n* \`docker pull authelia/authelia:${BUILDKITE_TAG//v}\`" | sed -e 's/^ /## Breaking /' | sed -e '/./b' -e :n -e 'N;s/\n$//;tn'); EXIT=$?
 
-if [[ $EXIT -eq 0 ]];
+if [[ $EXIT == 0 ]];
   then
     exit
   else

.codecov.yml

@@ -0,0 +1,39 @@
+codecov:
+  require_ci_to_pass: yes
+
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+  require_changes: no
+
+coverage:
+  precision: 2
+  round: down
+  range: "70...100"
+  status:
+    project:
+      default: off
+      backend:
+        flags:
+          - backend
+      frontend:
+        flags:
+          - frontend
+
+flags:
+  backend:
+    paths:
+      - "cmd/authelia/"
+      - "internal/"
+      - "!internal/suites/"
+  frontend:
+    paths:
+      - "web/"
+
+parsers:
+  gcov:
+    branch_detection:
+      conditional: yes
+      loop: yes
+      method: no
+      macro: no

.gitignore

@@ -7,6 +7,7 @@ npm-debug.log*
 
 # Coverage reports
 coverage/
+coverage.txt
 
 .vscode/
 

Dockerfile.coverage

@@ -0,0 +1,65 @@
+# ========================================
+# ===== Build image for the frontend =====
+# ========================================
+FROM node:14-alpine AS builder-frontend
+
+WORKDIR /node/src/app
+COPY web .
+
+# Install the dependencies and build
+RUN yarn install --frozen-lockfile && INLINE_RUNTIME_CHUNK=false yarn build
+
+# =======================================
+# ===== Build image for the backend =====
+# =======================================
+FROM golang:1.14.4-alpine AS builder-backend
+
+ARG BUILD_TAG
+ARG BUILD_COMMIT
+
+# gcc and musl-dev are required for building go-sqlite3
+RUN apk --no-cache add gcc musl-dev
+
+WORKDIR /go/src/app
+
+COPY go.mod go.sum ./
+COPY --from=builder-frontend /node/src/app/build public_html
+
+RUN go mod download
+
+COPY cmd cmd
+COPY internal internal
+
+# Prepare static files to be embedded in Go binary
+RUN go get -u aletheia.icu/broccoli && \
+cd internal/server && \
+go generate .
+
+# Set the build version and time
+RUN echo "Write tag ${BUILD_TAG} and commit ${BUILD_COMMIT} in binary." && \
+    sed -i "s/__BUILD_TAG__/${BUILD_TAG}/" cmd/authelia/constants.go && \
+    sed -i "s/__BUILD_COMMIT__/${BUILD_COMMIT}/" cmd/authelia/constants.go
+
+# Build binary for collecting integration test coverage
+RUN cd cmd/authelia && \
+go test -c --tags coverage -covermode=atomic -o authelia -coverpkg github.com/authelia/authelia/...
+
+# ===================================
+# ===== Authelia official image =====
+# ===================================
+FROM alpine:3.12.0
+
+RUN apk --no-cache add ca-certificates tzdata
+
+WORKDIR /usr/app
+
+COPY --from=builder-backend /go/src/app/cmd/authelia/authelia ./
+
+EXPOSE 9091
+
+VOLUME /etc/authelia
+VOLUME /var/lib/authelia
+
+ENV PATH="/usr/app:${PATH}"
+
+CMD ["./authelia", "-test.coverprofile=/app/coverage.txt", "COVERAGE", "--config", "/etc/authelia/configuration.yml"]

cmd/authelia-scripts/cmd_docker.go

@@ -15,7 +15,7 @@ import (
 
 var arch string
 
-var supportedArch = []string{"amd64", "arm32v7", "arm64v8", "darwin"}
+var supportedArch = []string{"amd64", "arm32v7", "arm64v8", "coverage"}
 var defaultArch = "amd64"
 var buildkiteQEMU = os.Getenv("BUILDKITE_AGENT_META_DATA_QEMU")
 var ciBranch = os.Getenv("BUILDKITE_BRANCH")

cmd/authelia-scripts/cmd_unittest.go

@@ -13,7 +13,7 @@ import (
 func RunUnitTest(cobraCmd *cobra.Command, args []string) {
 	log.SetLevel(log.TraceLevel)
 
-	if err := utils.Shell("go test $(go list ./... | grep -v suites)").Run(); err != nil {
+	if err := utils.Shell("go test -coverprofile=coverage.txt -covermode=atomic $(go list ./... | grep -v suites)").Run(); err != nil {
 		log.Fatal(err)
 	}
 

cmd/authelia/main_test.go

@@ -0,0 +1,43 @@
+// +build coverage
+
+package main
+
+import (
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+func TestAuthelia(t *testing.T) {
+	var (
+		args []string
+	)
+
+	for _, arg := range os.Args {
+		switch {
+		case strings.HasPrefix(arg, "COVERAGE"):
+		case strings.HasPrefix(arg, "-test"):
+		default:
+			args = append(args, arg)
+		}
+	}
+
+	waitCh := make(chan int, 1)
+	os.Args = args
+
+	go func() {
+		main()
+		close(waitCh)
+	}()
+
+	signalCh := make(chan os.Signal, 1)
+	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGHUP)
+	select {
+	case <-signalCh:
+		return
+	case <-waitCh:
+		return
+	}
+}

internal/suites/example/compose/authelia/docker-compose.backend.dist.yml

@@ -11,6 +11,8 @@ services:
       - 'traefik.http.routers.authelia_backend.entrypoints=https'
       - 'traefik.http.routers.authelia_backend.tls=true'
       - 'traefik.http.services.authelia_backend.loadbalancer.server.scheme=https'
+    volumes:
+      - '../..:/app'
     environment:
       - ENVIRONMENT=dev
     restart: always

web/package.json

@@ -12,12 +12,12 @@
     "@types/chai": "^4.2.11",
     "@types/classnames": "^2.2.10",
     "@types/enzyme": "^3.10.5",
-    "@types/jest": "25.2.3",
+    "@types/jest": "^25.2.3",
-    "@types/node": "14.0.10",
+    "@types/node": "^14.0.10",
     "@types/qrcode.react": "^1.0.1",
     "@types/query-string": "^6.3.0",
     "@types/react": "^16.9.35",
-    "@types/react-dom": "16.9.8",
+    "@types/react-dom": "^16.9.8",
     "@types/react-ga": "^2.3.0",
     "@types/react-router-dom": "^5.1.5",
     "axios": "^0.19.2",
@@ -34,15 +34,15 @@
     "react-loading": "^2.0.3",
     "react-otp-input": "^1.0.1",
     "react-router-dom": "^5.2.0",
-    "react-scripts": "3.4.1",
+    "react-scripts": "^3.4.1",
     "react-test-renderer": "^16.13.1",
-    "typescript": "3.9.5",
+    "typescript": "^3.9.5",
     "u2f-api": "^1.1.1"
   },
   "scripts": {
     "start": "react-scripts start",
     "build": "react-scripts build",
-    "test": "react-scripts test --no-cache",
+    "test": "react-scripts test --coverage --no-cache",
     "eject": "react-scripts eject"
   },
   "eslintConfig": {