refactor(suites): stop integration tests on first failure (#3270)

* refactor(suites): stop integration tests on first failure

* refactor(suites): remove additional nginx instance

* refactor(suites): log relevant containers

* refactor(suites): add traefik2 logs to stdout

* refactor(suites): explicitly enable traefik for tests

* refactor(suites): remove redis restart and duplicate pathprefix tests

* ci(buildkite): allow manual retry on integration tests

.buildkite/steps/e2etests.sh

@@ -4,9 +4,11 @@ set -eu
 for SUITE_NAME in $(authelia-scripts suites list); do
 cat << EOF
   - label: ":selenium: ${SUITE_NAME} Suite"
-    command: "authelia-scripts --log-level debug suites test ${SUITE_NAME} --headless"
+    command: "authelia-scripts --log-level debug suites test ${SUITE_NAME} --failfast --headless"
     retry:
       automatic: true
+      manual:
+        permit_on_passed: true
 EOF
 if [[ "${SUITE_NAME}" = "ActiveDirectory" ]]; then
 cat << EOF

cmd/authelia-scripts/cmd_suites.go

@@ -26,10 +26,12 @@ var ErrNoRunningSuite = errors.New("no running suite")
 // runningSuiteFile name of the file containing the currently running suite.
 var runningSuiteFile = ".suite"
 
+var failfast bool
 var headless bool
 var testPattern string
 
 func init() {
+	SuitesTestCmd.Flags().BoolVar(&failfast, "failfast", false, "Stops tests on first failure")
 	SuitesTestCmd.Flags().BoolVar(&headless, "headless", false, "Run tests in headless mode")
 	SuitesTestCmd.Flags().StringVar(&testPattern, "test", "", "The single test to run")
 }
@@ -279,7 +281,12 @@ func runSuiteTests(suiteName string, withEnv bool) error {
 		timeout = fmt.Sprintf("%ds", int64(suite.TestTimeout/time.Second))
 	}
 
-	testCmdLine := fmt.Sprintf("go test -count=1 -v ./internal/suites -timeout %s ", timeout)
+	fail := ""
+	if failfast {
+		fail = "-failfast"
+	}
+
+	testCmdLine := fmt.Sprintf("go test -count=1 -v ./internal/suites -timeout %s %s ", timeout, fail)
 
 	if testPattern != "" {
 		testCmdLine += fmt.Sprintf("-run '%s'", testPattern)

internal/suites/example/compose/authelia/docker-compose.backend.dev.yml

@@ -24,6 +24,7 @@ services:
       - 'traefik.frontend.rule=Host:login.example.com;PathPrefix:/api,/locales'
       - 'traefik.protocol=https'
       # Traefik 2.x
+      - 'traefik.enable=true'
       - 'traefik.http.routers.authelia_backend.rule=Host(`login.example.com`) && PathPrefix(`/.well-known`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/.well-known`) || Host(`login.example.com`) && PathPrefix(`/api`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/api`) || Host(`login.example.com`) && PathPrefix(`/locales`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/locales`) || Host(`login.example.com`) && Path(`/jwks.json`) || Host(`login.example.com`) && Path(`${PathPrefix}/jwks.json`)'  # yamllint disable-line rule:line-length
       - 'traefik.http.routers.authelia_backend.entrypoints=https'
       - 'traefik.http.routers.authelia_backend.tls=true'

internal/suites/example/compose/authelia/docker-compose.backend.dist.yml

@@ -5,14 +5,16 @@ services:
     image: authelia:dist
     labels:
       # Traefik 1.x
-      - 'traefik.frontend.rule=Host:login.example.com;PathPrefix:/api,/locales'
+      - 'traefik.frontend.rule=Host:login.example.com'
       - 'traefik.protocol=https'
       # Traefik 2.x
-      - 'traefik.http.routers.authelia_backend.rule=Host(`login.example.com`) && PathPrefix(`/.well-known`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/.well-known`) || Host(`login.example.com`) && PathPrefix(`/api`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/api`) || Host(`login.example.com`) && PathPrefix(`/locales`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/locales`) || Host(`login.example.com`) && Path(`/jwks.json`) || Host(`login.example.com`) && Path(`${PathPrefix}/jwks.json`)'  # yamllint disable-line rule:line-length
+      - 'traefik.enable=true'
+      - 'traefik.http.routers.authelia_backend.rule=Host(`login.example.com`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/`)'  # yamllint disable-line rule:line-length
       - 'traefik.http.routers.authelia_backend.entrypoints=https'
       - 'traefik.http.routers.authelia_backend.tls=true'
       - 'traefik.http.services.authelia_backend.loadbalancer.server.scheme=https'
-      - 'traefik.http.services.authelia_backend.passHostHeader=true'
+    healthcheck:
+      disable: true
     volumes:
       - '../..:/authelia'
     environment:

internal/suites/example/compose/authelia/docker-compose.frontend.dev.yml

@@ -19,6 +19,7 @@ services:
       # Traefik 1.x
       - 'traefik.frontend.rule=Host:login.example.com'
       # Traefik 2.x
+      - 'traefik.enable=true'
       - 'traefik.http.routers.authelia_frontend.rule=Host(`login.example.com`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/`)'  # yamllint disable-line rule:line-length
       - 'traefik.http.routers.authelia_frontend.entrypoints=https'
       - 'traefik.http.routers.authelia_frontend.tls=true'

internal/suites/example/compose/authelia/docker-compose.frontend.dist.yml

@@ -9,6 +9,7 @@ services:
       # Traefik 1.x
       - 'traefik.frontend.rule=Host:login.example.com'
       # Traefik 2.x
+      - 'traefik.enable=true'
       - 'traefik.http.routers.authelia_frontend.rule=Host(`login.example.com`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/`)'  # yamllint disable-line rule:line-length
       - 'traefik.http.routers.authelia_frontend.entrypoints=https'
       - 'traefik.http.routers.authelia_frontend.tls=true'

internal/suites/example/compose/haproxy/haproxy.cfg

@@ -73,7 +73,8 @@ backend be_authelia
     server authelia-backend authelia-backend:9091 resolvers docker ssl verify none
 
 backend fe_authelia
-    server authelia-frontend authelia-frontend:3000 resolvers docker
+    server authelia-frontend authelia-frontend:3000 check resolvers docker
+    server authelia-backend authelia-backend:9091 check backup resolvers docker ssl verify none
 
 backend be_httpbin
     acl remote_user_exist var(req.auth_response_header.remote_user) -m found

internal/suites/example/compose/httpbin/docker-compose.yml

@@ -14,6 +14,7 @@ services:
       - 'traefik.frontend.auth.forward.trustForwardHeader=true'
       - 'traefik.frontend.auth.forward.authResponseHeaders=Remote-User, Remote-Groups, Remote-Name, Remote-Email'
       # Traefik 2.x
+      - 'traefik.enable=true'
       - 'traefik.http.routers.httpbin.rule=Host(`public.example.com`) && Path(`/headers`)'
       - 'traefik.http.routers.httpbin.priority=150'
       - 'traefik.http.routers.httpbin.tls=true'

internal/suites/example/compose/nginx/backend/docker-compose.yml

@@ -11,6 +11,7 @@ services:
       - 'traefik.frontend.auth.forward.trustForwardHeader=true'
       - 'traefik.frontend.auth.forward.authResponseHeaders=Remote-User,Remote-Groups,Remote-Name,Remote-Email'
       # Traefik 2.x
+      - 'traefik.enable=true'
       - 'traefik.http.routers.protectedapps.rule=Host(`home.example.com`,`public.example.com`,`secure.example.com`,`admin.example.com`,`singlefactor.example.com`)'  # yamllint disable-line rule:line-length
       - 'traefik.http.routers.protectedapps.entrypoints=https'
       - 'traefik.http.routers.protectedapps.tls=true'

internal/suites/example/compose/oidc-client/docker-compose.yml

@@ -11,6 +11,7 @@ services:
     expose:
       - 8080
     labels:
+      - 'traefik.enable=true'
       - 'traefik.http.routers.oidc.rule=Host(`oidc.example.com`)'
       - 'traefik.http.routers.oidc.priority=150'
       - 'traefik.http.routers.oidc.tls=true'

internal/suites/example/compose/smtp/docker-compose.yml

@@ -8,6 +8,7 @@ services:
     labels:
       - 'traefik.frontend.rule=Host:mail.example.com'
       - 'traefik.port=1080'
+      - 'traefik.enable=true'
       - 'traefik.http.routers.mail.rule=Host(`mail.example.com`)'
       - 'traefik.http.routers.mail.entrypoints=https'
       - 'traefik.http.routers.mail.tls=true'

internal/suites/example/compose/traefik2/docker-compose.yml

@@ -6,6 +6,7 @@ services:
     volumes:
       - '/var/run/docker.sock:/var/run/docker.sock'
     labels:
+      - 'traefik.enable=true'
       - 'traefik.http.routers.api.rule=Host(`traefik.example.com`)'
       - 'traefik.http.routers.api.entrypoints=https'
       - 'traefik.http.routers.api.service=api@internal'
@@ -16,13 +17,14 @@ services:
       - 'traefik.http.middlewares.authelia.forwardauth.trustForwardHeader=true'
       - 'traefik.http.middlewares.authelia.forwardauth.authResponseHeaders=Remote-User, Remote-Groups, Remote-Name, Remote-Email'  # yamllint disable-line rule:line-length
     command:
+      - '--accesslog=true'
       - '--api'
       - '--providers.docker=true'
       - '--entrypoints.https=true'
       - '--entrypoints.https.address=:8080'
       - '--log=true'
       - '--log.level=DEBUG'
-      - '--log.filepath=/var/log/traefik.log'
+      - '--providers.docker.exposedByDefault=false'
       - '--serversTransport.insecureSkipVerify=true'
     networks:
       authelianet:

internal/suites/suite_haproxy.go

@@ -2,6 +2,7 @@ package suites
 
 import (
 	"fmt"
+	"os"
 	"time"
 )
 
@@ -19,6 +20,18 @@ func init() {
 		"internal/suites/example/compose/httpbin/docker-compose.yml",
 	})
 
+	if os.Getenv("CI") == t {
+		dockerEnvironment = NewDockerEnvironment([]string{
+			"internal/suites/docker-compose.yml",
+			"internal/suites/HAProxy/docker-compose.yml",
+			"internal/suites/example/compose/authelia/docker-compose.backend.{}.yml",
+			"internal/suites/example/compose/nginx/backend/docker-compose.yml",
+			"internal/suites/example/compose/haproxy/docker-compose.yml",
+			"internal/suites/example/compose/smtp/docker-compose.yml",
+			"internal/suites/example/compose/httpbin/docker-compose.yml",
+		})
+	}
+
 	setup := func(suitePath string) error {
 		if err := dockerEnvironment.Up(); err != nil {
 			return err
@@ -35,12 +48,14 @@ func init() {
 
 		fmt.Println(backendLogs)
 
-		frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil)
-		if err != nil {
-			return err
-		}
+		if os.Getenv("CI") != t {
+			frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil)
+			if err != nil {
+				return err
+			}
 
-		fmt.Println(frontendLogs)
+			fmt.Println(frontendLogs)
+		}
 
 		haproxyLogs, err := dockerEnvironment.Logs("haproxy", nil)
 		if err != nil {

internal/suites/suite_oidc_traefik.go

@@ -2,6 +2,7 @@ package suites
 
 import (
 	"fmt"
+	"os"
 	"time"
 )
 
@@ -20,6 +21,19 @@ func init() {
 		"internal/suites/example/compose/redis/docker-compose.yml",
 	})
 
+	if os.Getenv("CI") == t {
+		dockerEnvironment = NewDockerEnvironment([]string{
+			"internal/suites/docker-compose.yml",
+			"internal/suites/OIDCTraefik/docker-compose.yml",
+			"internal/suites/example/compose/authelia/docker-compose.backend.{}.yml",
+			"internal/suites/example/compose/nginx/backend/docker-compose.yml",
+			"internal/suites/example/compose/traefik2/docker-compose.yml",
+			"internal/suites/example/compose/smtp/docker-compose.yml",
+			"internal/suites/example/compose/oidc-client/docker-compose.yml",
+			"internal/suites/example/compose/redis/docker-compose.yml",
+		})
+	}
+
 	setup := func(suitePath string) error {
 		// TODO(c.michaud): use version in tags for oidc-client but in the meantime we pull the image to make sure it's
 		// up to date.
@@ -46,12 +60,14 @@ func init() {
 
 		fmt.Println(backendLogs)
 
-		frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil)
-		if err != nil {
-			return err
-		}
+		if os.Getenv("CI") != t {
+			frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil)
+			if err != nil {
+				return err
+			}
 
-		fmt.Println(frontendLogs)
+			fmt.Println(frontendLogs)
+		}
 
 		oidcClientLogs, err := dockerEnvironment.Logs("oidc-client", nil)
 		if err != nil {

internal/suites/suite_pathprefix.go

@@ -2,6 +2,7 @@ package suites
 
 import (
 	"fmt"
+	"os"
 	"time"
 )
 
@@ -19,6 +20,18 @@ func init() {
 		"internal/suites/example/compose/httpbin/docker-compose.yml",
 	})
 
+	if os.Getenv("CI") == t {
+		dockerEnvironment = NewDockerEnvironment([]string{
+			"internal/suites/docker-compose.yml",
+			"internal/suites/PathPrefix/docker-compose.yml",
+			"internal/suites/example/compose/authelia/docker-compose.backend.{}.yml",
+			"internal/suites/example/compose/nginx/backend/docker-compose.yml",
+			"internal/suites/example/compose/traefik2/docker-compose.yml",
+			"internal/suites/example/compose/smtp/docker-compose.yml",
+			"internal/suites/example/compose/httpbin/docker-compose.yml",
+		})
+	}
+
 	setup := func(suitePath string) error {
 		if err := dockerEnvironment.Up(); err != nil {
 			return err
@@ -35,12 +48,14 @@ func init() {
 
 		fmt.Println(backendLogs)
 
-		frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil)
-		if err != nil {
-			return err
-		}
+		if os.Getenv("CI") != t {
+			frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil)
+			if err != nil {
+				return err
+			}
 
-		fmt.Println(frontendLogs)
+			fmt.Println(frontendLogs)
+		}
 
 		return nil
 	}

internal/suites/suite_traefik.go

@@ -2,6 +2,7 @@ package suites
 
 import (
 	"fmt"
+	"os"
 	"time"
 )
 
@@ -19,6 +20,18 @@ func init() {
 		"internal/suites/example/compose/httpbin/docker-compose.yml",
 	})
 
+	if os.Getenv("CI") == t {
+		dockerEnvironment = NewDockerEnvironment([]string{
+			"internal/suites/docker-compose.yml",
+			"internal/suites/Traefik/docker-compose.yml",
+			"internal/suites/example/compose/authelia/docker-compose.backend.{}.yml",
+			"internal/suites/example/compose/nginx/backend/docker-compose.yml",
+			"internal/suites/example/compose/traefik/docker-compose.yml",
+			"internal/suites/example/compose/smtp/docker-compose.yml",
+			"internal/suites/example/compose/httpbin/docker-compose.yml",
+		})
+	}
+
 	setup := func(suitePath string) error {
 		if err := dockerEnvironment.Up(); err != nil {
 			return err
@@ -35,12 +48,14 @@ func init() {
 
 		fmt.Println(backendLogs)
 
-		frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil)
-		if err != nil {
-			return err
-		}
+		if os.Getenv("CI") != t {
+			frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil)
+			if err != nil {
+				return err
+			}
 
-		fmt.Println(frontendLogs)
+			fmt.Println(frontendLogs)
+		}
 
 		return nil
 	}

internal/suites/suite_traefik2.go

@@ -2,6 +2,7 @@ package suites
 
 import (
 	"fmt"
+	"os"
 	"time"
 )
 
@@ -20,6 +21,19 @@ var traefik2DockerEnvironment = NewDockerEnvironment([]string{
 })
 
 func init() {
+	if os.Getenv("CI") == t {
+		traefik2DockerEnvironment = NewDockerEnvironment([]string{
+			"internal/suites/docker-compose.yml",
+			"internal/suites/Traefik2/docker-compose.yml",
+			"internal/suites/example/compose/authelia/docker-compose.backend.{}.yml",
+			"internal/suites/example/compose/redis/docker-compose.yml",
+			"internal/suites/example/compose/nginx/backend/docker-compose.yml",
+			"internal/suites/example/compose/traefik2/docker-compose.yml",
+			"internal/suites/example/compose/smtp/docker-compose.yml",
+			"internal/suites/example/compose/httpbin/docker-compose.yml",
+		})
+	}
+
 	setup := func(suitePath string) error {
 		if err := traefik2DockerEnvironment.Up(); err != nil {
 			return err
@@ -36,12 +50,28 @@ func init() {
 
 		fmt.Println(backendLogs)
 
-		frontendLogs, err := traefik2DockerEnvironment.Logs("authelia-frontend", nil)
+		if os.Getenv("CI") != t {
+			frontendLogs, err := traefik2DockerEnvironment.Logs("authelia-frontend", nil)
+			if err != nil {
+				return err
+			}
+
+			fmt.Println(frontendLogs)
+		}
+
+		redisLogs, err := traefik2DockerEnvironment.Logs("redis", nil)
 		if err != nil {
 			return err
 		}
 
-		fmt.Println(frontendLogs)
+		fmt.Println(redisLogs)
+
+		traefikLogs, err := traefik2DockerEnvironment.Logs("traefik", nil)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(traefikLogs)
 
 		return nil
 	}

internal/suites/suite_traefik2_test.go

@@ -1,10 +1,7 @@
 package suites
 
 import (
-	"context"
-	"fmt"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/suite"
 )
@@ -29,35 +26,8 @@ func (s *Traefik2Suite) TestCustomHeaders() {
 	suite.Run(s.T(), NewCustomHeadersScenario())
 }
 
-func (s *Traefik2Suite) TestShouldKeepSessionAfterRedisRestart() {
-	ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
-	defer func() {
-		cancel()
-		s.collectCoverage(s.Page)
-		s.collectScreenshot(ctx.Err(), s.Page)
-		s.MustClose()
-		err := s.RodSession.Stop()
-		s.Require().NoError(err)
-	}()
-
-	browser, err := StartRod()
-	s.Require().NoError(err)
-	s.RodSession = browser
-
-	s.Page = s.doCreateTab(s.T(), HomeBaseURL)
-	s.verifyIsHome(s.T(), s.Page)
-	secret := s.doRegisterThenLogout(s.T(), s.Context(ctx), "john", "password")
-
-	s.doLoginTwoFactor(s.T(), s.Context(ctx), "john", "password", false, secret, "")
-
-	s.doVisit(s.T(), s.Context(ctx), fmt.Sprintf("%s/secret.html", SecureBaseURL))
-	s.verifySecretAuthorized(s.T(), s.Context(ctx))
-
-	err = traefik2DockerEnvironment.Restart("redis")
-	s.Require().NoError(err)
-
-	s.doVisit(s.T(), s.Context(ctx), fmt.Sprintf("%s/secret.html", SecureBaseURL))
-	s.verifySecretAuthorized(s.T(), s.Context(ctx))
+func (s *Traefik2Suite) TestResetPasswordScenario() {
+	suite.Run(s.T(), NewResetPasswordScenario())
 }
 
 func TestTraefik2Suite(t *testing.T) {