[MISC] Add CLI suite (#1597)

This change adds a new integration testing suite "CLI".

The intent of this suite is to test, validate and capture coverage for Authelia's commands via the CLI.

.buildkite/hooks/post-command

@@ -15,7 +15,7 @@ if [[ ! $BUILDKITE_BRANCH =~ ^(v.*) ]] && [[ $BUILDKITE_COMMAND_EXIT_STATUS == 0
     if [[ $BUILDKITE_AGENT_META_DATA_CODECOV == "verbose" ]]; then
       BUILDKITE_AGENT_META_DATA_CODECOV="-v"
     fi
-    bash <(curl -s --connect-timeout 10 --retry 10 --retry-max-time 0 https://codecov.io/bash) -Z -c -f "coverage.txt" -F backend ${BUILDKITE_AGENT_META_DATA_CODECOV}
+    bash <(curl -s --connect-timeout 10 --retry 10 --retry-max-time 0 https://codecov.io/bash) -Z -c -s 'coverage*.txt' -F backend ${BUILDKITE_AGENT_META_DATA_CODECOV}
     if [[ $BUILDKITE_LABEL =~ ":selenium:" ]]; then
       cd web && yarn report
     fi

.gitignore

@@ -46,4 +46,5 @@ qemu-*-static
 
 public_html.gen.go
 
+authelia
 __debug_bin

internal/suites/CLI/configuration.yml

@@ -0,0 +1,42 @@
+###############################################################
+#                Authelia minimal configuration               #
+###############################################################
+
+port: 9091
+tls_cert: /config/ssl/cert.pem
+tls_key: /config/ssl/key.pem
+
+log_level: debug
+
+jwt_secret: unsecure_secret
+
+authentication_backend:
+  file:
+    path: /config/users.yml
+
+session:
+  secret: unsecure_session_secret
+  domain: example.com
+  expiration: 3600 # 1 hour
+  inactivity: 300 # 5 minutes
+  remember_me_duration: 1y
+
+storage:
+  local:
+    path: /config/db.sqlite
+
+access_control:
+  default_policy: bypass
+  rules:
+    - domain: "public.example.com"
+      policy: bypass
+    - domain: "admin.example.com"
+      policy: two_factor
+    - domain: "secure.example.com"
+      policy: two_factor
+    - domain: "singlefactor.example.com"
+      policy: one_factor
+
+notifier:
+  filesystem:
+    filename: /config/notification.txt

internal/suites/CLI/docker-compose.yml

@@ -0,0 +1,7 @@
+version: '3'
+services:
+  authelia-backend:
+    volumes:
+      - './CLI/configuration.yml:/config/configuration.yml:ro'
+      - './CLI/users.yml:/config/users.yml'
+      - './common/ssl:/config/ssl:ro'

internal/suites/CLI/users.yml

@@ -0,0 +1,33 @@
+###############################################################
+#                         Users Database                      #
+###############################################################
+
+# This file can be used if you do not have an LDAP set up.
+
+# List of users
+users:
+  john:
+    displayname: "John Doe"
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"
+    email: john.doe@authelia.com
+    groups:
+      - admins
+      - dev
+
+  harry:
+    displayname: "Harry Potter"
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"
+    email: harry.potter@authelia.com
+    groups: []
+
+  bob:
+    displayname: "Bob Dylan"
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"
+    email: bob.dylan@authelia.com
+    groups:
+      - dev
+
+  james:
+    displayname: "James Dean"
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"
+    email: james.dean@authelia.com

internal/suites/docker.go

@@ -55,11 +55,19 @@ func (de *DockerEnvironment) Restart(service string) error {
 	return de.createCommandWithStdout(fmt.Sprintf("restart %s", service)).Run()
 }
 
-// Down spawn a docker environment.
+// Down destroy a docker environment.
 func (de *DockerEnvironment) Down() error {
 	return de.createCommandWithStdout("down -v").Run()
 }
 
+// Exec execute a command within a given service of the environment.
+func (de *DockerEnvironment) Exec(service string, command []string) (string, error) {
+	cmd := de.createCommand(fmt.Sprintf("exec -T %s %s", service, strings.Join(command, " ")))
+	content, err := cmd.CombinedOutput()
+
+	return string(content), err
+}
+
 // Logs get logs of a given service of the environment.
 func (de *DockerEnvironment) Logs(service string, flags []string) (string, error) {
 	cmd := de.createCommand(fmt.Sprintf("logs %s %s", strings.Join(flags, " "), service))

internal/suites/environment.go

@@ -74,7 +74,7 @@ func waitUntilAutheliaIsReady(dockerEnvironment *DockerEnvironment, suite string
 		return err
 	}
 
-	if os.Getenv("CI") != stringTrue {
+	if os.Getenv("CI") != stringTrue && suite != "CLI" {
 		if err := waitUntilAutheliaFrontendIsReady(dockerEnvironment); err != nil {
 			return err
 		}

internal/suites/example/compose/authelia/Dockerfile.backend

@@ -12,6 +12,8 @@ RUN mkdir -p /config && chown dev:dev /config
 
 USER dev
 
+ENV PATH="/app:${PATH}"
+
 VOLUME /config
 
 EXPOSE 9091

internal/suites/example/compose/authelia/resources/run-backend-dev.sh

@@ -4,6 +4,6 @@ set -e
 
 while true;
 do
-    dlv --listen 0.0.0.0:2345 --headless=true --continue --accept-multiclient debug cmd/authelia/*.go -- --config /config/configuration.yml
+    dlv --listen 0.0.0.0:2345 --headless=true --output=./authelia --continue --accept-multiclient debug cmd/authelia/*.go -- --config /config/configuration.yml
     sleep 10
 done

internal/suites/suite_cli.go

@@ -0,0 +1,49 @@
+package suites
+
+import (
+	"fmt"
+	"time"
+)
+
+var cliSuiteName = "CLI"
+
+func init() {
+	dockerEnvironment := NewDockerEnvironment([]string{
+		"internal/suites/docker-compose.yml",
+		"internal/suites/CLI/docker-compose.yml",
+		"internal/suites/example/compose/authelia/docker-compose.backend.{}.yml",
+	})
+
+	setup := func(suitePath string) error {
+		if err := dockerEnvironment.Up(); err != nil {
+			return err
+		}
+
+		return waitUntilAutheliaIsReady(dockerEnvironment, cliSuiteName)
+	}
+
+	displayAutheliaLogs := func() error {
+		backendLogs, err := dockerEnvironment.Logs("authelia-backend", nil)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(backendLogs)
+
+		return nil
+	}
+
+	teardown := func(suitePath string) error {
+		err := dockerEnvironment.Down()
+		return err
+	}
+
+	GlobalRegistry.Register(cliSuiteName, Suite{
+		SetUp:           setup,
+		SetUpTimeout:    5 * time.Minute,
+		OnSetupTimeout:  displayAutheliaLogs,
+		TestTimeout:     2 * time.Minute,
+		TearDown:        teardown,
+		TearDownTimeout: 2 * time.Minute,
+	})
+}

internal/suites/suite_cli_test.go

@@ -0,0 +1,141 @@
+package suites
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type CLISuite struct {
+	*CommandSuite
+}
+
+func NewCLISuite() *CLISuite {
+	return &CLISuite{CommandSuite: new(CommandSuite)}
+}
+
+func (s *CLISuite) SetupSuite() {
+	dockerEnvironment := NewDockerEnvironment([]string{
+		"internal/suites/docker-compose.yml",
+		"internal/suites/CLI/docker-compose.yml",
+		"internal/suites/example/compose/authelia/docker-compose.backend.{}.yml",
+	})
+	s.DockerEnvironment = dockerEnvironment
+}
+
+func (s *CLISuite) SetupTest() {
+	testArg := ""
+	coverageArg := ""
+
+	if os.Getenv("CI") == stringTrue {
+		testArg = "-test.coverprofile=/authelia/coverage-$(date +%s).txt"
+		coverageArg = "COVERAGE"
+	}
+
+	s.testArg = testArg
+	s.coverageArg = coverageArg
+}
+
+func (s *CLISuite) TestShouldValidateConfig() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "validate-config", "/config/configuration.yml"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "Configuration parsed successfully without errors")
+}
+
+func (s *CLISuite) TestShouldFailValidateConfig() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "validate-config", "/config/invalid.yml"})
+	s.Assert().NotNil(err)
+	s.Assert().Contains(output, "Error Loading Configuration: stat /config/invalid.yml: no such file or directory")
+}
+
+func (s *CLISuite) TestShouldHashPasswordArgon2id() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "hash-password", "test", "-m", "32", "-s", "test1234"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "Password hash: $argon2id$v=19$m=32768,t=1,p=8")
+}
+
+func (s *CLISuite) TestShouldHashPasswordSHA512() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "hash-password", "test", "-z"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "Password hash: $6$rounds=50000")
+}
+
+func (s *CLISuite) TestShouldGenerateCertificateRSA() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "*.example.com", "--dir", "/tmp/"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "wrote /tmp/cert.pem")
+	s.Assert().Contains(output, "wrote /tmp/key.pem")
+}
+
+func (s *CLISuite) TestShouldGenerateCertificateRSAWithIPAddress() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "127.0.0.1", "--dir", "/tmp/"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "wrote /tmp/cert.pem")
+	s.Assert().Contains(output, "wrote /tmp/key.pem")
+}
+
+func (s *CLISuite) TestShouldGenerateCertificateRSAWithStartDate() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "*.example.com", "--dir", "/tmp/", "--start-date", "'Jan 1 15:04:05 2011'"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "wrote /tmp/cert.pem")
+	s.Assert().Contains(output, "wrote /tmp/key.pem")
+}
+
+func (s *CLISuite) TestShouldFailGenerateCertificateRSAWithStartDate() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "*.example.com", "--dir", "/tmp/", "--start-date", "Jan"})
+	s.Assert().NotNil(err)
+	s.Assert().Contains(output, "Failed to parse creation date: parsing time \"Jan\" as \"Jan 2 15:04:05 2006\": cannot parse \"\" as \"2\"")
+}
+
+func (s *CLISuite) TestShouldGenerateCertificateCA() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "*.example.com", "--dir", "/tmp/", "--ca"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "wrote /tmp/cert.pem")
+	s.Assert().Contains(output, "wrote /tmp/key.pem")
+}
+
+func (s *CLISuite) TestShouldGenerateCertificateEd25519() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "*.example.com", "--dir", "/tmp/", "--ed25519"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "wrote /tmp/cert.pem")
+	s.Assert().Contains(output, "wrote /tmp/key.pem")
+}
+
+func (s *CLISuite) TestShouldFailGenerateCertificateECDSA() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "*.example.com", "--dir", "/tmp/", "--ecdsa-curve", "invalid"})
+	s.Assert().NotNil(err)
+	s.Assert().Contains(output, "Unrecognized elliptic curve: \"invalid\"")
+}
+
+func (s *CLISuite) TestShouldGenerateCertificateECDSAP224() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "*.example.com", "--dir", "/tmp/", "--ecdsa-curve", "P224"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "wrote /tmp/cert.pem")
+	s.Assert().Contains(output, "wrote /tmp/key.pem")
+}
+
+func (s *CLISuite) TestShouldGenerateCertificateECDSAP256() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "*.example.com", "--dir", "/tmp/", "--ecdsa-curve", "P256"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "wrote /tmp/cert.pem")
+	s.Assert().Contains(output, "wrote /tmp/key.pem")
+}
+
+func (s *CLISuite) TestShouldGenerateCertificateECDSAP384() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "*.example.com", "--dir", "/tmp/", "--ecdsa-curve", "P384"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "wrote /tmp/cert.pem")
+	s.Assert().Contains(output, "wrote /tmp/key.pem")
+}
+
+func (s *CLISuite) TestShouldGenerateCertificateECDSAP521() {
+	output, err := s.Exec("authelia-backend", []string{"authelia", s.testArg, s.coverageArg, "certificates", "generate", "--host", "*.example.com", "--dir", "/tmp/", "--ecdsa-curve", "P521"})
+	s.Assert().Nil(err)
+	s.Assert().Contains(output, "wrote /tmp/cert.pem")
+	s.Assert().Contains(output, "wrote /tmp/key.pem")
+}
+
+func TestCLISuite(t *testing.T) {
+	suite.Run(t, NewCLISuite())
+}

internal/suites/suites.go

@@ -12,6 +12,16 @@ type SeleniumSuite struct {
 	*WebDriverSession
 }
 
+// CommandSuite is a command line interface suite.
+type CommandSuite struct {
+	suite.Suite
+
+	testArg     string //nolint:structcheck // TODO: Remove when bug fixed: https://github.com/golangci/golangci-lint/issues/537.
+	coverageArg string //nolint:structcheck // TODO: Remove when bug fixed: https://github.com/golangci/golangci-lint/issues/537.
+
+	*DockerEnvironment
+}
+
 // WebDriver return the webdriver of the suite.
 func (s *SeleniumSuite) WebDriver() selenium.WebDriver {
 	return s.WebDriverSession.WebDriver