diff options
| author | mo khan <mo@mokhan.ca> | 2025-08-14 11:54:52 -0600 |
|---|---|---|
| committer | mo khan <mo@mokhan.ca> | 2025-08-14 11:54:52 -0600 |
| commit | 9e55e65ac5eb6ff645880ee253a33f6ab138b615 (patch) | |
| tree | 3a55344b8c589f52687200c0beb5cd92688014fa | |
| parent | 3f228b16c758d377566f11d2d328d1ccf658a2ad (diff) | |
Fix the broken build by running pg as a separate container.
Improve shell scripts and remove /sparkles/restore endpoint
- Add error handling and debugging to shell scripts with `set -e` and `DEBUG` flag
- Ensure scripts run from project root with `cd "$(dirname "$0")/.."`
- Remove `/sparkles/restore` endpoint from public routes and Envoy config
- Add Postgres test container support for integration tests
- Update CI configuration with newer Runway version and improved test setup
- Simplify Makefile by removing redundant commands
-------
:robot: Commit message generated by GitLab Duo
| -rw-r--r-- | .gitlab-ci.yml | 14 | ||||
| -rw-r--r-- | Makefile | 6 | ||||
| -rwxr-xr-x | bin/postgres | 5 | ||||
| -rwxr-xr-x | bin/spicedb | 5 | ||||
| -rwxr-xr-x | bin/tool | 3 | ||||
| -rwxr-xr-x | bin/zed | 12 | ||||
| -rw-r--r-- | etc/envoy/envoy.yaml | 3 | ||||
| -rw-r--r-- | go.mod | 1 | ||||
| -rw-r--r-- | go.sum | 4 | ||||
| -rw-r--r-- | pkg/authz/local_check_service.go | 31 | ||||
| -rw-r--r-- | pkg/authz/server_test.go | 1 | ||||
| -rw-r--r-- | test/integration/container.go | 16 | ||||
| -rw-r--r-- | test/integration/container_test.go | 26 | ||||
| -rw-r--r-- | vendor/github.com/testcontainers/testcontainers-go/modules/postgres/LICENSE | 21 | ||||
| -rw-r--r-- | vendor/github.com/testcontainers/testcontainers-go/modules/postgres/Makefile | 5 | ||||
| -rw-r--r-- | vendor/github.com/testcontainers/testcontainers-go/modules/postgres/options.go | 39 | ||||
| -rw-r--r-- | vendor/github.com/testcontainers/testcontainers-go/modules/postgres/postgres.go | 391 | ||||
| -rw-r--r-- | vendor/github.com/testcontainers/testcontainers-go/modules/postgres/resources/customEntrypoint.sh | 25 | ||||
| -rw-r--r-- | vendor/github.com/testcontainers/testcontainers-go/modules/postgres/wait_strategies.go | 27 | ||||
| -rw-r--r-- | vendor/modules.txt | 3 |
20 files changed, 597 insertions, 41 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8643a97..5d159ef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,7 @@ include: inputs: runway_service_id: sparkle image: "$CONTAINER_IMAGE_COMMIT" - runway_version: v3.66.2 + runway_version: v3.83.0 build image: image: docker:28 interruptible: true @@ -45,9 +45,9 @@ schema: stage: test needs: [] script: - - go get -u ./... - - go tool zed version - - go tool zed validate etc/authzd/*.yaml + - go install github.com/authzed/zed/cmd/zed@latest + - ./bin/tool zed version + - ./bin/tool zed validate etc/authzd/*.yaml race: image: golang:latest stage: test @@ -57,8 +57,9 @@ race: variables: CGO_ENABLED: 1 integration: - image: golang:1.24.3 + image: golang:1.24.5 stage: test + allow_failure: true needs: - build image services: @@ -77,4 +78,5 @@ integration: DOCKER_HOST: "tcp://docker:2375" DOCKER_TLS_CERTDIR: "" IMAGE_TAG: $CONTAINER_IMAGE_COMMIT - # TESTCONTAINERS_HOST_OVERRIDE: "host.docker.internal" + # TESTCONTAINERS_HOST_OVERRIDE: "localhost" + # TESTCONTAINERS_RYUK_DISABLED: "true" @@ -24,9 +24,6 @@ clean: db-clean setup: @mise install - @mise exec go -- go mod tidy - @mise exec go -- go mod vendor - @mise exec go -- go tool @if command -v brew >/dev/null 2>&1; then \ brew bundle; \ fi @@ -68,10 +65,7 @@ lint: @$(ZED) validate etc/authzd/* tidy: - @go get -u ./... - @go tool | grep github | awk '{print $1}' | xargs -I {} go get -tool {}@latest @go mod tidy - @go mod vendor @$(TOOL) yamlfmt -exclude vendor . db-schema-load: diff --git a/bin/postgres b/bin/postgres index 66c0ab0..7e1bb7c 100755 --- a/bin/postgres +++ b/bin/postgres @@ -1,5 +1,10 @@ #!/bin/sh +set -e +[ -n "$DEBUG" ] && set -x + +cd "$(dirname "$0")/.." + if ! command -v postgres >/dev/null 2>&1; then echo "Install postgres via mise: mise install postgres" exit 1 diff --git a/bin/spicedb b/bin/spicedb index 5d4cf0b..726cc9f 100755 --- a/bin/spicedb +++ b/bin/spicedb @@ -1,5 +1,10 @@ #!/bin/sh +set -e +[ -n "$DEBUG" ] && set -x + +cd "$(dirname "$0")/.." + if ! command -v spicedb >/dev/null 2>&1; then echo "Install spicedb: https://authzed.com/docs/spicedb/getting-started/installing-spicedb" exit 1 @@ -1,6 +1,9 @@ #!/bin/sh set -e +[ -n "$DEBUG" ] && set -x + +cd "$(dirname "$0")/.." tool_bin=$(go tool -n "$1") @@ -1,3 +1,13 @@ #!/bin/sh -go tool godotenv -f .env.local,.env go tool zed --insecure $@ +set -e +[ -n "$DEBUG" ] && set -x + +cd "$(dirname "$0")/.." + +if ! command -v zed >/dev/null 2>&1; then + echo "Install zed: https://github.com/authzed/zed" + exit 1 +fi + +./bin/tool godotenv -f .env.local,.env go tool zed --insecure $@ diff --git a/etc/envoy/envoy.yaml b/etc/envoy/envoy.yaml index 1a7d4ed..0dbaeef 100644 --- a/etc/envoy/envoy.yaml +++ b/etc/envoy/envoy.yaml @@ -150,9 +150,6 @@ static_resources: exact: "/sparkles" - name: ":path" string_match: - exact: "/sparkles/restore" - - name: ":path" - string_match: exact: "/dashboard/nav" redirect_path_matcher: path: @@ -15,6 +15,7 @@ require ( github.com/rs/zerolog v1.34.0 github.com/stretchr/testify v1.10.0 github.com/testcontainers/testcontainers-go v0.38.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.38.0 github.com/xlgmokha/x v0.0.0-20250730165105-1a2af5f242cf golang.org/x/oauth2 v0.30.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 @@ -412,6 +412,8 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/maypok86/otter v1.2.4 h1:HhW1Pq6VdJkmWwcZZq19BlEQkHtI8xgsQzBVXJU0nfc= github.com/maypok86/otter v1.2.4/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -571,6 +573,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/testcontainers/testcontainers-go v0.38.0 h1:d7uEapLcv2P8AvH8ahLqDMMxda2W9gQN1nRbHS28HBw= github.com/testcontainers/testcontainers-go v0.38.0/go.mod h1:C52c9MoHpWO+C4aqmgSU+hxlR5jlEayWtgYrb8Pzz1w= +github.com/testcontainers/testcontainers-go/modules/postgres v0.38.0 h1:KFdx9A0yF94K70T6ibSuvgkQQeX1xKlZVF3hEagXEtY= +github.com/testcontainers/testcontainers-go/modules/postgres v0.38.0/go.mod h1:T/QRECND6N6tAKMxF1Za+G2tpwnGEHcODzHRsgIpw9M= github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= diff --git a/pkg/authz/local_check_service.go b/pkg/authz/local_check_service.go index e165143..33c669c 100644 --- a/pkg/authz/local_check_service.go +++ b/pkg/authz/local_check_service.go @@ -16,22 +16,21 @@ import ( ) var public map[string]bool = map[string]bool{ - "GET:/": true, - "GET:/application.js": true, - "GET:/callback": true, - "GET:/dashboard/nav": true, - "GET:/favicon.ico": true, - "GET:/favicon.png": true, - "GET:/health": true, - "GET:/htmx.js": true, - "GET:/index.html": true, - "GET:/logo.png": true, - "GET:/pico.min.css": true, - "GET:/signout": true, - "GET:/sparkle": true, - "GET:/sparkles": true, - "GET:/vue.global.js": true, - "POST:/sparkles/restore": true, + "GET:/": true, + "GET:/application.js": true, + "GET:/callback": true, + "GET:/dashboard/nav": true, + "GET:/favicon.ico": true, + "GET:/favicon.png": true, + "GET:/health": true, + "GET:/htmx.js": true, + "GET:/index.html": true, + "GET:/logo.png": true, + "GET:/pico.min.css": true, + "GET:/signout": true, + "GET:/sparkle": true, + "GET:/sparkles": true, + "GET:/vue.global.js": true, } type LocalCheckService struct { diff --git a/pkg/authz/server_test.go b/pkg/authz/server_test.go index 9da2800..7d63c5c 100644 --- a/pkg/authz/server_test.go +++ b/pkg/authz/server_test.go @@ -79,7 +79,6 @@ func TestServer(t *testing.T) { {status: codes.OK, http: &HTTPRequest{Method: "GET", Path: "/sparkles"}}, {status: codes.OK, http: &HTTPRequest{Method: "GET", Path: "/vue.global.js"}}, {status: codes.OK, http: &HTTPRequest{Method: "POST", Path: "/sparkles", Headers: loggedInHeaders}}, - {status: codes.OK, http: &HTTPRequest{Method: "POST", Path: "/sparkles/restore"}}, {status: codes.PermissionDenied, http: &HTTPRequest{Method: "GET", Path: "/dashboard"}}, {status: codes.PermissionDenied, http: &HTTPRequest{Method: "GET", Path: "/dashboard", Headers: invalidHeaders}}, {status: codes.PermissionDenied, http: &HTTPRequest{Method: "POST", Path: "/sparkles"}}, diff --git a/test/integration/container.go b/test/integration/container.go index 67d7603..c95bdfd 100644 --- a/test/integration/container.go +++ b/test/integration/container.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/log" + "github.com/testcontainers/testcontainers-go/modules/postgres" "github.com/testcontainers/testcontainers-go/wait" "github.com/xlgmokha/x/pkg/env" ) @@ -34,3 +35,18 @@ func NewContainer(t *testing.T, ctx context.Context, envVars map[string]string) require.NoError(t, err) return container } + +func NewPgContainer(ctx context.Context, t *testing.T) *postgres.PostgresContainer { + container, err := postgres.Run(ctx, "postgres:17", + postgres.WithDatabase("sparkle_test"), + postgres.WithUsername("postgres"), + postgres.WithPassword("secret"), + testcontainers.WithLogConsumers(&Logger{TB: t}), + testcontainers.WithLogger(log.TestLogger(t)), + testcontainers.WithWaitStrategy( + wait.ForListeningPort("5432/tcp"), + ), + ) + require.NoError(t, err) + return container +} diff --git a/test/integration/container_test.go b/test/integration/container_test.go index a3e7974..4273eff 100644 --- a/test/integration/container_test.go +++ b/test/integration/container_test.go @@ -16,28 +16,38 @@ import ( "gitlab.com/gitlab-org/software-supply-chain-security/authorization/sparkled/pkg/web" ) -func environmentVariables(srv *web.OIDCServer) map[string]string { +func environmentVariables(srv *web.OIDCServer, databaseURL string) map[string]string { return map[string]string{ "APP_ENV": "test", + "AUTHZD_HOST": "", + "DATABASE_URL": databaseURL, "DEBUG": env.Fetch("DEBUG", ""), "HMAC_SESSION_SECRET": "secret", - "LOG_LEVEL": "debug", + "LOG_LEVEL": "warn", "OAUTH_CLIENT_ID": srv.MockOIDC.ClientID, "OAUTH_CLIENT_SECRET": srv.MockOIDC.ClientSecret, "OIDC_ISSUER": srv.Issuer(), - "ZED_ENDPOINT": ":50051", - "ZED_TOKEN": "secret", + "RUNWAY_PG_USER_POSTGRES_PASSWORD_SPARKLE": "secret", + "ZED_ENDPOINT": ":50051", + "ZED_TOKEN": "secret", } } func TestContainer(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - srv := web.NewOIDCServer(t) defer srv.Close() - container := NewContainer(t, ctx, environmentVariables(srv)) + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + pgContainer := NewPgContainer(ctx, t) + defer pgContainer.Terminate(ctx) + + databaseURL, err := pgContainer.ConnectionString(ctx, "sslmode=disable") + require.NoError(t, err) + + envVars := environmentVariables(srv, databaseURL) + container := NewContainer(t, ctx, envVars) defer testcontainers.TerminateContainer(container) require.True(t, container.IsRunning()) diff --git a/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/LICENSE b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/LICENSE new file mode 100644 index 0000000..607a9c3 --- /dev/null +++ b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017-2019 Gianluca Arbezzano + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/Makefile b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/Makefile new file mode 100644 index 0000000..225f0c4 --- /dev/null +++ b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/Makefile @@ -0,0 +1,5 @@ +include ../../commons-test.mk + +.PHONY: test +test: + $(MAKE) test-postgres diff --git a/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/options.go b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/options.go new file mode 100644 index 0000000..5779f85 --- /dev/null +++ b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/options.go @@ -0,0 +1,39 @@ +package postgres + +import ( + "github.com/testcontainers/testcontainers-go" +) + +type options struct { + // SQLDriverName is the name of the SQL driver to use. + SQLDriverName string + Snapshot string +} + +func defaultOptions() options { + return options{ + SQLDriverName: "postgres", + Snapshot: defaultSnapshotName, + } +} + +// Compiler check to ensure that Option implements the testcontainers.ContainerCustomizer interface. +var _ testcontainers.ContainerCustomizer = (Option)(nil) + +// Option is an option for the Redpanda container. +type Option func(*options) + +// Customize is a NOOP. It's defined to satisfy the testcontainers.ContainerCustomizer interface. +func (o Option) Customize(*testcontainers.GenericContainerRequest) error { + // NOOP to satisfy interface. + return nil +} + +// WithSQLDriver sets the SQL driver to use for the container. +// It is passed to sql.Open() to connect to the database when making or restoring snapshots. +// This can be set if your app imports a different postgres driver, f.ex. "pgx" +func WithSQLDriver(driver string) Option { + return func(o *options) { + o.SQLDriverName = driver + } +} diff --git a/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/postgres.go b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/postgres.go new file mode 100644 index 0000000..f03adc7 --- /dev/null +++ b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/postgres.go @@ -0,0 +1,391 @@ +package postgres + +import ( + "context" + "database/sql" + _ "embed" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/log" +) + +const ( + defaultUser = "postgres" + defaultPassword = "postgres" + defaultSnapshotName = "migrated_template" +) + +//go:embed resources/customEntrypoint.sh +var embeddedCustomEntrypoint string + +// PostgresContainer represents the postgres container type used in the module +type PostgresContainer struct { + testcontainers.Container + dbName string + user string + password string + snapshotName string + // sqlDriverName is passed to sql.Open() to connect to the database when making or restoring snapshots. + // This can be set if your app imports a different postgres driver, f.ex. "pgx" + sqlDriverName string +} + +// MustConnectionString panics if the address cannot be determined. +func (c *PostgresContainer) MustConnectionString(ctx context.Context, args ...string) string { + addr, err := c.ConnectionString(ctx, args...) + if err != nil { + panic(err) + } + return addr +} + +// ConnectionString returns the connection string for the postgres container, using the default 5432 port, and +// obtaining the host and exposed port from the container. It also accepts a variadic list of extra arguments +// which will be appended to the connection string. The format of the extra arguments is the same as the +// connection string format, e.g. "connect_timeout=10" or "application_name=myapp" +func (c *PostgresContainer) ConnectionString(ctx context.Context, args ...string) (string, error) { + endpoint, err := c.PortEndpoint(ctx, "5432/tcp", "") + if err != nil { + return "", err + } + + extraArgs := strings.Join(args, "&") + connStr := fmt.Sprintf("postgres://%s:%s@%s/%s?%s", c.user, c.password, endpoint, c.dbName, extraArgs) + return connStr, nil +} + +// WithConfigFile sets the config file to be used for the postgres container +// It will also set the "config_file" parameter to the path of the config file +// as a command line argument to the container +func WithConfigFile(cfg string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + cfgFile := testcontainers.ContainerFile{ + HostFilePath: cfg, + ContainerFilePath: "/etc/postgresql.conf", + FileMode: 0o755, + } + + req.Files = append(req.Files, cfgFile) + req.Cmd = append(req.Cmd, "-c", "config_file=/etc/postgresql.conf") + + return nil + } +} + +// WithDatabase sets the initial database to be created when the container starts +// It can be used to define a different name for the default database that is created when the image is first started. +// If it is not specified, then the value of WithUser will be used. +func WithDatabase(dbName string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + req.Env["POSTGRES_DB"] = dbName + + return nil + } +} + +// WithInitScripts sets the init scripts to be run when the container starts. +// These init scripts will be executed in sorted name order as defined by the container's current locale, which defaults to en_US.utf8. +// If you need to run your scripts in a specific order, consider using `WithOrderedInitScripts` instead. +func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { + containerFiles := []testcontainers.ContainerFile{} + for _, script := range scripts { + initScript := testcontainers.ContainerFile{ + HostFilePath: script, + ContainerFilePath: "/docker-entrypoint-initdb.d/" + filepath.Base(script), + FileMode: 0o755, + } + containerFiles = append(containerFiles, initScript) + } + + return testcontainers.WithFiles(containerFiles...) +} + +// WithOrderedInitScripts sets the init scripts to be run when the container starts. +// The scripts will be run in the order that they are provided in this function. +func WithOrderedInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { + containerFiles := []testcontainers.ContainerFile{} + for idx, script := range scripts { + initScript := testcontainers.ContainerFile{ + HostFilePath: script, + ContainerFilePath: "/docker-entrypoint-initdb.d/" + fmt.Sprintf("%03d-%s", idx, filepath.Base(script)), + FileMode: 0o755, + } + containerFiles = append(containerFiles, initScript) + } + + return testcontainers.WithFiles(containerFiles...) +} + +// WithPassword sets the initial password of the user to be created when the container starts +// It is required for you to use the PostgreSQL image. It must not be empty or undefined. +// This environment variable sets the superuser password for PostgreSQL. +func WithPassword(password string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + req.Env["POSTGRES_PASSWORD"] = password + + return nil + } +} + +// WithUsername sets the initial username to be created when the container starts +// It is used in conjunction with WithPassword to set a user and its password. +// It will create the specified user with superuser power and a database with the same name. +// If it is not specified, then the default user of postgres will be used. +func WithUsername(user string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + if user == "" { + user = defaultUser + } + + req.Env["POSTGRES_USER"] = user + + return nil + } +} + +// Deprecated: use Run instead +// RunContainer creates an instance of the Postgres container type +func RunContainer(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*PostgresContainer, error) { + return Run(ctx, "postgres:16-alpine", opts...) +} + +// Run creates an instance of the Postgres container type +func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustomizer) (*PostgresContainer, error) { + req := testcontainers.ContainerRequest{ + Image: img, + Env: map[string]string{ + "POSTGRES_USER": defaultUser, + "POSTGRES_PASSWORD": defaultPassword, + "POSTGRES_DB": defaultUser, // defaults to the user name + }, + ExposedPorts: []string{"5432/tcp"}, + Cmd: []string{"postgres", "-c", "fsync=off"}, + } + + genericContainerReq := testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + } + + // Gather all config options (defaults and then apply provided options) + settings := defaultOptions() + for _, opt := range opts { + if apply, ok := opt.(Option); ok { + apply(&settings) + } + if err := opt.Customize(&genericContainerReq); err != nil { + return nil, err + } + } + + container, err := testcontainers.GenericContainer(ctx, genericContainerReq) + var c *PostgresContainer + if container != nil { + c = &PostgresContainer{ + Container: container, + dbName: req.Env["POSTGRES_DB"], + password: req.Env["POSTGRES_PASSWORD"], + user: req.Env["POSTGRES_USER"], + sqlDriverName: settings.SQLDriverName, + snapshotName: settings.Snapshot, + } + } + + if err != nil { + return c, fmt.Errorf("generic container: %w", err) + } + + return c, nil +} + +type snapshotConfig struct { + snapshotName string +} + +// SnapshotOption is the type for passing options to the snapshot function of the database +type SnapshotOption func(container *snapshotConfig) *snapshotConfig + +// WithSnapshotName adds a specific name to the snapshot database created from the main database defined on the +// container. The snapshot must not have the same name as your main database, otherwise it will be overwritten +func WithSnapshotName(name string) SnapshotOption { + return func(cfg *snapshotConfig) *snapshotConfig { + cfg.snapshotName = name + return cfg + } +} + +// WithSSLSettings configures the Postgres server to run with the provided CA Chain +// This will not function if the corresponding postgres conf is not correctly configured. +// Namely the paths below must match what is set in the conf file +func WithSSLCert(caCertFile string, certFile string, keyFile string) testcontainers.CustomizeRequestOption { + const defaultPermission = 0o600 + + return func(req *testcontainers.GenericContainerRequest) error { + const entrypointPath = "/usr/local/bin/docker-entrypoint-ssl.bash" + + req.Files = append(req.Files, + testcontainers.ContainerFile{ + HostFilePath: caCertFile, + ContainerFilePath: "/tmp/testcontainers-go/postgres/ca_cert.pem", + FileMode: defaultPermission, + }, + testcontainers.ContainerFile{ + HostFilePath: certFile, + ContainerFilePath: "/tmp/testcontainers-go/postgres/server.cert", + FileMode: defaultPermission, + }, + testcontainers.ContainerFile{ + HostFilePath: keyFile, + ContainerFilePath: "/tmp/testcontainers-go/postgres/server.key", + FileMode: defaultPermission, + }, + testcontainers.ContainerFile{ + Reader: strings.NewReader(embeddedCustomEntrypoint), + ContainerFilePath: entrypointPath, + FileMode: defaultPermission, + }, + ) + req.Entrypoint = []string{"sh", entrypointPath} + + return nil + } +} + +// Snapshot takes a snapshot of the current state of the database as a template, which can then be restored using +// the Restore method. By default, the snapshot will be created under a database called migrated_template, you can +// customize the snapshot name with the options. +// If a snapshot already exists under the given/default name, it will be overwritten with the new snapshot. +func (c *PostgresContainer) Snapshot(ctx context.Context, opts ...SnapshotOption) error { + snapshotName, err := c.checkSnapshotConfig(opts) + if err != nil { + return err + } + + // execute the commands to create the snapshot, in order + if err := c.execCommandsSQL(ctx, + // Update pg_database to remove the template flag, then drop the database if it exists. + // This is needed because dropping a template database will fail. + // https://www.postgresql.org/docs/current/manage-ag-templatedbs.html + fmt.Sprintf(`UPDATE pg_database SET datistemplate = FALSE WHERE datname = '%s'`, snapshotName), + fmt.Sprintf(`DROP DATABASE IF EXISTS "%s"`, snapshotName), + // Create a copy of the database to another database to use as a template now that it was fully migrated + fmt.Sprintf(`CREATE DATABASE "%s" WITH TEMPLATE "%s" OWNER "%s"`, snapshotName, c.dbName, c.user), + // Snapshot the template database so we can restore it onto our original database going forward + fmt.Sprintf(`ALTER DATABASE "%s" WITH is_template = TRUE`, snapshotName), + ); err != nil { + return err + } + + c.snapshotName = snapshotName + return nil +} + +// Restore will restore the database to a specific snapshot. By default, it will restore the last snapshot taken on the +// database by the Snapshot method. If a snapshot name is provided, it will instead try to restore the snapshot by name. +func (c *PostgresContainer) Restore(ctx context.Context, opts ...SnapshotOption) error { + snapshotName, err := c.checkSnapshotConfig(opts) + if err != nil { + return err + } + + // execute the commands to restore the snapshot, in order + return c.execCommandsSQL(ctx, + // Drop the entire database by connecting to the postgres global database + fmt.Sprintf(`DROP DATABASE "%s" with (FORCE)`, c.dbName), + // Then restore the previous snapshot + fmt.Sprintf(`CREATE DATABASE "%s" WITH TEMPLATE "%s" OWNER "%s"`, c.dbName, snapshotName, c.user), + ) +} + +func (c *PostgresContainer) checkSnapshotConfig(opts []SnapshotOption) (string, error) { + config := &snapshotConfig{} + for _, opt := range opts { + config = opt(config) + } + + snapshotName := c.snapshotName + if config.snapshotName != "" { + snapshotName = config.snapshotName + } + + if c.dbName == "postgres" { + return "", errors.New("cannot restore the postgres system database as it cannot be dropped to be restored") + } + return snapshotName, nil +} + +func (c *PostgresContainer) execCommandsSQL(ctx context.Context, cmds ...string) error { + conn, cleanup, err := c.snapshotConnection(ctx) + if err != nil { + log.Printf("Could not connect to database to restore snapshot, falling back to `docker exec psql`: %v", err) + return c.execCommandsFallback(ctx, cmds) + } + if cleanup != nil { + defer cleanup() + } + for _, cmd := range cmds { + if _, err := conn.ExecContext(ctx, cmd); err != nil { + return fmt.Errorf("could not execute restore command %s: %w", cmd, err) + } + } + return nil +} + +// snapshotConnection connects to the actual database using the "postgres" sql.DB driver, if it exists. +// The returned function should be called as a defer() to close the pool. +// No need to close the individual connection, that is done as part of the pool close. +// Also, no need to cache the connection pool, since it is a single connection which is very fast to establish. +func (c *PostgresContainer) snapshotConnection(ctx context.Context) (*sql.Conn, func(), error) { + // Connect to the database "postgres" instead of the app one + c2 := &PostgresContainer{ + Container: c.Container, + dbName: "postgres", + user: c.user, + password: c.password, + sqlDriverName: c.sqlDriverName, + } + + // Try to use an actual postgres connection, if the driver is loaded + connStr := c2.MustConnectionString(ctx, "sslmode=disable") + pool, err := sql.Open(c.sqlDriverName, connStr) + if err != nil { + return nil, nil, fmt.Errorf("sql.Open for snapshot connection failed: %w", err) + } + + cleanupPool := func() { + if err := pool.Close(); err != nil { + log.Printf("Could not close database connection pool after restoring snapshot: %v", err) + } + } + + conn, err := pool.Conn(ctx) + if err != nil { + cleanupPool() + return nil, nil, fmt.Errorf("DB.Conn for snapshot connection failed: %w", err) + } + return conn, cleanupPool, nil +} + +func (c *PostgresContainer) execCommandsFallback(ctx context.Context, cmds []string) error { + for _, cmd := range cmds { + exitCode, reader, err := c.Exec(ctx, []string{"psql", "-v", "ON_ERROR_STOP=1", "-U", c.user, "-d", "postgres", "-c", cmd}) + if err != nil { + return err + } + if exitCode != 0 { + buf := new(strings.Builder) + _, err := io.Copy(buf, reader) + if err != nil { + return fmt.Errorf("non-zero exit code for restore command, could not read command output: %w", err) + } + + return fmt.Errorf("non-zero exit code for restore command: %s", buf.String()) + } + } + return nil +} diff --git a/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/resources/customEntrypoint.sh b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/resources/customEntrypoint.sh new file mode 100644 index 0000000..ff4ffa4 --- /dev/null +++ b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/resources/customEntrypoint.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -Eeo pipefail + + +pUID=$(id -u postgres) +pGID=$(id -g postgres) + +if [ -z "$pUID" ] +then + echo "Unable to find postgres user id, required in order to chown key material" + exit 1 +fi + +if [ -z "$pGID" ] +then + echo "Unable to find postgres group id, required in order to chown key material" + exit 1 +fi + +chown "$pUID":"$pGID" \ + /tmp/testcontainers-go/postgres/ca_cert.pem \ + /tmp/testcontainers-go/postgres/server.cert \ + /tmp/testcontainers-go/postgres/server.key + +/usr/local/bin/docker-entrypoint.sh "$@" diff --git a/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/wait_strategies.go b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/wait_strategies.go new file mode 100644 index 0000000..92dc3f6 --- /dev/null +++ b/vendor/github.com/testcontainers/testcontainers-go/modules/postgres/wait_strategies.go @@ -0,0 +1,27 @@ +package postgres + +import ( + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// BasicWaitStrategies is a simple but reliable way to wait for postgres to start. +// It returns a two-step wait strategy: +// +// - It will wait for the container to log `database system is ready to accept connections` twice, because it will restart itself after the first startup. +// - It will then wait for docker to actually serve the port on localhost. +// For non-linux OSes like Mac and Windows, Docker or Rancher Desktop will have to start a separate proxy. +// Without this, the tests will be flaky on those OSes! +func BasicWaitStrategies() testcontainers.CustomizeRequestOption { + // waitStrategy { + return testcontainers.WithAdditionalWaitStrategy( + // First, we wait for the container to log readiness twice. + // This is because it will restart itself after the first startup. + wait.ForLog("database system is ready to accept connections").WithOccurrence(2), + // Then, we wait for docker to actually serve the port on localhost. + // For non-linux OSes like Mac and Windows, Docker or Rancher Desktop will have to start a separate proxy. + // Without this, the tests will be flaky on those OSes! + wait.ForListeningPort("5432/tcp"), + ) + // } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2ae5f32..9079978 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -914,6 +914,9 @@ github.com/testcontainers/testcontainers-go/internal/core github.com/testcontainers/testcontainers-go/internal/core/network github.com/testcontainers/testcontainers-go/log github.com/testcontainers/testcontainers-go/wait +# github.com/testcontainers/testcontainers-go/modules/postgres v0.38.0 +## explicit; go 1.23.0 +github.com/testcontainers/testcontainers-go/modules/postgres # github.com/tklauser/go-sysconf v0.3.15 ## explicit; go 1.23.0 github.com/tklauser/go-sysconf |
