diff --git a/.github/workflows/e2e-tests-kind.yml b/.github/workflows/e2e-tests-kind.yml new file mode 100644 index 0000000..42a4aef --- /dev/null +++ b/.github/workflows/e2e-tests-kind.yml @@ -0,0 +1,61 @@ +name: E2E Tests (kind) + +on: + workflow_call: + inputs: + image: + required: true + type: string + +env: + REGISTRY: quay.io + +jobs: + e2e-tests-kind: + runs-on: ubuntu-latest + env: + SKIP_OLM_TESTS: "true" + ROXIE_RESOURCE_PROFILE: "tiny" + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Log in to Quay.io + uses: docker/login-action@v4.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Extract roxie binary from image + run: | + docker create --name roxie-extract "${{ inputs.image }}" + docker cp roxie-extract:/usr/local/bin/roxie "$GITHUB_WORKSPACE/roxie" + docker rm roxie-extract + + - name: Install roxie binary + run: | + cp "${GITHUB_WORKSPACE}/roxie" /usr/local/bin/roxie + chmod +x /usr/local/bin/roxie + roxie version + + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + cluster_name: roxie-e2e + + - name: Run e2e tests + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: | + make run-test-e2e diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 4a08664..4f4b0bd 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -31,6 +31,7 @@ jobs: KUBECONFIG: /github/home/artifacts/kubeconfig INFRA_TOKEN: ${{ secrets.INFRA_CI_TOKEN }} INFRACTL: bin/infractl -k -e localhost:8443 + ROXIE_RESOURCE_PROFILE: "small" steps: - name: Checkout uses: actions/checkout@v7 diff --git a/.github/workflows/main-push.yml b/.github/workflows/main-push.yml index 32b0322..9beecba 100644 --- a/.github/workflows/main-push.yml +++ b/.github/workflows/main-push.yml @@ -35,6 +35,13 @@ jobs: image: ${{ needs.build-roxie-image.outputs.image }} secrets: inherit + e2e-tests-kind: + needs: [ build-roxie-image ] + uses: ./.github/workflows/e2e-tests-kind.yml + with: + image: ${{ needs.build-roxie-image.outputs.image }} + secrets: inherit + delete-dev-cluster: if: ${{ always() && needs.create-dev-cluster.result == 'success' }} needs: [ create-dev-cluster, e2e-tests ] diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5c542cb..8a0a2e1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -77,6 +77,14 @@ jobs: skip-olm-tests: 'false' secrets: inherit + e2e-tests-kind: + if: github.event.pull_request.head.repo.full_name == github.repository + needs: [ build-roxie-image ] + uses: ./.github/workflows/e2e-tests-kind.yml + with: + image: ${{ needs.build-roxie-image.outputs.image }} + secrets: inherit + delete-gke-cluster: if: ${{ always() && needs.create-gke-cluster.result == 'success' && github.event.pull_request.head.repo.full_name == github.repository }} needs: [ create-gke-cluster, e2e-tests ] diff --git a/internal/clusterdefaults/clusterdefaults.go b/internal/clusterdefaults/clusterdefaults.go index 3d40388..2bb704d 100644 --- a/internal/clusterdefaults/clusterdefaults.go +++ b/internal/clusterdefaults/clusterdefaults.go @@ -30,7 +30,7 @@ func ApplyClusterDefaults( return nil, fmt.Errorf("deep-copying cluster defaults for cluster type %s: %w", clusterType, err) } - if err := mergo.Merge(config, defaultsCopy, mergo.WithoutDereference); err != nil { + if err := mergo.Merge(config, defaultsCopy, mergo.WithoutDereference, mergo.WithAppendSlice); err != nil { return nil, fmt.Errorf("merging-in cluster defaults for cluster type %s: %w", clusterType, err) } diff --git a/internal/deployer/config.go b/internal/deployer/config.go index a7f0f9e..37c2f77 100644 --- a/internal/deployer/config.go +++ b/internal/deployer/config.go @@ -361,7 +361,11 @@ func (c *CentralConfig) CustomResource() (map[string]interface{}, error) { return nil, fmt.Errorf("resource profile 'auto' should have been resolved before building the CR") } if c.ResourceProfile != types.ResourceProfileAcsDefaults { - if err := helpers.DeepMerge(cr, getCentralResourcesOperator(c.ResourceProfile)); err != nil { + profileCopy, err := helpers.MergeMaps(getCentralResourcesOperator(c.ResourceProfile)) + if err != nil { + return nil, fmt.Errorf("copying resource profile: %w", err) + } + if err := helpers.DeepMerge(cr, profileCopy); err != nil { return nil, fmt.Errorf("merging resource profile into Central CR: %w", err) } } @@ -462,7 +466,11 @@ func (s *SecuredClusterConfig) CustomResource() (map[string]interface{}, error) return nil, fmt.Errorf("resource profile 'auto' must have been resolved before building the CR") } if s.ResourceProfile != types.ResourceProfileAcsDefaults { - if err := helpers.DeepMerge(cr, getSecuredClusterResourcesOperator(s.ResourceProfile)); err != nil { + profileCopy, err := helpers.MergeMaps(getSecuredClusterResourcesOperator(s.ResourceProfile)) + if err != nil { + return nil, fmt.Errorf("copying resource profile: %w", err) + } + if err := helpers.DeepMerge(cr, profileCopy); err != nil { return nil, fmt.Errorf("merging resource profile into SecuredCluster CR: %w", err) } } diff --git a/internal/deployer/constants.go b/internal/deployer/constants.go index 3152d35..bda15c8 100644 --- a/internal/deployer/constants.go +++ b/internal/deployer/constants.go @@ -6,82 +6,138 @@ var ( centralCrName = "stackrox-central-services" securedClusterCrName = "stackrox-secured-cluster-services" + centralDbPVCSizeTiny = "10Gi" centralDbPVCSizeSmall = "30Gi" defaultHAProxyBindPort = 8080 + centralResourcesTiny = map[string]interface{}{ + "requests": map[string]any{ + "memory": "300Mi", + "cpu": "90m", + }, + "limits": map[string]any{ + "memory": "2Gi", + "cpu": "1", + }, + } + centralResourcesSmall = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "1Gi", "cpu": "500m", }, - "limits": map[string]string{ + "limits": map[string]any{ + "memory": "2Gi", + "cpu": "1", + }, + } + + centralDbResourcesTiny = map[string]interface{}{ + "requests": map[string]any{ + "memory": "400Mi", + "cpu": "80m", + }, + "limits": map[string]any{ "memory": "2Gi", "cpu": "1", }, } centralDbResourcesSmall = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "1Gi", "cpu": "500m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "2Gi", "cpu": "1", }, } centralScannerResourcesSmall = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "500Mi", "cpu": "500m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "2500Mi", "cpu": "2000m", }, } centralScannerDbResourcesSmall = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "512Mi", "cpu": "200m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "2Gi", "cpu": "1000m", }, } + centralScannerV4DbResourcesTiny = map[string]interface{}{ + "requests": map[string]any{ + "memory": "400Mi", + "cpu": "80m", + }, + "limits": map[string]any{ + "memory": "2000Mi", + "cpu": "1000m", + }, + } + centralScannerV4DbResourcesSmall = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "512Mi", "cpu": "400m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "2000Mi", "cpu": "1000m", }, } + centralScannerV4IndexerResourcesTiny = map[string]interface{}{ + "requests": map[string]any{ + "memory": "300Mi", + "cpu": "80m", + }, + "limits": map[string]any{ + "memory": "2Gi", + "cpu": "2000m", + }, + } + centralScannerV4IndexerResourcesSmall = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "512Mi", "cpu": "400m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "2Gi", "cpu": "2000m", }, } + centralScannerV4MatcherResourcesTiny = map[string]interface{}{ + "requests": map[string]any{ + "memory": "300Mi", + "cpu": "80m", + }, + "limits": map[string]any{ + "memory": "2Gi", + "cpu": "1000m", + }, + } + centralScannerV4MatcherResourcesSmall = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "512Mi", "cpu": "400m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "2Gi", "cpu": "1000m", }, @@ -89,12 +145,23 @@ var ( // Secured Cluster + securedClusterSensorResourcesTiny = map[string]interface{}{ + "requests": map[string]any{ + "memory": "300Mi", + "cpu": "80m", + }, + "limits": map[string]any{ + "memory": "2Gi", + "cpu": "1000m", + }, + } + securedClusterSensorResourcesSmall = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "500Mi", "cpu": "500m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "2Gi", "cpu": "1000m", }, @@ -103,88 +170,88 @@ var ( // Medium resources - midpoint between small and default (with reasonable rounding) centralResourcesMedium = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "2Gi", "cpu": "1000m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "5Gi", "cpu": "2500m", }, } centralDbResourcesMedium = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "4Gi", "cpu": "2000m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "9Gi", "cpu": "4500m", }, } centralScannerResourcesMedium = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "1Gi", "cpu": "750m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "3Gi", "cpu": "2000m", }, } centralScannerDbResourcesMedium = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "512Mi", "cpu": "200m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "3Gi", "cpu": "1500m", }, } centralScannerV4DbResourcesMedium = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "2Gi", "cpu": "700m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "5Gi", "cpu": "2500m", }, } centralScannerV4IndexerResourcesMedium = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "512Mi", "cpu": "1000m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "2500Mi", "cpu": "3000m", }, } centralScannerV4MatcherResourcesMedium = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "1Gi", "cpu": "450m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "2500Mi", "cpu": "1000m", }, } securedClusterSensorResourcesMedium = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "2Gi", "cpu": "1000m", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "5Gi", "cpu": "2500m", }, @@ -193,36 +260,36 @@ var ( // CI resources - based on deploy/common/ci-values.yaml from stackrox repo. centralDbResourcesCI = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "1Gi", "cpu": "1", }, - "limits": map[string]string{ + "limits": map[string]any{ "memory": "16Gi", "cpu": "8", }, } centralScannerV4DbResourcesCI = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "cpu": "500m", }, } centralScannerV4IndexerResourcesCI = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "cpu": "400m", }, } centralScannerV4MatcherResourcesCI = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "cpu": "400m", }, } securedClusterSensorResourcesCI = map[string]interface{}{ - "requests": map[string]string{ + "requests": map[string]any{ "memory": "500Mi", "cpu": "500m", }, diff --git a/internal/deployer/deploy_via_operator.go b/internal/deployer/deploy_via_operator.go index 59e3ae5..c607b2e 100644 --- a/internal/deployer/deploy_via_operator.go +++ b/internal/deployer/deploy_via_operator.go @@ -356,6 +356,38 @@ func (d *Deployer) createAdminPasswordSecret(ctx context.Context) error { func getCentralResourcesOperator(resourcesProfile types.ResourceProfile) map[string]interface{} { switch resourcesProfile { + case types.ResourceProfileTiny: + return map[string]interface{}{ + "spec": map[string]interface{}{ + "central": map[string]interface{}{ + "resources": centralResourcesTiny, + "db": map[string]interface{}{ + "resources": centralDbResourcesTiny, + "persistence": map[string]interface{}{ + "persistentVolumeClaim": map[string]interface{}{ + "size": centralDbPVCSizeTiny, + }, + }, + }, + }, + "scanner": map[string]interface{}{ + "scannerComponent": "Disabled", + }, + "scannerV4": map[string]interface{}{ + "db": map[string]interface{}{ + "resources": centralScannerV4DbResourcesTiny, + }, + "indexer": map[string]interface{}{ + "resources": centralScannerV4IndexerResourcesTiny, + "scaling": noScaling, + }, + "matcher": map[string]interface{}{ + "resources": centralScannerV4MatcherResourcesTiny, + "scaling": noScaling, + }, + }, + }, + } case types.ResourceProfileSmall: return map[string]interface{}{ "spec": map[string]interface{}{ @@ -870,9 +902,45 @@ func (d *Deployer) deploySecuredClusterOperator(ctx context.Context) error { func getSecuredClusterResourcesOperator(resourceProfile types.ResourceProfile) map[string]interface{} { switch resourceProfile { + case types.ResourceProfileTiny: + return map[string]interface{}{ + "spec": map[string]interface{}{ + "admissionControl": map[string]interface{}{ + "replicas": 1, + }, + "sensor": map[string]interface{}{ + "resources": securedClusterSensorResourcesTiny, + }, + "scanner": map[string]interface{}{ + "scannerComponent": "Disabled", + }, + "scannerV4": map[string]interface{}{ + "scannerComponent": "Disabled", + }, + // The CRS init container's resource requests are not exposed in the CR spec, + // so we use the operator's overlay mechanism to lower them for tiny clusters. + "overlays": []any{ + map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "sensor", + "optional": true, + "patches": []any{ + map[string]any{ + "path": "spec.template.spec.initContainers[name:crs].resources.requests", + "value": `{"cpu":"80m", "memory": "150Mi"}`, + }, + }, + }, + }, + }, + } case types.ResourceProfileSmall: return map[string]interface{}{ "spec": map[string]interface{}{ + "admissionControl": map[string]interface{}{ + "replicas": 1, + }, "sensor": map[string]interface{}{ "resources": securedClusterSensorResourcesSmall, }, diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index 19c71b2..09d6384 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -416,6 +416,7 @@ func (d *Deployer) prepareCredentials(ctx context.Context) error { func (d *Deployer) deployCentral(ctx context.Context) error { log.Infof("Deploying Central to namespace %s", d.config.Central.Namespace) + log.Infof("Central resource profile: %s", d.config.Central.ResourceProfile) if d.namespaceExists(d.config.Central.Namespace) { log.Info("Existing Central deployment found, tearing down...") if err := d.teardownCentral(ctx); err != nil { @@ -439,6 +440,7 @@ func (d *Deployer) deployCentral(ctx context.Context) error { func (d *Deployer) deploySecuredCluster(ctx context.Context) error { log.Infof("Deploying SecuredCluster to namespace %s", d.config.SecuredCluster.Namespace) + log.Infof("SecuredCluster resource profile: %s", d.config.SecuredCluster.ResourceProfile) if d.namespaceExists(d.config.SecuredCluster.Namespace) { log.Info("Existing SecuredCluster deployment found, tearing down...") if err := d.teardownSecuredCluster(ctx); err != nil { diff --git a/internal/helpers/helpers.go b/internal/helpers/helpers.go index 2a4602d..b56160b 100644 --- a/internal/helpers/helpers.go +++ b/internal/helpers/helpers.go @@ -5,9 +5,9 @@ import ( "fmt" "os" "os/exec" - "reflect" "strings" + "dario.cat/mergo" "gopkg.in/yaml.v3" ) @@ -107,29 +107,10 @@ func deepCopySlice(s []interface{}) []interface{} { return result } -// deepMerge recursively merges overlay into base -func DeepMerge(base, overlay map[string]interface{}) error { - for k, v := range overlay { - if IsNil(v) { - continue - } - if baseVal, ok := base[k]; ok { - // Both are maps - merge recursively - if baseMap, baseIsMap := baseVal.(map[string]interface{}); baseIsMap { - if overlayMap, overlayIsMap := v.(map[string]interface{}); overlayIsMap { - if err := DeepMerge(baseMap, overlayMap); err != nil { - return err - } - continue - } else { - return fmt.Errorf("incompatible types in maps to merge (map vs. %T)", v) - } - } - } - // Override with overlay value - base[k] = v - } - return nil +// DeepMerge recursively merges overlay into base. +// Slices are appended, with values from overlay appearing after values from base. +func DeepMerge(base, overlay map[string]any) error { + return mergo.Merge(&base, &overlay, mergo.WithOverride, mergo.WithAppendSlice) } func MapToStruct(m map[string]interface{}, out interface{}) error { @@ -140,18 +121,6 @@ func MapToStruct(m map[string]interface{}, out interface{}) error { return yaml.Unmarshal(bytes, out) } -// IsNil uses reflection to reliably check if the provided argument is a Nil pointer. -func IsNil(i interface{}) bool { - if i == nil { - return true - } - switch reflect.TypeOf(i).Kind() { - case reflect.Pointer, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice: - return reflect.ValueOf(i).IsNil() - } - return false -} - // ShellQuote wraps s in POSIX single quotes, safe for use in shell eval contexts. func ShellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" diff --git a/internal/helpers/helpers_test.go b/internal/helpers/helpers_test.go index 8ba6423..1cbc0bc 100644 --- a/internal/helpers/helpers_test.go +++ b/internal/helpers/helpers_test.go @@ -147,6 +147,49 @@ func TestLoadYAMLFileInvalidYAML(t *testing.T) { } } +func TestDeepMergeAppendsSlices(t *testing.T) { + base := map[string]interface{}{ + "spec": map[string]interface{}{ + "imagePullSecrets": []interface{}{ + map[string]interface{}{"name": "secret-a"}, + }, + "keep": "this", + }, + } + + overlay := map[string]interface{}{ + "spec": map[string]interface{}{ + "imagePullSecrets": []interface{}{ + map[string]interface{}{"name": "secret-b"}, + }, + }, + } + + err := DeepMerge(base, overlay) + require.NoError(t, err) + + spec := base["spec"].(map[string]interface{}) + require.Equal(t, "this", spec["keep"]) + + secrets := spec["imagePullSecrets"].([]interface{}) + require.Len(t, secrets, 2) + require.Equal(t, map[string]interface{}{"name": "secret-a"}, secrets[0]) + require.Equal(t, map[string]interface{}{"name": "secret-b"}, secrets[1]) +} + +func TestDeepMergeOverrideOnTypeMismatch(t *testing.T) { + base := map[string]interface{}{ + "items": []interface{}{"a"}, + } + overlay := map[string]interface{}{ + "items": "not-a-slice", + } + + err := DeepMerge(base, overlay) + require.NoError(t, err) + require.Equal(t, "not-a-slice", base["items"]) +} + func TestDeepCopy(t *testing.T) { original := map[string]interface{}{ "a": "value", diff --git a/internal/types/resources.go b/internal/types/resources.go index 657b2dd..8caa26f 100644 --- a/internal/types/resources.go +++ b/internal/types/resources.go @@ -11,6 +11,7 @@ type ResourceProfile int const ( ResourceProfileAcsDefaults ResourceProfile = iota ResourceProfileAuto + ResourceProfileTiny ResourceProfileSmall ResourceProfileMedium ResourceProfileCI @@ -20,6 +21,7 @@ var ( resourceProfileNames = map[ResourceProfile]string{ ResourceProfileAcsDefaults: "acs-defaults", ResourceProfileAuto: "auto", + ResourceProfileTiny: "tiny", ResourceProfileSmall: "small", ResourceProfileMedium: "medium", ResourceProfileCI: "ci", diff --git a/tests/e2e/helpers.go b/tests/e2e/helpers.go index 6ed694d..193943e 100644 --- a/tests/e2e/helpers.go +++ b/tests/e2e/helpers.go @@ -38,11 +38,19 @@ const ( ) var ( - commonDeployArgs = []string{"--resources=small", "--skip-user-config"} + commonDeployArgs = deployArgsFromEnv() roxieBinary = "roxie" ) +func deployArgsFromEnv() []string { + profile := os.Getenv("ROXIE_RESOURCE_PROFILE") + if profile == "" { + profile = "small" + } + return []string{"--resources=" + profile, "--skip-user-config"} +} + func registerClusterDumpAndCleanup(t *testing.T) { t.Helper() ensureCleanCluster(t)