diff --git a/.github/docker/local.sh b/.github/docker/local.sh index 83e0c576d5..48a0d4d00e 100755 --- a/.github/docker/local.sh +++ b/.github/docker/local.sh @@ -81,7 +81,6 @@ function run_before_migration() { function run_latest_images() { export CLOUDSERVER_IMAGE=$CLOUDSERVER_IMAGE_ORIGINAL export VAULT_IMAGE=$VAULT_IMAGE_ORIGINAL - export S3KMS=$kmsProvider # S3 export KMS_BACKEND=$([[ "$kmsProvider" == "aws" ]] && echo "aws") # vault only aws is supported cd tests/functional/sse-kms-migration @@ -110,14 +109,12 @@ function run_latest_images() { function run_migration() { echo ==== RUN MIGRATION ==== cd ../.. - export S3KMS=$kmsProvider # S3 export KMS_BACKEND=$([[ "$kmsProvider" == "aws" ]] && echo "aws") # vault only aws is supported yarn run ft_sse_migration | tee /tmp/artifacts/$JOB_NAME/migration.log } function run_after_migration() { sleep 1 - export S3KMS=$kmsProvider # S3 export KMS_BACKEND=$([[ "$kmsProvider" == "aws" ]] && echo "aws") # vault only aws is supported yarn run ft_sse_arn | tee /tmp/artifacts/$JOB_NAME/arnPrefix.log } diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 57a205233e..7894e8c1ac 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -860,7 +860,10 @@ jobs: PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }} MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} JOB_NAME: ${{ github.job }} - S3KMS: kmip + COMPOSE_FILE: docker-compose.yaml:docker-compose.sse.yaml + # Never started here, but compose needs it to load the sse-migration profile in cleanup + VAULT_IMAGE: ghcr.io/scality/vault:7.70.32 + # The kmip entry of the config has no transport, these complete it S3KMIP_PORT: 5696 S3KMIP_HOSTS: pykmip.local S3KMIP_COMPOUND_CREATE: false @@ -877,10 +880,24 @@ jobs: - name: Copy KMIP certs run: cp -r ./certs /tmp/ssl-kmip working-directory: .github/pykmip + - name: Merge config.json, kmip config and the KMSs next to it + run: | + jq -s ' + .[2].kms as $nonDefaultKms | (.[0] * .[1]) | .kms += $nonDefaultKms + ' \ + configs/base.json \ + configs/kmipFromEnv.json \ + configs/nonDefaultKmsEntries.json \ + > config.json + working-directory: tests/functional/sse-kms-migration - name: Setup CI services - run: docker compose --profile pykmip up -d + run: docker compose up -d --quiet-pull redis pykmip cloudserver-sse-before-migration working-directory: .github/docker - name: Run file KMIP tests + env: + # Functional tests need the running config to know the KMSs next to the default one + # absolute path to override default root config + S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json run: |- set -ex -o pipefail; # the data daemon binds after 8000 and nothing in startup waits for it @@ -891,7 +908,7 @@ jobs: - name: Cleanup and upload coverage uses: ./.github/actions/cleanup-and-coverage with: - profiles: pykmip + profiles: pykmip sse-migration codecov-token: ${{ secrets.CODECOV_TOKEN }} if: always() - name: Upload test results to Codecov @@ -922,7 +939,6 @@ jobs: PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }} JOB_NAME: ${{ github.job }} COMPOSE_FILE: docker-compose.yaml:docker-compose.sse.yaml - S3KMS: kmip steps: - name: Checkout uses: actions/checkout@v4 @@ -959,7 +975,6 @@ jobs: # yarn run does a cd into the test folder # absolute path to override default root config S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json - S3KMS: kmip S3_END_TO_END: true # to use the default credentials profile and not vault profile run: |- set -ex -o pipefail; @@ -1105,7 +1120,6 @@ jobs: docker compose up -d --quiet-pull ${{ matrix.kms.container }} vault-sse-migration cloudserver-sse-migration working-directory: .github/docker env: - S3KMS: ${{ matrix.kms.provider }} # S3 KMS_BACKEND: ${{ matrix.kms.provider == 'aws' && 'aws' || '' }} # vault only supports aws - name: Wait for services kms vault and s3 run: |- @@ -1126,7 +1140,6 @@ jobs: # KMS provider and sseMigration # absolute path to override default root config S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json - S3KMS: ${{ matrix.kms.provider }} run: yarn run ft_sse_migration | tee /tmp/artifacts/${{ env.JOB_NAME }}/migration.log - name: Run SSE arnPrefix tests shell: bash # for pipefail @@ -1135,7 +1148,6 @@ jobs: # KMS provider and sseMigration # absolute path to override default root config S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json - S3KMS: ${{ matrix.kms.provider }} run: yarn run ft_sse_arn | tee /tmp/artifacts/${{ env.JOB_NAME }}/arnPrefix.log - name: Print docker compose logs run: |- diff --git a/CLAUDE.md b/CLAUDE.md index a7317adecd..859d37f427 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -126,15 +126,26 @@ HA deployments. | `mem` (default) | - | In-memory accounts from `conf/authdata.json` | | `vault` | 8500 | External Vault IAM service (vaultd) | -### KMS Backends (`S3KMS`) - -| Backend | Description | -| ---------------- | ---------------------------- | -| `file` (default) | Local file-based key storage | -| `mem` | In-memory (testing only) | -| `kmip` | External KMIP server | -| `aws` | AWS KMS | -| `scality` | Scality KMS | +### KMS Backends (`kms` in `config.json`, or `S3KMS`) + +KMSs are configured as a `kms` array in `config.json`. Each entry has a +`protocol`, a unique `providerName`, the settings of that protocol, and exactly +one entry has `"default": true`. The KMS of the default entry is used. + +| Protocol | Description | +| --------- | ------------------------------------------------------ | +| `file` | Local file-based key storage (`providerName: scality`) | +| `kmip` | External KMIP server or cluster | +| `aws_kms` | AWS KMS | +| `scality` | SafeNet KeySecure | + +Without a `kms` array, `S3KMS` (or `S3BACKEND`) selects a KMS without +settings: `file` (default), `mem` (testing only) or `scality`, which uses the +`file` KMS unless the SafeNet client module `scality-kms` is installed. With a +`kms` array, `S3KMS` is ignored and `S3BACKEND` only sets the other backends. + +The `S3KMIP_*` variables apply to the default entry when it is a `kmip` entry: +its own settings override them, and it can take its host from `S3KMIP_HOSTS`. ### Path Configuration diff --git a/config.json b/config.json index 502b9915dc..bccf6e03c2 100644 --- a/config.json +++ b/config.json @@ -138,16 +138,6 @@ ], "defaultEncryptionKeyPerAccount": true, "kmsHideScalityArn": false, - "kmsAWS": { - "providerName": "aws", - "region": "us-east-1", - "endpoint": "http://127.0.0.1:8080", - "ak": "tbd", - "sk": "tbd" - }, - "kmip": { - "providerName": "thales" - }, "apiBodySizeLimits": { "multiObjectDelete": 2097152, "bucketPutPolicy": 20480 diff --git a/lib/Config.js b/lib/Config.js index 4481517473..5bfcda4a64 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -668,28 +668,43 @@ class Config extends EventEmitter { this._getLocationConfig(); const config = this._getConfig(); this._configureBackends(); + this._parseKms(config); this._sseMigration(config); } - _parseKmsAWS(config) { - if (!config.kmsAWS) { - return {}; - } + _parseKmsSafeNet(block) { + const kms = {}; + assert(typeof block.userName === 'string', 'bad config: userName of the SafeNet KMS entry must be a string'); + assert(typeof block.password === 'string', 'bad config: password of the SafeNet KMS entry must be a string'); + kms.providerName = block.providerName; + kms.userName = block.userName; + kms.password = block.password; + if (block.helperProgram !== undefined) { + assert(typeof block.helperProgram === 'string'); + kms.helperProgram = block.helperProgram; + } + if (block.propertiesFile !== undefined) { + assert(typeof block.propertiesFile === 'string'); + kms.propertiesFile = block.propertiesFile; + } + if (block.maxSessions !== undefined) { + assert(typeof block.maxSessions === 'number'); + kms.maxSessions = block.maxSessions; + } + return kms; + } + + _parseKmsAWS(block) { let kmsAWS = {}; - const { providerName, region, endpoint, ak, sk, tls, noAwsArn } = config.kmsAWS; + const { providerName, region, endpoint, ak, sk, tls, noAwsArn } = block; - assert(providerName, 'Configuration Error: providerName must be defined in kmsAWS'); - assert( - isValidProvider(providerName), - 'Configuration Error: kmsAWS.providerNamer must be lowercase alphanumeric only', - ); - assert(endpoint, 'Configuration Error: endpoint must be defined in kmsAWS'); - assert(ak, 'Configuration Error: ak must be defined in kmsAWS'); - assert(sk, 'Configuration Error: sk must be defined in kmsAWS'); + assert(endpoint, 'Configuration Error: endpoint must be defined in the aws_kms KMS entry'); + assert(ak, 'Configuration Error: ak must be defined in the aws_kms KMS entry'); + assert(sk, 'Configuration Error: sk must be defined in the aws_kms KMS entry'); assert( ['undefined', 'boolean'].some(type => type === typeof noAwsArn), - 'Configuration Error:: kmsAWS.noAwsArn must be a boolean or not set', + 'Configuration Error:: noAwsArn of the aws_kms KMS entry must be a boolean or not set', ); kmsAWS = { @@ -737,7 +752,13 @@ class Config extends EventEmitter { return kmsAWS; } - _parseKmipTransport(transportKmip) { + /** + * @param {object} transportKmip - a KMIP transport from the config + * @param {object} environment - `process.env` for the default kms entry, + * whose settings override the S3KMIP_* variables, `{}` for the others + * @returns {object} the transport passed to the KMIP client + */ + _parseKmipTransport(transportKmip, environment) { const transport = { /** Specify the request pipeline depth here. * If for some reason the server sends the replies @@ -747,16 +768,16 @@ class Config extends EventEmitter { * benefit to tune this value for performance improvement. * Note: 0 is not an appropriate value and will fall back to 1. */ - pipelineDepth: process.env.S3KMIP_PIPELINE_DEPTH || 8, + pipelineDepth: environment.S3KMIP_PIPELINE_DEPTH || 8, tls: { - port: process.env.S3KMIP_PORT || 5696, + port: environment.S3KMIP_PORT || 5696, // ignore multiple hosts via env // prefer array of transport in config file // for customization per host - host: process.env.S3KMIP_HOSTS || process.env.S3KMIP_HOST, - key: this._loadTlsFile(process.env.S3KMIP_KEY || undefined), - cert: this._loadTlsFile(process.env.S3KMIP_CERT || undefined), - ca: (process.env.S3KMIP_CA ? process.env.S3KMIP_CA.split(',') : []).map(ca => this._loadTlsFile(ca)), + host: environment.S3KMIP_HOSTS || environment.S3KMIP_HOST, + key: this._loadTlsFile(environment.S3KMIP_KEY || undefined), + cert: this._loadTlsFile(environment.S3KMIP_CERT || undefined), + ca: (environment.S3KMIP_CA ? environment.S3KMIP_CA.split(',') : []).map(ca => this._loadTlsFile(ca)), }, }; if (transportKmip.pipelineDepth) { @@ -788,11 +809,22 @@ class Config extends EventEmitter { transport.tls.ca = this._loadTlsFile(ca); } } + assert( + transport.tls.host, + 'bad config: KMIP transport must define tls.host, or take it from S3KMIP_HOSTS in the default kms entry', + ); return transport; } - _parseKmsKmip(config) { - this.kmip = { + /** + * @param {object} block - the settings of a kmip kms entry + * @param {boolean} applyS3KmipVariables - whether the S3KMIP_* variables + * apply, which is only for the default kms entry + * @returns {object} the settings passed to the KMIP client + */ + _parseKmsKmip(block, applyS3KmipVariables) { + const environment = applyS3KmipVariables ? process.env : {}; + const kmip = { client: { /** Enable this option if the KMIP Server supports * Create and Activate in one operation. @@ -801,53 +833,147 @@ class Config extends EventEmitter { * time for `now' instead of client specified activation date * which also targets the present instant. */ - compoundCreateActivate: process.env.S3KMIP_COMPOUND_CREATE === 'true' || false, + compoundCreateActivate: environment.S3KMIP_COMPOUND_CREATE === 'true' || false, /** Set the bucket name attribute name here if the KMIP * server supports storing custom attributes along * with the keys. */ - bucketNameAttributeName: process.env.S3KMIP_BUCKET_ATTRIBUTE_NAME || '', + bucketNameAttributeName: environment.S3KMIP_BUCKET_ATTRIBUTE_NAME || '', }, - transport: this._parseKmipTransport({}), retries: 0, + providerName: block.providerName, }; - if (config.kmip) { - assert(config.kmip.providerName, 'config.kmip.providerName must be defined'); - assert( - isValidProvider(config.kmip.providerName), - 'config.kmip.providerName must be lowercase alphanumeric only', - ); - this.kmip.providerName = config.kmip.providerName; - if (config.kmip.client) { - if (config.kmip.client.compoundCreateActivate) { - assert(typeof config.kmip.client.compoundCreateActivate === 'boolean'); - this.kmip.client.compoundCreateActivate = config.kmip.client.compoundCreateActivate; - } - if (config.kmip.client.bucketNameAttributeName) { - assert(typeof config.kmip.client.bucketNameAttributeName === 'string'); - this.kmip.client.bucketNameAttributeName = config.kmip.client.bucketNameAttributeName; - } + if (block.client) { + if (block.client.compoundCreateActivate) { + assert(typeof block.client.compoundCreateActivate === 'boolean'); + kmip.client.compoundCreateActivate = block.client.compoundCreateActivate; } - if (config.kmip.transport) { - if (Array.isArray(config.kmip.transport)) { - this.kmip.transport = config.kmip.transport.map(t => this._parseKmipTransport(t)); - if (config.kmip.retries) { - assert( - typeof config.kmip.retries === 'number', - 'bad config: KMIP Cluster retries must be a number', - ); - assert( - config.kmip.retries <= this.kmip.transport.length - 1, - 'bad config: KMIP Cluster retries must be lower or equal to the number of hosts - 1', - ); - } else { - this.kmip.retries = this.kmip.transport.length - 1; - } - } else { - this.kmip.transport = this._parseKmipTransport(config.kmip.transport); - } + if (block.client.bucketNameAttributeName) { + assert(typeof block.client.bucketNameAttributeName === 'string'); + kmip.client.bucketNameAttributeName = block.client.bucketNameAttributeName; } } + if (!block.transport) { + kmip.transport = this._parseKmipTransport({}, environment); + } else if (Array.isArray(block.transport)) { + kmip.transport = block.transport.map(t => this._parseKmipTransport(t, environment)); + if (block.retries) { + assert(typeof block.retries === 'number', 'bad config: KMIP Cluster retries must be a number'); + assert( + block.retries <= kmip.transport.length - 1, + 'bad config: KMIP Cluster retries must be lower or equal to the number of hosts - 1', + ); + } else { + kmip.retries = kmip.transport.length - 1; + } + } else { + kmip.transport = this._parseKmipTransport(block.transport, environment); + } + return kmip; + } + + /** + * Parse one entry of the `kms` array. + * @param {object} entry - `protocol`, `providerName`, `default` and the + * settings of that protocol + * @returns {object} `{ protocol, providerName, default, settings }`, with + * `settings` passed to the KMS client + */ + _parseKmsEntry(entry) { + assert( + entry !== null && typeof entry === 'object' && !Array.isArray(entry), + 'bad config: each kms entry must be an object', + ); + const { protocol, default: isDefault, ...block } = entry; + const parsers = { + [KmsProtocol.file]: () => ({}), + [KmsProtocol.kmip]: () => this._parseKmsKmip(block, isDefault === true), + [KmsProtocol.aws_kms]: () => this._parseKmsAWS(block), + [KmsProtocol.scality]: () => this._parseKmsSafeNet(block), + }; + assert( + Object.hasOwn(parsers, protocol), + `bad config: kms entry protocol must be one of ${Object.keys(parsers).join(', ')}, got "${protocol}"`, + ); + assert( + isDefault === undefined || typeof isDefault === 'boolean', + 'bad config: kms entry default must be a boolean', + ); + const { providerName } = block; + assert(providerName, `bad config: ${protocol} kms entry must define providerName`); + assert(isValidProvider(providerName), 'bad config: kms entry providerName must be lowercase alphanumeric only'); + // The file backend always builds keys under the "scality" provider + assert( + protocol !== KmsProtocol.file || providerName === 'scality', + 'bad config: file kms entry providerName must be "scality"', + ); + return { protocol, providerName, default: isDefault === true, settings: parsers[protocol]() }; + } + + /** + * The KMS selected by S3KMS or S3BACKEND, used when config.json has no + * `kms` array. The KMS gets no settings: `scality` uses the `file` KMS + * unless the SafeNet client module `scality-kms` is installed. + * @returns {object} a `kms` entry, as returned by _parseKmsEntry + */ + _kmsFromEnvironment() { + const protocols = { + mem: KmsProtocol.mem, + file: KmsProtocol.file, + cdmi: KmsProtocol.file, + scality: KmsProtocol.scality, + }; + const name = process.env.S3KMS || process.env.S3BACKEND || 'file'; + assert( + !['kmip', 'aws'].includes(name), + `bad config: S3KMS=${name} needs an entry with a providerName in the kms array`, + ); + assert( + Object.hasOwn(protocols, name), + `bad environment variable: S3KMS should be one of ${Object.keys(protocols).join('/')}`, + ); + return { protocol: protocols[name], default: true, settings: {} }; + } + + _parseKms(config) { + assert( + config.kmip === undefined && config.kmsAWS === undefined, + 'bad config: kmip and kmsAWS are replaced by entries of the kms array', + ); + this.kmsIgnoredEnvironment = []; + if (config.kms === undefined) { + this.kms = [this._kmsFromEnvironment()]; + return; + } + assert( + Array.isArray(config.kms), + 'bad config: kms must be an array of KMS entries, each with a protocol, a providerName and its settings', + ); + assert(config.kms.length > 0, 'bad config: kms must have at least one entry'); + const entries = config.kms.map(entry => this._parseKmsEntry(entry)); + const providerNames = new Set(); + entries.forEach(({ providerName }) => { + assert( + !providerNames.has(providerName), + `bad config: kms entry providerName "${providerName}" is not unique`, + ); + providerNames.add(providerName); + }); + const defaultCount = entries.filter(entry => entry.default).length; + assert( + defaultCount === 1, + `bad config: exactly one kms entry must have "default": true, found ${defaultCount}`, + ); + this.kms = entries; + // S3BACKEND still sets the other backends, only S3KMS has nothing left to do + this.kmsIgnoredEnvironment = ['S3KMS'].filter(name => process.env[name]); + } + + /** + * @returns {object} the `kms` entry marked as default + */ + getDefaultKms() { + return this.kms.find(entry => entry.default); } _getLocationConfig() { @@ -1599,35 +1725,6 @@ class Config extends EventEmitter { } } - this.kms = {}; - if (config.kms) { - assert(config.kms.providerName, 'config.kms.providerName must be provided'); - assert( - isValidProvider(config.kms.providerName), - 'config.kms.providerName must be lowercase alphanumeric only', - ); - assert(typeof config.kms.userName === 'string'); - assert(typeof config.kms.password === 'string'); - this.kms.providerName = config.kms.providerName; - this.kms.userName = config.kms.userName; - this.kms.password = config.kms.password; - if (config.kms.helperProgram !== undefined) { - assert(typeof config.kms.helperProgram === 'string'); - this.kms.helperProgram = config.kms.helperProgram; - } - if (config.kms.propertiesFile !== undefined) { - assert(typeof config.kms.propertiesFile === 'string'); - this.kms.propertiesFile = config.kms.propertiesFile; - } - if (config.kms.maxSessions !== undefined) { - assert(typeof config.kms.maxSessions === 'number'); - this.kms.maxSessions = config.kms.maxSessions; - } - } - - this._parseKmsKmip(config); - this.kmsAWS = this._parseKmsAWS(config); - const globalEncryptionEnabled = config.globalEncryptionEnabled; this.globalEncryptionEnabled = globalEncryptionEnabled || false; assert(typeof this.globalEncryptionEnabled === 'boolean', 'config.globalEncryptionEnabled must be a boolean'); @@ -1953,7 +2050,6 @@ class Config extends EventEmitter { let auth = 'mem'; let data = 'multiple'; let metadata = 'file'; - let kms = 'file'; let quota = 'none'; if (process.env.S3BACKEND) { const validBackends = ['mem', 'file', 'scality', 'cdmi']; @@ -1964,7 +2060,6 @@ class Config extends EventEmitter { auth = process.env.S3BACKEND; data = process.env.S3BACKEND; metadata = process.env.S3BACKEND; - kms = process.env.S3BACKEND; } if (process.env.S3VAULT) { auth = process.env.S3VAULT; @@ -2012,9 +2107,6 @@ class Config extends EventEmitter { if (process.env.S3METADATA) { metadata = process.env.S3METADATA; } - if (process.env.S3KMS) { - kms = process.env.S3KMS; - } if (process.env.S3QUOTA) { quota = process.env.S3QUOTA; } @@ -2022,7 +2114,6 @@ class Config extends EventEmitter { auth, data, metadata, - kms, quota, }; @@ -2080,11 +2171,10 @@ class Config extends EventEmitter { ); if (this.sseMigration.previousKeyType === KmsType.external) { - if ([KmsProtocol.file, KmsProtocol.mem].includes(this.backends.kms)) { + const defaultProtocol = this.getDefaultKms().protocol; + if ([KmsProtocol.file, KmsProtocol.mem].includes(defaultProtocol)) { assert.fail( - `sseMigration.previousKeyType "external" can't migrate to "internal" KMS provider ${ - this.backends.kms - }`, + `sseMigration.previousKeyType "external" can't migrate to "internal" KMS provider ${defaultProtocol}`, ); } // We'd have to compare protocol & providerName diff --git a/lib/kms/wrapper.js b/lib/kms/wrapper.js index dc7fd0cc72..fe4682f030 100644 --- a/lib/kms/wrapper.js +++ b/lib/kms/wrapper.js @@ -21,13 +21,13 @@ const { validateKeyDetail, } = require('@scality/arsenal/build/lib/network/KMSInterface'); -function getScalityKms() { +function getScalityKms(settings) { let scalityKMS; let scalityKMSImpl; try { const ScalityKMS = require('scality-kms'); - scalityKMS = new ScalityKMS(config.kms); + scalityKMS = new ScalityKMS(settings); scalityKMSImpl = 'scalityKms'; } catch (error) { logger.warn('scality kms unavailable. Using file kms backend unless mem specified.', { error }); @@ -38,56 +38,70 @@ function getScalityKms() { } const kmsFactory = { - mem: () => ({ client: inMemory, implName: 'memoryKms' }), - file: () => ({ client: file, implName: 'fileKms' }), - cdmi: () => ({ client: file, implName: 'fileKms' }), - scality: () => { - const { scalityKMS, scalityKMSImpl } = getScalityKms(); + [KmsProtocol.mem]: () => ({ client: inMemory, implName: 'memoryKms' }), + [KmsProtocol.file]: () => ({ client: file, implName: 'fileKms' }), + [KmsProtocol.scality]: settings => { + const { scalityKMS, scalityKMSImpl } = getScalityKms(settings); return { client: scalityKMS, implName: scalityKMSImpl }; }, - kmip: () => { - if (!config.kmip) { - throw new Error('KMIP KMS driver configuration is missing.'); - } - const client = Array.isArray(config.kmip.transport) - ? new KMIPClusterClient({ kmip: config.kmip, logger, retries: config.kmip.retries }) - : new KMIPClient({ kmip: config.kmip }); + [KmsProtocol.kmip]: settings => { + const client = Array.isArray(settings.transport) + ? new KMIPClusterClient({ kmip: settings, logger, retries: settings.retries }) + : new KMIPClient({ kmip: settings }); return { client, implName: 'kmip' }; }, - aws: () => { - if (!config.kmsAWS) { - throw new Error('AWS KMS driver configuration is missing.'); - } - return { client: new KmsAWSClient({ kmsAWS: config.kmsAWS }), implName: 'aws' }; - }, + [KmsProtocol.aws_kms]: settings => ({ client: new KmsAWSClient({ kmsAWS: settings }), implName: 'aws' }), }; -function getClient(kms) { - const impl = kmsFactory[kms]; +function getClient(protocol, settings) { + const impl = kmsFactory[protocol]; if (!impl) { - throw new Error(`KMS backend is not configured: ${kms}`); + throw new Error(`KMS backend is not configured: ${protocol}`); } - return impl(); + return impl(settings); +} + +function getClientIdentifier(backend) { + return `${backend.type}:${backend.protocol}:${backend.provider}`; +} + +/** + * Build a client for every entry of the `kms` config. + * @param {object[]} entries - `config.kms` + * @returns {object} `{ clientInstances, defaultInstance }`, with + * `clientInstances` keyed by `type:protocol:provider` + */ +function createClients(entries) { + const instances = {}; + let defaultInstance; + entries.forEach(entry => { + const instance = getClient(entry.protocol, entry.settings); + instances[getClientIdentifier(instance.client.backend)] = instance; + if (entry.default) { + defaultInstance = instance; + } + }); + return { clientInstances: instances, defaultInstance }; +} + +if (config.kmsIgnoredEnvironment.length > 0) { + logger.warn('KMS is selected by the kms config, ignoring environment variables', { + variables: config.kmsIgnoredEnvironment, + }); } /** * Note: non current instance from previous keys won't be healthchecked * `{ [`type:protocol:provider`]: clientDetails }` */ -const clientInstances = {}; +const { clientInstances, defaultInstance } = createClients(config.kms); -const { client, implName } = getClient(config.backends.kms); -const { type, protocol, provider } = client.backend; -const currentIdentifier = `${type}:${protocol}:${provider}`; -clientInstances[currentIdentifier] = { client, implName }; +const { client, implName } = defaultInstance; +const currentIdentifier = getClientIdentifier(client.backend); +// Only keys of the default KMS are accepted, keys naming the other KMSs are refused const availableBackends = [client.backend]; -const mapKmsProtocolToClient = { - [KmsProtocol.aws_kms]: 'aws', - // others already match -}; - let previousBackend; let previousIdentifier; @@ -98,14 +112,13 @@ if (config.sseMigration) { config.sseMigration.previousKeyProvider, ); availableBackends.push(previousBackend); - previousIdentifier = `${previousBackend.type}:${previousBackend.protocol}:${previousBackend.provider}`; + previousIdentifier = getClientIdentifier(previousBackend); // Pre instantiate previous backend as for now only internal backend (file) is supported // for future multiple external backend we should consider keeping open connection to // external backend, healthcheck and idle timeout (if migration is finished) // a config flag could help toggle this behavior - const previousKms = mapKmsProtocolToClient[previousBackend.protocol] || previousBackend.protocol; - const previousInstance = getClient(previousKms); + const previousInstance = getClient(previousBackend.protocol, {}); clientInstances[previousIdentifier] = previousInstance; } @@ -585,3 +598,4 @@ class KMS { } module.exports = KMS; +module.exports.createClients = createClients; diff --git a/tests/functional/kmip/serverside_encryption.js b/tests/functional/kmip/serverside_encryption.js index 355875479b..eda225775e 100644 --- a/tests/functional/kmip/serverside_encryption.js +++ b/tests/functional/kmip/serverside_encryption.js @@ -1,7 +1,16 @@ -const { S3Client, PutObjectCommand, CopyObjectCommand, CreateMultipartUploadCommand } = require('@aws-sdk/client-s3'); +const { + S3Client, + PutObjectCommand, + CopyObjectCommand, + CreateMultipartUploadCommand, + PutBucketEncryptionCommand, +} = require('@aws-sdk/client-s3'); const { v4: uuidv4 } = require('uuid'); +const { promisify } = require('util'); const config = require('../config.json'); const { auth } = require('@scality/arsenal'); +const { KmsType, isValidProtocol, makeScalityArnPrefix } = require('@scality/arsenal/build/lib/network/KMSInterface'); +const { config: serverConfig } = require('../../../lib/Config'); const http = require('http'); const https = require('https'); const assert = require('assert'); @@ -204,3 +213,65 @@ describe('KMIP backed server-side encryption', () => { ); }); }); + +describe('KMIP backed server-side encryption with other KMSs configured', () => { + const createBucket = promisify(_createBucket); + const nonDefaultKeyArns = serverConfig.kms + .filter(entry => !entry.default) + .map(({ protocol, providerName }) => { + const type = isValidProtocol(KmsType.internal, protocol) ? KmsType.internal : KmsType.external; + return `${makeScalityArnPrefix(type, protocol, providerName)}${'ab'.repeat(32)}`; + }); + let bucketName; + + function assertInvalidArgument(err) { + assert.strictEqual(err.name, 'InvalidArgument'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + return true; + } + + before(() => { + assert(nonDefaultKeyArns.length > 0, 'the server config must have KMSs next to the default one'); + }); + + beforeEach(async () => { + bucketName = uuidv4(); + await createBucket(bucketName, false); + }); + + nonDefaultKeyArns.forEach(keyArn => { + it(`should refuse PutObject with key ${keyArn}`, () => + assert.rejects( + s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: uuidv4(), + Body: 'I am the best content ever', + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: keyArn, + }), + ), + assertInvalidArgument, + )); + + it(`should refuse PutBucketEncryption with key ${keyArn}`, () => + assert.rejects( + s3.send( + new PutBucketEncryptionCommand({ + Bucket: bucketName, + ServerSideEncryptionConfiguration: { + Rules: [ + { + ApplyServerSideEncryptionByDefault: { + SSEAlgorithm: 'aws:kms', + KMSMasterKeyID: keyArn, + }, + }, + ], + }, + }), + ), + assertInvalidArgument, + )); + }); +}); diff --git a/tests/functional/sse-kms-migration/arnPrefix.js b/tests/functional/sse-kms-migration/arnPrefix.js index 60231be12a..4d02f3fb27 100644 --- a/tests/functional/sse-kms-migration/arnPrefix.js +++ b/tests/functional/sse-kms-migration/arnPrefix.js @@ -660,17 +660,18 @@ describe('KMS error', () => { 'does not exist in this region, or you are not allowed to access\\.', ), }; - if (helpers.config.backends.kms === 'kmip') { + const kmsProtocol = helpers.config.getDefaultKms().protocol; + if (kmsProtocol === 'kmip') { expected = expectedKMIP; - } else if (helpers.config.backends.kms === 'aws') { + } else if (kmsProtocol === 'aws_kms') { expected = expectedAWS; } else { - throw new Error(`Unsupported KMS backend: ${helpers.config.backends.kms}`); + throw new Error(`Unsupported KMS backend: ${kmsProtocol}`); } function assertKmsError(action, keyId) { return err => { - if (helpers.config.backends.kms === 'aws' && action === 'Decrypt') { + if (kmsProtocol === 'aws_kms' && action === 'Decrypt') { assert.strictEqual(err.name, expectedLocalKms.code); assert.match(err.message, expectedLocalKms.msg(action, keyId)); return true; diff --git a/tests/functional/sse-kms-migration/configs/aws.json b/tests/functional/sse-kms-migration/configs/aws.json index 913e031c99..699050e09f 100644 --- a/tests/functional/sse-kms-migration/configs/aws.json +++ b/tests/functional/sse-kms-migration/configs/aws.json @@ -1,10 +1,14 @@ { - "kmsAWS": { - "noAwsArn": true, - "providerName": "local", - "region": "us-east-1", - "endpoint": "http://0:8080", - "ak": "456", - "sk": "123" - } + "kms": [ + { + "protocol": "aws_kms", + "providerName": "local", + "default": true, + "noAwsArn": true, + "region": "us-east-1", + "endpoint": "http://0:8080", + "ak": "456", + "sk": "123" + } + ] } diff --git a/tests/functional/sse-kms-migration/configs/kmip-cluster.json b/tests/functional/sse-kms-migration/configs/kmip-cluster.json index 0723ab7260..61eb42d94b 100644 --- a/tests/functional/sse-kms-migration/configs/kmip-cluster.json +++ b/tests/functional/sse-kms-migration/configs/kmip-cluster.json @@ -1,50 +1,54 @@ { - "kmip": { - "providerName": "thales", - "client": { - "compoundCreateActivate": false - }, - "transport": [ - { - "pipelineDepth": 8, - "tls": { - "port": 5696, - "host": "1.pykmip.local", - "key": "/tmp/ssl-kmip/kmip-client-key.pem", - "cert": "/tmp/ssl-kmip/kmip-client-cert.pem", - "ca": "/tmp/ssl-kmip/kmip-ca.pem" - } - }, - { - "pipelineDepth": 8, - "tls": { - "port": 5696, - "host": "127.0.0.2", - "key": "/tmp/ssl-kmip/kmip-client-key.pem", - "cert": "/tmp/ssl-kmip/kmip-client-cert.pem", - "ca": "/tmp/ssl-kmip/kmip-ca.pem" - } - }, - { - "pipelineDepth": 8, - "tls": { - "port": 5696, - "host": "3.pykmip.local", - "key": "/tmp/ssl-kmip/kmip-client-key.pem", - "cert": "/tmp/ssl-kmip/kmip-client-cert.pem", - "ca": "/tmp/ssl-kmip/kmip-ca.pem" - } + "kms": [ + { + "protocol": "kmip", + "providerName": "thales", + "default": true, + "client": { + "compoundCreateActivate": false }, - { - "pipelineDepth": 8, - "tls": { - "port": 5696, - "host": "127.0.0.4", - "key": "/tmp/ssl-kmip/kmip-client-key.pem", - "cert": "/tmp/ssl-kmip/kmip-client-cert.pem", - "ca": "/tmp/ssl-kmip/kmip-ca.pem" + "transport": [ + { + "pipelineDepth": 8, + "tls": { + "port": 5696, + "host": "1.pykmip.local", + "key": "/tmp/ssl-kmip/kmip-client-key.pem", + "cert": "/tmp/ssl-kmip/kmip-client-cert.pem", + "ca": "/tmp/ssl-kmip/kmip-ca.pem" + } + }, + { + "pipelineDepth": 8, + "tls": { + "port": 5696, + "host": "127.0.0.2", + "key": "/tmp/ssl-kmip/kmip-client-key.pem", + "cert": "/tmp/ssl-kmip/kmip-client-cert.pem", + "ca": "/tmp/ssl-kmip/kmip-ca.pem" + } + }, + { + "pipelineDepth": 8, + "tls": { + "port": 5696, + "host": "3.pykmip.local", + "key": "/tmp/ssl-kmip/kmip-client-key.pem", + "cert": "/tmp/ssl-kmip/kmip-client-cert.pem", + "ca": "/tmp/ssl-kmip/kmip-ca.pem" + } + }, + { + "pipelineDepth": 8, + "tls": { + "port": 5696, + "host": "127.0.0.4", + "key": "/tmp/ssl-kmip/kmip-client-key.pem", + "cert": "/tmp/ssl-kmip/kmip-client-cert.pem", + "ca": "/tmp/ssl-kmip/kmip-ca.pem" + } } - } - ] - } + ] + } + ] } diff --git a/tests/functional/sse-kms-migration/configs/kmip.json b/tests/functional/sse-kms-migration/configs/kmip.json index ca76a95795..ab244e237e 100644 --- a/tests/functional/sse-kms-migration/configs/kmip.json +++ b/tests/functional/sse-kms-migration/configs/kmip.json @@ -1,18 +1,22 @@ { - "kmip": { - "providerName": "thales", - "client": { - "compoundCreateActivate": false - }, - "transport": { - "pipelineDepth": 8, - "tls": { - "port": 5696, - "host": "pykmip.local", - "key": "/tmp/ssl-kmip/kmip-client-key.pem", - "cert": "/tmp/ssl-kmip/kmip-client-cert.pem", - "ca": "/tmp/ssl-kmip/kmip-ca.pem" + "kms": [ + { + "protocol": "kmip", + "providerName": "thales", + "default": true, + "client": { + "compoundCreateActivate": false + }, + "transport": { + "pipelineDepth": 8, + "tls": { + "port": 5696, + "host": "pykmip.local", + "key": "/tmp/ssl-kmip/kmip-client-key.pem", + "cert": "/tmp/ssl-kmip/kmip-client-cert.pem", + "ca": "/tmp/ssl-kmip/kmip-ca.pem" + } } } - } + ] } diff --git a/tests/functional/sse-kms-migration/configs/kmipFromEnv.json b/tests/functional/sse-kms-migration/configs/kmipFromEnv.json new file mode 100644 index 0000000000..f190b0d53a --- /dev/null +++ b/tests/functional/sse-kms-migration/configs/kmipFromEnv.json @@ -0,0 +1,9 @@ +{ + "kms": [ + { + "protocol": "kmip", + "providerName": "thales", + "default": true + } + ] +} diff --git a/tests/functional/sse-kms-migration/configs/nonDefaultKmsEntries.json b/tests/functional/sse-kms-migration/configs/nonDefaultKmsEntries.json new file mode 100644 index 0000000000..ed7e6a8bab --- /dev/null +++ b/tests/functional/sse-kms-migration/configs/nonDefaultKmsEntries.json @@ -0,0 +1,16 @@ +{ + "kms": [ + { + "protocol": "aws_kms", + "providerName": "unused", + "region": "us-east-1", + "endpoint": "http://127.0.0.1:1", + "ak": "456", + "sk": "123" + }, + { + "protocol": "file", + "providerName": "scality" + } + ] +} diff --git a/tests/functional/sse-kms-migration/load.js b/tests/functional/sse-kms-migration/load.js index 786eb841e7..2e611dc86c 100644 --- a/tests/functional/sse-kms-migration/load.js +++ b/tests/functional/sse-kms-migration/load.js @@ -12,7 +12,7 @@ const BUCKET_NUMBER = 10; const OBJECT_NUMBER = 200; const TOTAL_OBJECTS = BUCKET_NUMBER * OBJECT_NUMBER; -const KMS_NODES = helpers.config.kmip.transport.length; +const KMS_NODES = helpers.config.getDefaultKms().settings.transport.length; const TOTAL_OBJECTS_PER_NODE = Math.floor(TOTAL_OBJECTS / KMS_NODES); /** diff --git a/tests/unit/encryption/kmsClients.js b/tests/unit/encryption/kmsClients.js new file mode 100644 index 0000000000..703cf579ff --- /dev/null +++ b/tests/unit/encryption/kmsClients.js @@ -0,0 +1,79 @@ +const assert = require('assert'); +const sinon = require('sinon'); + +const { ConfigObject: Config } = require('../../../lib/Config'); +const logger = require('../../../lib/utilities/logger'); +const { createClients } = require('../../../lib/kms/wrapper'); + +function parseKmsEntries(kms) { + const config = new Config(); + config._parseKms({ kms }); + return config.kms; +} + +describe('KMS createClients', () => { + afterEach(() => { + sinon.restore(); + }); + + it('should build a client for every entry, keyed by its backend', () => { + const entries = parseKmsEntries([ + { + protocol: 'kmip', + providerName: 'thales', + default: true, + transport: { tls: { host: 'kmip.example.com' } }, + }, + { + protocol: 'aws_kms', + providerName: 'acme', + endpoint: 'https://kms.example.com', + ak: 'accessKey', + sk: 'secretKey', + }, + { protocol: 'file', providerName: 'scality' }, + ]); + + const { clientInstances, defaultInstance } = createClients(entries); + + assert.deepStrictEqual( + Object.entries(clientInstances).map(([identifier, instance]) => [identifier, instance.implName]), + [ + ['external:kmip:thales', 'kmip'], + ['external:aws_kms:acme', 'aws'], + ['internal:file:scality', 'fileKms'], + ], + ); + assert.strictEqual(defaultInstance, clientInstances['external:kmip:thales']); + }); + + it('should build a cluster client for a kmip entry with several hosts', () => { + const entries = parseKmsEntries([ + { + protocol: 'kmip', + providerName: 'thales', + default: true, + transport: [{ tls: { host: 'kmip1' } }, { tls: { host: 'kmip2' } }], + }, + ]); + + const { defaultInstance } = createClients(entries); + + assert.strictEqual(defaultInstance.client.clients.length, 2); + assert.strictEqual(defaultInstance.client.retries, 1); + assert.strictEqual(defaultInstance.client.backend.arnPrefix, 'arn:scality:kms:external:kmip:thales:key/'); + }); + + it('should use the file backend for a SafeNet entry when the SafeNet client module is not installed', () => { + const warn = sinon.stub(logger, 'warn'); + const entries = parseKmsEntries([ + { protocol: 'scality', providerName: 'safenet', default: true, userName: 'user', password: 'password' }, + ]); + + const { clientInstances, defaultInstance } = createClients(entries); + + assert.deepStrictEqual(Object.keys(clientInstances), ['internal:file:scality']); + assert.strictEqual(defaultInstance.implName, 'fileKms'); + assert(warn.calledOnceWith('scality kms unavailable. Using file kms backend unless mem specified.')); + }); +}); diff --git a/tests/unit/testConfigs/parseKms.js b/tests/unit/testConfigs/parseKms.js new file mode 100644 index 0000000000..7d8f3f2fe2 --- /dev/null +++ b/tests/unit/testConfigs/parseKms.js @@ -0,0 +1,462 @@ +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { ConfigObject: Config } = require('../../../lib/Config'); + +const kmipEntry = { + protocol: 'kmip', + providerName: 'thales', + default: true, + transport: { tls: { host: 'kmip.example.com', port: 5696 } }, +}; + +const awsEntry = { + protocol: 'aws_kms', + providerName: 'acme', + endpoint: 'https://kms.example.com', + ak: 'accessKey', + sk: 'secretKey', +}; + +const fileEntry = { + protocol: 'file', + providerName: 'scality', +}; + +const safeNetEntry = { + protocol: 'scality', + providerName: 'safenet', + default: true, + userName: 'user', + password: 'password', +}; + +const kmsEnvironmentVariables = [ + 'S3KMS', + 'S3BACKEND', + 'S3KMIP_PIPELINE_DEPTH', + 'S3KMIP_PORT', + 'S3KMIP_HOSTS', + 'S3KMIP_HOST', + 'S3KMIP_KEY', + 'S3KMIP_CERT', + 'S3KMIP_CA', + 'S3KMIP_COMPOUND_CREATE', + 'S3KMIP_BUCKET_ATTRIBUTE_NAME', +]; + +describe('parseKms Function', () => { + let configInstance; + let savedEnvironment; + + beforeEach(() => { + configInstance = new Config(); + savedEnvironment = {}; + kmsEnvironmentVariables.forEach(name => { + savedEnvironment[name] = process.env[name]; + delete process.env[name]; + }); + }); + + afterEach(() => { + kmsEnvironmentVariables.forEach(name => { + if (savedEnvironment[name] === undefined) { + delete process.env[name]; + } else { + process.env[name] = savedEnvironment[name]; + } + }); + }); + + describe('with a kms array', () => { + it('should parse a kmip entry', () => { + configInstance._parseKms({ kms: [kmipEntry] }); + assert.strictEqual(configInstance.kms.length, 1); + const [entry] = configInstance.kms; + assert.strictEqual(entry.protocol, 'kmip'); + assert.strictEqual(entry.providerName, 'thales'); + assert.strictEqual(entry.default, true); + assert.strictEqual(entry.settings.providerName, 'thales'); + assert.strictEqual(entry.settings.transport.tls.host, 'kmip.example.com'); + assert.strictEqual(entry.settings.transport.tls.port, 5696); + assert.strictEqual(entry.settings.transport.pipelineDepth, 8); + assert.deepStrictEqual(entry.settings.client, { + compoundCreateActivate: false, + bucketNameAttributeName: '', + }); + }); + + it('should parse a kmip entry with a cluster transport', () => { + const kmipCluster = { + ...kmipEntry, + transport: [{ tls: { host: 'kmip1' } }, { tls: { host: 'kmip2' } }], + }; + configInstance._parseKms({ kms: [kmipCluster] }); + const { settings } = configInstance.kms[0]; + assert.deepStrictEqual( + settings.transport.map(transport => transport.tls.host), + ['kmip1', 'kmip2'], + ); + assert.strictEqual(settings.retries, 1); + }); + + it('should parse an aws_kms entry', () => { + configInstance._parseKms({ kms: [{ ...awsEntry, default: true }] }); + const [entry] = configInstance.kms; + assert.strictEqual(entry.protocol, 'aws_kms'); + assert.deepStrictEqual(entry.settings, { + providerName: 'acme', + endpoint: 'https://kms.example.com', + ak: 'accessKey', + sk: 'secretKey', + }); + }); + + it('should parse a file entry', () => { + configInstance._parseKms({ kms: [{ ...fileEntry, default: true }] }); + assert.deepStrictEqual(configInstance.kms, [ + { protocol: 'file', providerName: 'scality', default: true, settings: {} }, + ]); + }); + + it('should parse a SafeNet entry without its protocol and default fields', () => { + configInstance._parseKms({ kms: [safeNetEntry] }); + assert.deepStrictEqual(configInstance.kms[0].settings, { + providerName: 'safenet', + userName: 'user', + password: 'password', + }); + }); + + it('should parse several entries and mark the default one', () => { + configInstance._parseKms({ kms: [awsEntry, kmipEntry, fileEntry] }); + assert.deepStrictEqual( + configInstance.kms.map(entry => [entry.protocol, entry.default]), + [ + ['aws_kms', false], + ['kmip', true], + ['file', false], + ], + ); + assert.strictEqual(configInstance.getDefaultKms().providerName, 'thales'); + }); + + it('should not select the KMS from S3KMS or S3BACKEND and record S3KMS as ignored', () => { + process.env.S3KMS = 'file'; + process.env.S3BACKEND = 'mem'; + configInstance._parseKms({ kms: [kmipEntry] }); + assert.strictEqual(configInstance.getDefaultKms().protocol, 'kmip'); + assert.deepStrictEqual(configInstance.kmsIgnoredEnvironment, ['S3KMS']); + }); + + it('should record no ignored environment variable when none is set', () => { + configInstance._parseKms({ kms: [kmipEntry] }); + assert.deepStrictEqual(configInstance.kmsIgnoredEnvironment, []); + }); + + [ + ['kmip', { kmip: { providerName: 'thales' } }], + ['kmsAWS', { kmsAWS: { providerName: 'acme' } }], + ].forEach(([key, config]) => { + it(`should reject the ${key} key`, () => { + assert.throws( + () => configInstance._parseKms(config), + /kmip and kmsAWS are replaced by entries of the kms array/, + ); + }); + }); + + it('should reject a kms object', () => { + assert.throws( + () => configInstance._parseKms({ kms: { providerName: 'safenet', userName: 'u', password: 'p' } }), + /kms must be an array of KMS entries/, + ); + }); + + it('should reject an empty kms array', () => { + assert.throws(() => configInstance._parseKms({ kms: [] }), /kms must have at least one entry/); + }); + + it('should reject an entry that is not an object', () => { + assert.throws(() => configInstance._parseKms({ kms: ['kmip'] }), /each kms entry must be an object/); + }); + + ['mem', 'cdmi', 'aws', undefined].forEach(protocol => { + it(`should reject the protocol ${protocol}`, () => { + assert.throws( + () => configInstance._parseKms({ kms: [{ ...kmipEntry, protocol }] }), + new RegExp(`protocol must be one of file, kmip, aws_kms, scality, got "${protocol}"`), + ); + }); + }); + + it('should reject an entry without providerName', () => { + const { providerName, ...entry } = kmipEntry; // eslint-disable-line no-unused-vars + assert.throws(() => configInstance._parseKms({ kms: [entry] }), /kmip kms entry must define providerName/); + }); + + it('should reject an invalid providerName', () => { + assert.throws( + () => configInstance._parseKms({ kms: [{ ...kmipEntry, providerName: 'Thales' }] }), + /providerName must be lowercase alphanumeric only/, + ); + }); + + it('should reject a providerName used twice', () => { + assert.throws( + () => configInstance._parseKms({ kms: [kmipEntry, { ...awsEntry, providerName: 'thales' }] }), + /providerName "thales" is not unique/, + ); + }); + + it('should reject a file entry with a providerName other than scality', () => { + assert.throws( + () => configInstance._parseKms({ kms: [{ ...fileEntry, providerName: 'local', default: true }] }), + /file kms entry providerName must be "scality"/, + ); + }); + + it('should reject a default that is not a boolean', () => { + assert.throws( + () => configInstance._parseKms({ kms: [{ ...kmipEntry, default: 'true' }] }), + /kms entry default must be a boolean/, + ); + }); + + it('should reject entries without a default', () => { + assert.throws( + () => configInstance._parseKms({ kms: [awsEntry, fileEntry] }), + /exactly one kms entry must have "default": true, found 0/, + ); + }); + + it('should reject entries with two defaults', () => { + assert.throws( + () => configInstance._parseKms({ kms: [kmipEntry, { ...awsEntry, default: true }] }), + /exactly one kms entry must have "default": true, found 2/, + ); + }); + + it('should reject a kmip entry without transport', () => { + const { transport, ...entry } = kmipEntry; // eslint-disable-line no-unused-vars + assert.throws(() => configInstance._parseKms({ kms: [entry] }), /KMIP transport must define tls.host/); + }); + + it('should reject a kmip transport without host', () => { + assert.throws( + () => configInstance._parseKms({ kms: [{ ...kmipEntry, transport: { tls: { port: 5696 } } }] }), + /KMIP transport must define tls.host/, + ); + }); + + it('should reject a kmip cluster transport without host', () => { + const transport = [{ tls: { host: 'kmip1' } }, { pipelineDepth: 8 }]; + assert.throws( + () => configInstance._parseKms({ kms: [{ ...kmipEntry, transport }] }), + /KMIP transport must define tls.host/, + ); + }); + + it('should reject an aws_kms entry without endpoint', () => { + const { endpoint, ...entry } = awsEntry; // eslint-disable-line no-unused-vars + assert.throws( + () => configInstance._parseKms({ kms: [{ ...entry, default: true }] }), + /endpoint must be defined in the aws_kms KMS entry/, + ); + }); + + it('should reject a SafeNet entry without userName', () => { + const { userName, ...entry } = safeNetEntry; // eslint-disable-line no-unused-vars + assert.throws( + () => configInstance._parseKms({ kms: [entry] }), + /userName of the SafeNet KMS entry must be a string/, + ); + }); + }); + + describe('without a kms array', () => { + it('should select the file KMS when no environment variable is set', () => { + configInstance._parseKms({}); + assert.deepStrictEqual(configInstance.kms, [{ protocol: 'file', default: true, settings: {} }]); + assert.deepStrictEqual(configInstance.kmsIgnoredEnvironment, []); + }); + + [ + ['mem', 'mem'], + ['file', 'file'], + ['cdmi', 'file'], + ['scality', 'scality'], + ].forEach(([backend, protocol]) => { + it(`should select the ${protocol} KMS with S3BACKEND=${backend}`, () => { + process.env.S3BACKEND = backend; + configInstance._parseKms({}); + assert.strictEqual(configInstance.getDefaultKms().protocol, protocol); + }); + }); + + it('should select the KMS from S3KMS over S3BACKEND', () => { + process.env.S3BACKEND = 'mem'; + process.env.S3KMS = 'file'; + configInstance._parseKms({}); + assert.strictEqual(configInstance.getDefaultKms().protocol, 'file'); + }); + + ['kmip', 'aws'].forEach(name => { + it(`should reject S3KMS=${name}`, () => { + process.env.S3KMS = name; + assert.throws( + () => configInstance._parseKms({}), + new RegExp(`S3KMS=${name} needs an entry with a providerName in the kms array`), + ); + }); + }); + + it('should reject an unknown S3KMS', () => { + process.env.S3KMS = 'unknown'; + assert.throws(() => configInstance._parseKms({}), /S3KMS should be one of mem\/file\/cdmi\/scality/); + }); + }); + + describe('with S3KMIP_* environment variables', () => { + const kmipEntryWithoutTransport = { protocol: 'kmip', providerName: 'thales', default: true }; + let tlsDirectory; + + beforeEach(() => { + tlsDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'parseKms-')); + }); + + afterEach(() => { + fs.rmSync(tlsDirectory, { recursive: true, force: true }); + }); + + it('should take the transport and client of the default kmip entry from them', () => { + process.env.S3KMIP_HOSTS = 'kmip.example.com'; + process.env.S3KMIP_PORT = '5697'; + process.env.S3KMIP_PIPELINE_DEPTH = '4'; + process.env.S3KMIP_COMPOUND_CREATE = 'true'; + process.env.S3KMIP_BUCKET_ATTRIBUTE_NAME = 'bucket'; + configInstance._parseKms({ kms: [kmipEntryWithoutTransport] }); + const { settings } = configInstance.getDefaultKms(); + assert.strictEqual(settings.transport.tls.host, 'kmip.example.com'); + assert.strictEqual(settings.transport.tls.port, '5697'); + assert.strictEqual(settings.transport.pipelineDepth, '4'); + assert.deepStrictEqual(settings.client, { + compoundCreateActivate: true, + bucketNameAttributeName: 'bucket', + }); + }); + + it('should take the host from S3KMIP_HOST', () => { + process.env.S3KMIP_HOST = 'kmip.example.com'; + configInstance._parseKms({ kms: [kmipEntryWithoutTransport] }); + assert.strictEqual(configInstance.getDefaultKms().settings.transport.tls.host, 'kmip.example.com'); + }); + + it('should let the settings of the default kmip entry override them', () => { + process.env.S3KMIP_HOSTS = 'env.example.com'; + process.env.S3KMIP_PORT = '5697'; + process.env.S3KMIP_BUCKET_ATTRIBUTE_NAME = 'envBucket'; + configInstance._parseKms({ + kms: [ + { + ...kmipEntryWithoutTransport, + client: { bucketNameAttributeName: 'entryBucket' }, + transport: { tls: { host: 'entry.example.com', port: 5698 } }, + }, + ], + }); + const { settings } = configInstance.getDefaultKms(); + assert.strictEqual(settings.transport.tls.host, 'entry.example.com'); + assert.strictEqual(settings.transport.tls.port, 5698); + assert.strictEqual(settings.client.bucketNameAttributeName, 'entryBucket'); + }); + + it('should keep S3KMIP_COMPOUND_CREATE=true when the entry sets compoundCreateActivate to false', () => { + process.env.S3KMIP_COMPOUND_CREATE = 'true'; + configInstance._parseKms({ + kms: [{ ...kmipEntry, client: { compoundCreateActivate: false } }], + }); + assert.strictEqual(configInstance.getDefaultKms().settings.client.compoundCreateActivate, true); + }); + + it('should drop S3KMIP_CA when the entry sets tls without ca', () => { + const caFile = path.join(tlsDirectory, 'ca.pem'); + fs.writeFileSync(caFile, 'ca'); + process.env.S3KMIP_CA = caFile; + configInstance._parseKms({ + kms: [{ ...kmipEntryWithoutTransport, transport: { tls: { host: 'kmip1' } } }], + }); + assert.strictEqual(configInstance.getDefaultKms().settings.transport.tls.ca, undefined); + }); + + it('should load S3KMIP_CA when the entry has no tls', () => { + const caFile = path.join(tlsDirectory, 'ca.pem'); + fs.writeFileSync(caFile, 'ca'); + process.env.S3KMIP_HOSTS = 'kmip1'; + process.env.S3KMIP_CA = caFile; + configInstance._parseKms({ kms: [kmipEntryWithoutTransport] }); + assert.deepStrictEqual(configInstance.getDefaultKms().settings.transport.tls.ca, [Buffer.from('ca')]); + }); + + it('should apply to every transport of a default kmip cluster', () => { + process.env.S3KMIP_PIPELINE_DEPTH = '4'; + configInstance._parseKms({ + kms: [ + { + ...kmipEntryWithoutTransport, + transport: [{ tls: { host: 'kmip1' } }, { tls: { host: 'kmip2' } }], + }, + ], + }); + assert.deepStrictEqual( + configInstance.getDefaultKms().settings.transport.map(transport => transport.pipelineDepth), + ['4', '4'], + ); + }); + + it('should not apply to a kmip entry that is not the default', () => { + process.env.S3KMIP_COMPOUND_CREATE = 'true'; + process.env.S3KMIP_PIPELINE_DEPTH = '4'; + configInstance._parseKms({ + kms: [ + { ...awsEntry, default: true }, + { ...kmipEntry, default: false }, + ], + }); + const kmipSettings = configInstance.kms.find(entry => entry.protocol === 'kmip').settings; + assert.strictEqual(kmipSettings.client.compoundCreateActivate, false); + assert.strictEqual(kmipSettings.transport.pipelineDepth, 8); + }); + + it('should not give a host to a kmip entry that is not the default', () => { + process.env.S3KMIP_HOSTS = 'kmip.example.com'; + assert.throws( + () => + configInstance._parseKms({ + kms: [ + { ...awsEntry, default: true }, + { protocol: 'kmip', providerName: 'thales' }, + ], + }), + /KMIP transport must define tls.host/, + ); + }); + }); + + describe('with sseMigration from an external KMS', () => { + it('should name the protocol of the default KMS', () => { + configInstance._parseKms({ kms: [{ ...fileEntry, default: true }] }); + const sseMigration = { + previousKeyType: 'external', + previousKeyProtocol: 'kmip', + previousKeyProvider: 'thales', + }; + assert.throws( + () => configInstance._sseMigration({ sseMigration }), + /can't migrate to "internal" KMS provider file/, + ); + }); + }); +}); diff --git a/tests/unit/testConfigs/parseKmsAWS.js b/tests/unit/testConfigs/parseKmsAWS.js index c466bbee77..d8747b6415 100644 --- a/tests/unit/testConfigs/parseKmsAWS.js +++ b/tests/unit/testConfigs/parseKmsAWS.js @@ -12,25 +12,19 @@ describe('parseKmsAWS Function', () => { configInstance = new Config(); }); - it('should return an empty object if no kmsAWS config is provided', () => { - const config = {}; - const result = configInstance._parseKmsAWS(config); - assert.deepStrictEqual(result, {}); - }); - it('should throw an error if endpoint is not defined in kmsAWS', () => { const config = { kmsAWS: { providerName: 'tests', ak: 'ak', sk: 'sk' } }; - assert.throws(() => configInstance._parseKmsAWS(config), 'endpoint must be defined'); + assert.throws(() => configInstance._parseKmsAWS(config.kmsAWS), 'endpoint must be defined'); }); it('should throw an error if ak is not defined in kmsAWS', () => { const config = { kmsAWS: { providerName: 'tests', endpoint: 'https://example.com', sk: 'sk' } }; - assert.throws(() => configInstance._parseKmsAWS(config), 'ak must be defined'); + assert.throws(() => configInstance._parseKmsAWS(config.kmsAWS), 'ak must be defined'); }); it('should throw an error if sk is not defined in kmsAWS', () => { const config = { kmsAWS: { providerName: 'tests', endpoint: 'https://example.com', ak: 'ak' } }; - assert.throws(() => configInstance._parseKmsAWS(config), 'sk must be defined'); + assert.throws(() => configInstance._parseKmsAWS(config.kmsAWS), 'sk must be defined'); }); it('should return the expected kmsAWS object when valid config is provided', () => { @@ -42,7 +36,7 @@ describe('parseKmsAWS Function', () => { sk: 'secretKey', }, }; - const result = configInstance._parseKmsAWS(config); + const result = configInstance._parseKmsAWS(config.kmsAWS); assert.deepStrictEqual(result, { providerName: 'tests', endpoint: 'https://example.com', @@ -61,7 +55,7 @@ describe('parseKmsAWS Function', () => { region: 'us-west-2', }, }; - const result = configInstance._parseKmsAWS(config); + const result = configInstance._parseKmsAWS(config.kmsAWS); assert.deepStrictEqual(result, { providerName: 'tests', endpoint: 'https://example.com', @@ -85,7 +79,7 @@ describe('parseKmsAWS Function', () => { }, }, }; - const result = configInstance._parseKmsAWS(config); + const result = configInstance._parseKmsAWS(config.kmsAWS); assert.deepStrictEqual(result, { providerName: 'tests', endpoint: 'https://example.com', @@ -128,7 +122,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - assert.throws(() => configInstance._parseKmsAWS(config)); + assert.throws(() => configInstance._parseKmsAWS(config.kmsAWS)); }); it('should throw an error if tls.minVersion is not a string', () => { @@ -144,7 +138,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - assert.throws(() => configInstance._parseKmsAWS(config), { + assert.throws(() => configInstance._parseKmsAWS(config.kmsAWS), { message: 'bad config: KMS AWS TLS minVersion must be a string', }); }); @@ -162,7 +156,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - assert.throws(() => configInstance._parseKmsAWS(config), { + assert.throws(() => configInstance._parseKmsAWS(config.kmsAWS), { message: 'bad config: KMS AWS TLS maxVersion must be a string', }); }); @@ -180,7 +174,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - assert.throws(() => configInstance._parseKmsAWS(config), { + assert.throws(() => configInstance._parseKmsAWS(config.kmsAWS), { message: 'bad config: TLS file specification must be a string', }); }); @@ -196,7 +190,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - const result = configInstance._parseKmsAWS(config); + const result = configInstance._parseKmsAWS(config.kmsAWS); assert.deepStrictEqual(result.tls, {}); }); @@ -213,7 +207,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - const result = configInstance._parseKmsAWS(config); + const result = configInstance._parseKmsAWS(config.kmsAWS); assert.deepStrictEqual(result.tls.ca, [mockCertifContent, mockCertifContent]); assert(readFileSyncStub.calledTwice); @@ -234,7 +228,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - const result = configInstance._parseKmsAWS(config); + const result = configInstance._parseKmsAWS(config.kmsAWS); assert.deepStrictEqual(result.tls.cert, mockCertifContent); assert(readFileSyncStub.calledOnce); @@ -254,7 +248,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - const result = configInstance._parseKmsAWS(config); + const result = configInstance._parseKmsAWS(config.kmsAWS); assert.deepStrictEqual(result.tls.key, mockCertifContent); assert(readFileSyncStub.calledOnce); @@ -271,7 +265,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - const result = configInstance._parseKmsAWS(config); + const result = configInstance._parseKmsAWS(config.kmsAWS); assert.strictEqual(result.tls, undefined); assert(readFileSyncStub.notCalled); @@ -292,7 +286,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - const result = configInstance._parseKmsAWS(config); + const result = configInstance._parseKmsAWS(config.kmsAWS); assert.deepStrictEqual(result.tls.cert, mockCertifContent); assert(readFileSyncStub.calledOnce); @@ -320,7 +314,7 @@ describe('parseKmsAWS TLS section', () => { }, }; - const result = configInstance._parseKmsAWS(config); + const result = configInstance._parseKmsAWS(config.kmsAWS); assert.deepStrictEqual(result.tls.key, [mockCertifContent, mockCertifContent]); assert.deepStrictEqual(result.tls.cert, [mockCertifContent, mockCertifContent]);