From ca5e625a98f2d21020040d0f4db061b459c19480 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Oct 2026 21:35:10 +0000 Subject: [PATCH 1/4] feat(test-server): report caching CMM call counts and support test hooks --- test-server/src/bridge.ts | 75 ++++++++++++++++++++++++++++++--- test-server/src/model.ts | 9 ++++ test-server/src/server.ts | 13 +++++- test-server/test/server.test.ts | 40 ++++++++++++++++++ 4 files changed, 130 insertions(+), 7 deletions(-) diff --git a/test-server/src/bridge.ts b/test-server/src/bridge.ts index 6f321a831..43aad7059 100644 --- a/test-server/src/bridge.ts +++ b/test-server/src/bridge.ts @@ -50,11 +50,59 @@ import { MultiKeyringConfig, RawAesKeyringConfig, RawRsaKeyringConfig, + TestHooks, toSuiteId, } from './model' type Client = ReturnType & ReturnType +/* Calls a client's caching CMMs made to the CMMs they wrap. */ +export interface CallCounts { + getEncryptionMaterials: number + decryptMaterials: number +} + +/* Test-only wrapper around a caching CMM's underlying CMM. + * Counts the caching CMM's calls to it, and can delay each call + * so that concurrent cache misses overlap. + */ +class InstrumentedMaterialsManager implements NodeMaterialsManager { + constructor( + private readonly inner: NodeMaterialsManager, + private readonly counts: CallCounts, + private readonly delayMilliseconds: number + ) {} + + async getEncryptionMaterials( + request: Parameters[0] + ) { + this.counts.getEncryptionMaterials += 1 + await this.delay() + return this.inner.getEncryptionMaterials(request) + } + + async decryptMaterials( + request: Parameters[0] + ) { + this.counts.decryptMaterials += 1 + await this.delay() + return this.inner.decryptMaterials(request) + } + + private async delay() { + if (this.delayMilliseconds > 0) { + await new Promise((resolve) => + setTimeout(resolve, this.delayMilliseconds) + ) + } + } +} + +interface Instrumentation { + counts: CallCounts + delayMilliseconds: number +} + export interface OperationResult { data: Buffer encryptionContext?: EncryptionContext @@ -253,7 +301,10 @@ function buildKeyring(keyring: KeyringConfig): KeyringNode { } } -function buildCmm(cmm: CmmConfig): NodeMaterialsManager { +function buildCmm( + cmm: CmmConfig, + instrumentation: Instrumentation +): NodeMaterialsManager { const [name, config] = oneVariant(cmm, 'cmm') switch (name) { case 'Default': { @@ -265,7 +316,11 @@ function buildCmm(cmm: CmmConfig): NodeMaterialsManager { case 'Caching': { const cfg = config as CachingCmmConfig return new NodeCachingMaterialsManager({ - backingMaterials: buildCmm(cfg.underlyingCMM), + backingMaterials: new InstrumentedMaterialsManager( + buildCmm(cfg.underlyingCMM, instrumentation), + instrumentation.counts, + instrumentation.delayMilliseconds + ), cache: getLocalCryptographicMaterialsCache(100), maxAge: cfg.cacheLimitTtlSeconds * 1000, partition: cfg.partitionId, @@ -285,7 +340,8 @@ function buildCmm(cmm: CmmConfig): NodeMaterialsManager { export class EsdkClientBundle { constructor( private readonly client: Client, - private readonly cmm: NodeMaterialsManager + private readonly cmm: NodeMaterialsManager, + readonly callCounts: CallCounts ) {} async encrypt( @@ -385,9 +441,16 @@ async function collect(stream: NodeJS.ReadableStream): Promise { return Buffer.concat(chunks) } -export function buildClientBundle(config: EsdkClientConfig): EsdkClientBundle { +export function buildClientBundle( + config: EsdkClientConfig, + testHooks: TestHooks = {} +): EsdkClientBundle { const policy = commitmentPolicy(config.commitmentPolicy) - const cmm = buildCmm(config.cmm) + const callCounts = { getEncryptionMaterials: 0, decryptMaterials: 0 } + const cmm = buildCmm(config.cmm, { + counts: callCounts, + delayMilliseconds: testHooks.cachingCmmUnderlyingDelayMilliseconds || 0, + }) const client = buildClient({ commitmentPolicy: policy, maxEncryptedDataKeys: @@ -396,5 +459,5 @@ export function buildClientBundle(config: EsdkClientConfig): EsdkClientBundle { ? false : config.maxEncryptedDataKeys, }) - return new EsdkClientBundle(client, cmm) + return new EsdkClientBundle(client, cmm, callCounts) } diff --git a/test-server/src/model.ts b/test-server/src/model.ts index 6b5faba4c..2435680f1 100644 --- a/test-server/src/model.ts +++ b/test-server/src/model.ts @@ -109,6 +109,11 @@ export interface EsdkClientConfig { export interface CreateClientRequest { config?: EsdkClientConfig + testHooks?: TestHooks +} + +export interface TestHooks { + cachingCmmUnderlyingDelayMilliseconds?: number } export interface EncryptRequest { @@ -131,6 +136,10 @@ export interface EncryptStreamRequest extends EncryptRequest { export type DecryptStreamRequest = DecryptRequest +export interface GetCallCountsRequest { + clientId?: string +} + /* Modeled ESDKAlgorithmSuiteId name -> library AlgorithmSuiteIdentifier. */ const SUITE_BY_MODEL_NAME: { [name: string]: AlgorithmSuiteIdentifier } = { ALG_AES_128_GCM_IV12_TAG16_NO_KDF: diff --git a/test-server/src/server.ts b/test-server/src/server.ts index 4017b5a52..9b4409410 100644 --- a/test-server/src/server.ts +++ b/test-server/src/server.ts @@ -27,6 +27,7 @@ import { DecryptStreamRequest, EncryptRequest, EncryptStreamRequest, + GetCallCountsRequest, } from './model' const SMITHY_PROTOCOL = 'rpc-v2-cbor' @@ -62,7 +63,7 @@ async function createClient( if (!config) throw new ServerError('config is required') let bundle: EsdkClientBundle try { - bundle = buildClientBundle(config) + bundle = buildClientBundle(config, request.testHooks) } catch (err) { if (err instanceof ClientError || err instanceof ServerError) throw err throw new ServerError( @@ -153,6 +154,16 @@ async function dispatch( ) ) } + case 'GetCallCounts': { + const { callCounts } = registry.resolve( + (request as GetCallCountsRequest).clientId + ) + return { + cachingCmmGetEncryptionMaterialsCalls: + callCounts.getEncryptionMaterials, + cachingCmmDecryptMaterialsCalls: callCounts.decryptMaterials, + } + } default: throw new ServerError(`unknown operation: ${operation}`) } diff --git a/test-server/test/server.test.ts b/test-server/test/server.test.ts index ea9a7f70f..66efae140 100644 --- a/test-server/test/server.test.ts +++ b/test-server/test/server.test.ts @@ -396,6 +396,46 @@ describe('client construction', () => { ) }) + it('GetCallCounts reports the caching CMM calls to its underlying CMM', async () => { + const base = rawAesConfig() as { cmm: CborValue; commitmentPolicy: string } + const clientId = await createClient({ + commitmentPolicy: base.commitmentPolicy, + cmm: { + Caching: { underlyingCMM: base.cmm, cacheLimitTtlSeconds: 60 }, + }, + }) + const encrypted = await operation('Encrypt', { + clientId, + plaintext: PLAINTEXT, + }) + await operation('Encrypt', { clientId, plaintext: PLAINTEXT }) + await operation('Decrypt', { + clientId, + ciphertext: encrypted.body.ciphertext, + }) + await operation('Decrypt', { + clientId, + ciphertext: encrypted.body.ciphertext, + }) + + const counts = await operation('GetCallCounts', { clientId }) + expect(counts.status).to.equal(200) + expect(counts.body).to.deep.equal({ + cachingCmmGetEncryptionMaterialsCalls: 1, + cachingCmmDecryptMaterialsCalls: 1, + }) + }) + + it('GetCallCounts reports zero for a client with no caching CMM', async () => { + const clientId = await createClient(rawAesConfig()) + await operation('Encrypt', { clientId, plaintext: PLAINTEXT }) + const counts = await operation('GetCallCounts', { clientId }) + expect(counts.body).to.deep.equal({ + cachingCmmGetEncryptionMaterialsCalls: 0, + cachingCmmDecryptMaterialsCalls: 0, + }) + }) + it('multi-keyring with a raw-AES generator round-trips', async () => { const clientId = await createClient({ commitmentPolicy: 'REQUIRE_ENCRYPT_REQUIRE_DECRYPT', From 6be1405c1931dd234ab3c507747cd3a853a570cc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Oct 2026 21:38:17 +0000 Subject: [PATCH 2/4] chore(test-server): declare the caching CMM sharing bug and point at the caching tests --- test-server/bug-config.json | 3 ++- test-server/server-config.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test-server/bug-config.json b/test-server/bug-config.json index 0dd86cbba..c507789ee 100644 --- a/test-server/bug-config.json +++ b/test-server/bug-config.json @@ -2,5 +2,6 @@ "decrypt-accepts-trailing-bytes-commit-key", "create-client-accepts-reserved-aws-kms-namespace", "encrypt-accepts-reserved-prefix-encryption-context-key", - "encrypt-mishandles-proto-encryption-context-key" + "encrypt-mishandles-proto-encryption-context-key", + "caching-cmm-concurrent-misses-not-shared" ] diff --git a/test-server/server-config.json b/test-server/server-config.json index 180128721..c7c89c20d 100644 --- a/test-server/server-config.json +++ b/test-server/server-config.json @@ -2,7 +2,7 @@ "commonsRepository": { "name": "aws-crypto-tools-commons", "url": "git@github.com:aws/aws-crypto-tools-commons.git", - "branch": "lucmcdon/esdk-test-server-all-languages" + "branch": "lucmcdon/caching-cmm-tests" }, "product": "esdk" } From d188c356c826cddbc19b269166b8ce2479565098 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Oct 2026 22:20:52 +0000 Subject: [PATCH 3/4] feat(test-server): run calls concurrently and advance the cache clock on request --- modules/cache-material/src/cache_clock.ts | 26 +++ ...hing_cryptographic_materials_decorators.ts | 5 +- ...get_local_cryptographic_materials_cache.ts | 181 +---------------- .../local_cryptographic_materials_cache.ts | 189 ++++++++++++++++++ .../cache-material/test/cache_clock.test.ts | 88 ++++++++ test-server/src/bridge.ts | 64 +++--- test-server/src/model.ts | 22 +- test-server/src/server.ts | 52 ++++- 8 files changed, 405 insertions(+), 222 deletions(-) create mode 100644 modules/cache-material/src/cache_clock.ts create mode 100644 modules/cache-material/src/local_cryptographic_materials_cache.ts create mode 100644 modules/cache-material/test/cache_clock.test.ts diff --git a/modules/cache-material/src/cache_clock.ts b/modules/cache-material/src/cache_clock.ts new file mode 100644 index 000000000..3c35b9e6a --- /dev/null +++ b/modules/cache-material/src/cache_clock.ts @@ -0,0 +1,26 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/* The time source for a cache's entries, and for the caching CMMs that check their age. + * A cache built with getLocalCryptographicMaterialsCache uses Date.now. + * The ESDK test server builds caches with a clock it can move forward, + * so tests can expire entries without waiting. + * This module is not exported from the package index. + */ +import { CryptographicMaterialsCache } from './cryptographic_materials_cache' + +export type Clock = () => number + +const clocks = new WeakMap, Clock>() + +export function setClock( + cache: CryptographicMaterialsCache, + clock: Clock +): void { + clocks.set(cache, clock) +} + +/* A cache from another implementation has no registered clock, so it gets Date.now. */ +export function clockFor(cache: CryptographicMaterialsCache): Clock { + return clocks.get(cache) || Date.now +} diff --git a/modules/cache-material/src/caching_cryptographic_materials_decorators.ts b/modules/cache-material/src/caching_cryptographic_materials_decorators.ts index 5be6e144b..8915d9458 100644 --- a/modules/cache-material/src/caching_cryptographic_materials_decorators.ts +++ b/modules/cache-material/src/caching_cryptographic_materials_decorators.ts @@ -21,6 +21,7 @@ import { Entry, } from './cryptographic_materials_cache' import { CryptographicMaterialsCacheKeyHelpersInterface } from './build_cryptographic_materials_cache_key_helpers' +import { clockFor } from './cache_clock' export function decorateProperties( obj: CachingMaterialsManager, @@ -126,7 +127,7 @@ export function getEncryptionMaterials({ */ const testEntry = { response: material, - now: Date.now(), + now: clockFor(this._cache)(), messagesEncrypted: 1, bytesEncrypted: plaintextLength, } @@ -193,7 +194,7 @@ export function cacheEntryHasExceededLimits< this: CachingMaterialsManager, { now, messagesEncrypted, bytesEncrypted }: Entry ): boolean { - const age = Date.now() - now + const age = clockFor(this._cache)() - now return ( age > this._maxAge || messagesEncrypted > this._maxMessagesEncrypted || diff --git a/modules/cache-material/src/get_local_cryptographic_materials_cache.ts b/modules/cache-material/src/get_local_cryptographic_materials_cache.ts index d3150bc17..5c460ac9c 100644 --- a/modules/cache-material/src/get_local_cryptographic_materials_cache.ts +++ b/modules/cache-material/src/get_local_cryptographic_materials_cache.ts @@ -1,31 +1,9 @@ // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import LRU from 'lru-cache' -import { - EncryptionMaterial, - DecryptionMaterial, - SupportedAlgorithmSuites, - needs, - isEncryptionMaterial, - isDecryptionMaterial, - BranchKeyMaterial, - isBranchKeyMaterial, -} from '@aws-crypto/material-management' - -import { - CryptographicMaterialsCache, - Entry, - EncryptionMaterialEntry, - DecryptionMaterialEntry, - BranchKeyMaterialEntry, -} from './cryptographic_materials_cache' - -// define a broader type for local CMC entries that encompass BranchKeyMaterial -// entries as well -type LocalCmcEntry = - | BranchKeyMaterialEntry - | Entry +import { SupportedAlgorithmSuites } from '@aws-crypto/material-management' +import { CryptographicMaterialsCache } from './cryptographic_materials_cache' +import { localCryptographicMaterialsCache } from './local_cryptographic_materials_cache' export function getLocalCryptographicMaterialsCache< S extends SupportedAlgorithmSuites @@ -33,152 +11,9 @@ export function getLocalCryptographicMaterialsCache< capacity: number, proactiveFrequency: number = 1000 * 60 ): CryptographicMaterialsCache { - const cache = new LRU>({ - max: capacity, - dispose(_key, value) { - /* Zero out the unencrypted dataKey, when the material is removed from the cache. */ - value.response.zeroUnencryptedDataKey() - }, - }) - - /* It is not a guarantee that the last item in the LRU will be the Oldest Item. - * But such degenerative cases are not my concern. - * The LRU will not return things that are too old, - * so all this is just to try and proactively dispose material. - * - * To be clear, as an example say I add 9 items at T=0. - * If the MaxAge is 60 minutes, and at T=59 I add a 10th item. - * Then get each of the other 9 items. - * Now, at T=60, `mayEvictTail` will check the age of the tail - * and not evict it because the item has not aged out. - * If there is no get activity, - * it will take until T=120 before I again begin evicting items. - */ - ;(function proactivelyTryAndEvictTail() { - const timeout = setTimeout(() => { - mayEvictTail() - proactivelyTryAndEvictTail() - }, proactiveFrequency) - /* In Node.js the event loop will _only_ exit if there are no outstanding events. - * This means that if I did nothing the event loop would *always* be blocked. - * This is unfortunate and very bad for things like Lambda. - * So, I tell Node.js to not wait for this timer. - * See: https://nodejs.org/api/timers.html#timers_timeout_unref - */ - // @ts-ignore - timeout.unref && timeout.unref() - })() - - return { - putEncryptionMaterial( - key: string, - material: EncryptionMaterial, - plaintextLength: number, - maxAge?: number - ) { - /* Precondition: putEncryptionMaterial plaintextLength can not be negative. */ - needs(plaintextLength >= 0, 'Malformed plaintextLength') - /* Precondition: Only cache EncryptionMaterial. */ - needs(isEncryptionMaterial(material), 'Malformed response.') - /* Precondition: Only cache EncryptionMaterial that is cacheSafe. */ - needs(material.suite.cacheSafe, 'Can not cache non-cache safe material') - const entry = Object.seal({ - response: material, - bytesEncrypted: plaintextLength, - messagesEncrypted: 1, - now: Date.now(), - }) - - cache.set(key, entry, maxAge) - }, - - putDecryptionMaterial( - key: string, - material: DecryptionMaterial, - maxAge?: number - ) { - /* Precondition: Only cache DecryptionMaterial. */ - needs(isDecryptionMaterial(material), 'Malformed response.') - /* Precondition: Only cache DecryptionMaterial that is cacheSafe. */ - needs(material.suite.cacheSafe, 'Can not cache non-cache safe material') - const entry = Object.seal({ - response: material, - bytesEncrypted: 0, - messagesEncrypted: 0, - now: Date.now(), - }) - - cache.set(key, entry, maxAge) - }, - - putBranchKeyMaterial( - key: string, - material: BranchKeyMaterial, - maxAge?: number - ): void { - /* Precondition: Only cache BranchKeyMaterial */ - needs(isBranchKeyMaterial(material), 'Malformed response.') - - const entry = Object.seal({ - response: material, - now: Date.now(), - }) - - cache.set(key, entry, maxAge) - }, - - getEncryptionMaterial(key: string, plaintextLength: number) { - /* Precondition: plaintextLength can not be negative. */ - needs(plaintextLength >= 0, 'Malformed plaintextLength') - const entry = cache.get(key) - /* Check for early return (Postcondition): If this key does not have an EncryptionMaterial, return false. */ - if (!entry) return false - /* Postcondition: Only return EncryptionMaterial. */ - needs(isEncryptionMaterial(entry.response), 'Malformed response.') - - const encryptionMaterialEntry = entry as EncryptionMaterialEntry - encryptionMaterialEntry.bytesEncrypted += plaintextLength - encryptionMaterialEntry.messagesEncrypted += 1 - - return entry as EncryptionMaterialEntry - }, - - getDecryptionMaterial(key: string) { - const entry = cache.get(key) - /* Check for early return (Postcondition): If this key does not have a DecryptionMaterial, return false. */ - if (!entry) return false - /* Postcondition: Only return DecryptionMaterial. */ - needs(isDecryptionMaterial(entry.response), 'Malformed response.') - - return entry as DecryptionMaterialEntry - }, - - getBranchKeyMaterial(key: string): BranchKeyMaterialEntry | false { - const entry = cache.get(key) - - /* Postcondition: If this key does not have a BranchKeyMaterial, return false */ - if (!entry) return false - - /* Postcondition: Only return BranchKeyMaterial */ - needs(isBranchKeyMaterial(entry.response), 'Malformed response.') - return entry as BranchKeyMaterialEntry - }, - - del(key: string) { - cache.del(key) - }, - } - - function mayEvictTail() { - // @ts-ignore - const { tail } = cache.dumpLru() - /* Check for early return (Postcondition) UNTESTED: If there is no tail, then the cache is empty. */ - if (!tail) return - /* The underlying Yallist tail Node has a `value`. - * This value is a lru-cache Entry and has a `key`. - */ - const { key } = tail.value - // Peek will evict, but not update the "recently used"-ness of the key. - cache.peek(key) - } + return localCryptographicMaterialsCache( + capacity, + proactiveFrequency, + Date.now + ) } diff --git a/modules/cache-material/src/local_cryptographic_materials_cache.ts b/modules/cache-material/src/local_cryptographic_materials_cache.ts new file mode 100644 index 000000000..a7eb7d2af --- /dev/null +++ b/modules/cache-material/src/local_cryptographic_materials_cache.ts @@ -0,0 +1,189 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import LRU from 'lru-cache' +import { + EncryptionMaterial, + DecryptionMaterial, + SupportedAlgorithmSuites, + needs, + isEncryptionMaterial, + isDecryptionMaterial, + BranchKeyMaterial, + isBranchKeyMaterial, +} from '@aws-crypto/material-management' + +import { + CryptographicMaterialsCache, + Entry, + EncryptionMaterialEntry, + DecryptionMaterialEntry, + BranchKeyMaterialEntry, +} from './cryptographic_materials_cache' +import { Clock, setClock } from './cache_clock' + +// define a broader type for local CMC entries that encompass BranchKeyMaterial +// entries as well +type LocalCmcEntry = + | BranchKeyMaterialEntry + | Entry + +export function localCryptographicMaterialsCache< + S extends SupportedAlgorithmSuites +>( + capacity: number, + proactiveFrequency: number, + clock: Clock +): CryptographicMaterialsCache { + const cache = new LRU>({ + max: capacity, + dispose(_key, value) { + /* Zero out the unencrypted dataKey, when the material is removed from the cache. */ + value.response.zeroUnencryptedDataKey() + }, + }) + + /* It is not a guarantee that the last item in the LRU will be the Oldest Item. + * But such degenerative cases are not my concern. + * The LRU will not return things that are too old, + * so all this is just to try and proactively dispose material. + * + * To be clear, as an example say I add 9 items at T=0. + * If the MaxAge is 60 minutes, and at T=59 I add a 10th item. + * Then get each of the other 9 items. + * Now, at T=60, `mayEvictTail` will check the age of the tail + * and not evict it because the item has not aged out. + * If there is no get activity, + * it will take until T=120 before I again begin evicting items. + */ + ;(function proactivelyTryAndEvictTail() { + const timeout = setTimeout(() => { + mayEvictTail() + proactivelyTryAndEvictTail() + }, proactiveFrequency) + /* In Node.js the event loop will _only_ exit if there are no outstanding events. + * This means that if I did nothing the event loop would *always* be blocked. + * This is unfortunate and very bad for things like Lambda. + * So, I tell Node.js to not wait for this timer. + * See: https://nodejs.org/api/timers.html#timers_timeout_unref + */ + // @ts-ignore + timeout.unref && timeout.unref() + })() + + const cmc: CryptographicMaterialsCache = { + putEncryptionMaterial( + key: string, + material: EncryptionMaterial, + plaintextLength: number, + maxAge?: number + ) { + /* Precondition: putEncryptionMaterial plaintextLength can not be negative. */ + needs(plaintextLength >= 0, 'Malformed plaintextLength') + /* Precondition: Only cache EncryptionMaterial. */ + needs(isEncryptionMaterial(material), 'Malformed response.') + /* Precondition: Only cache EncryptionMaterial that is cacheSafe. */ + needs(material.suite.cacheSafe, 'Can not cache non-cache safe material') + const entry = Object.seal({ + response: material, + bytesEncrypted: plaintextLength, + messagesEncrypted: 1, + now: clock(), + }) + + cache.set(key, entry, maxAge) + }, + + putDecryptionMaterial( + key: string, + material: DecryptionMaterial, + maxAge?: number + ) { + /* Precondition: Only cache DecryptionMaterial. */ + needs(isDecryptionMaterial(material), 'Malformed response.') + /* Precondition: Only cache DecryptionMaterial that is cacheSafe. */ + needs(material.suite.cacheSafe, 'Can not cache non-cache safe material') + const entry = Object.seal({ + response: material, + bytesEncrypted: 0, + messagesEncrypted: 0, + now: clock(), + }) + + cache.set(key, entry, maxAge) + }, + + putBranchKeyMaterial( + key: string, + material: BranchKeyMaterial, + maxAge?: number + ): void { + /* Precondition: Only cache BranchKeyMaterial */ + needs(isBranchKeyMaterial(material), 'Malformed response.') + + const entry = Object.seal({ + response: material, + now: clock(), + }) + + cache.set(key, entry, maxAge) + }, + + getEncryptionMaterial(key: string, plaintextLength: number) { + /* Precondition: plaintextLength can not be negative. */ + needs(plaintextLength >= 0, 'Malformed plaintextLength') + const entry = cache.get(key) + /* Check for early return (Postcondition): If this key does not have an EncryptionMaterial, return false. */ + if (!entry) return false + /* Postcondition: Only return EncryptionMaterial. */ + needs(isEncryptionMaterial(entry.response), 'Malformed response.') + + const encryptionMaterialEntry = entry as EncryptionMaterialEntry + encryptionMaterialEntry.bytesEncrypted += plaintextLength + encryptionMaterialEntry.messagesEncrypted += 1 + + return entry as EncryptionMaterialEntry + }, + + getDecryptionMaterial(key: string) { + const entry = cache.get(key) + /* Check for early return (Postcondition): If this key does not have a DecryptionMaterial, return false. */ + if (!entry) return false + /* Postcondition: Only return DecryptionMaterial. */ + needs(isDecryptionMaterial(entry.response), 'Malformed response.') + + return entry as DecryptionMaterialEntry + }, + + getBranchKeyMaterial(key: string): BranchKeyMaterialEntry | false { + const entry = cache.get(key) + + /* Postcondition: If this key does not have a BranchKeyMaterial, return false */ + if (!entry) return false + + /* Postcondition: Only return BranchKeyMaterial */ + needs(isBranchKeyMaterial(entry.response), 'Malformed response.') + return entry as BranchKeyMaterialEntry + }, + + del(key: string) { + cache.del(key) + }, + } + + setClock(cmc, clock) + return cmc + + function mayEvictTail() { + // @ts-ignore + const { tail } = cache.dumpLru() + /* Check for early return (Postcondition) UNTESTED: If there is no tail, then the cache is empty. */ + if (!tail) return + /* The underlying Yallist tail Node has a `value`. + * This value is a lru-cache Entry and has a `key`. + */ + const { key } = tail.value + // Peek will evict, but not update the "recently used"-ness of the key. + cache.peek(key) + } +} diff --git a/modules/cache-material/test/cache_clock.test.ts b/modules/cache-material/test/cache_clock.test.ts new file mode 100644 index 000000000..8df211d99 --- /dev/null +++ b/modules/cache-material/test/cache_clock.test.ts @@ -0,0 +1,88 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/* eslint-env mocha */ + +import { expect } from 'chai' +import { + cacheEntryHasExceededLimits, + getEncryptionMaterials, +} from '../src/caching_cryptographic_materials_decorators' +import { localCryptographicMaterialsCache } from '../src/local_cryptographic_materials_cache' +import { getLocalCryptographicMaterialsCache } from '../src/get_local_cryptographic_materials_cache' +import { clockFor } from '../src/cache_clock' +import { buildCryptographicMaterialsCacheKeyHelpers } from '../src/build_cryptographic_materials_cache_key_helpers' +import { createHash, randomBytes } from 'crypto' +import { + AlgorithmSuiteIdentifier, + KeyringTraceFlag, + NodeAlgorithmSuite, + NodeEncryptionMaterial, + EncryptedDataKey, +} from '@aws-crypto/material-management' + +const suite = new NodeAlgorithmSuite( + AlgorithmSuiteIdentifier.ALG_AES128_GCM_IV12_TAG16_HKDF_SHA256 +) + +const cacheKeyHelpers = buildCryptographicMaterialsCacheKeyHelpers( + (input: string) => Buffer.from(input, 'utf8'), + (input: Uint8Array) => Buffer.from(input).toString('utf8'), + async (...data: (Uint8Array | string)[]) => + data + .map((item) => + typeof item === 'string' ? Buffer.from(item, 'hex') : item + ) + .reduce((hash, item) => hash.update(item), createHash('sha512')) + .digest() +) + +describe('cache clock', () => { + it('a cache from getLocalCryptographicMaterialsCache uses Date.now', () => { + expect(clockFor(getLocalCryptographicMaterialsCache(1))).to.equal(Date.now) + }) + + it('a caching CMM expires entries by the clock of its cache', async () => { + let now = 1000 + const cache = localCryptographicMaterialsCache(10, 60 * 1000, () => now) + let backingCalls = 0 + const cmm = { + _partition: 'partition', + _maxAge: 100, + _maxBytesEncrypted: 1000, + _maxMessagesEncrypted: 1000, + _cache: cache, + _backingMaterialsManager: { + async getEncryptionMaterials() { + backingCalls += 1 + return new NodeEncryptionMaterial(suite, {}) + .setUnencryptedDataKey(randomBytes(16), { + keyNamespace: 'k', + keyName: 'k', + flags: KeyringTraceFlag.WRAPPING_KEY_GENERATED_DATA_KEY, + }) + .addEncryptedDataKey( + new EncryptedDataKey({ + providerId: 'k', + providerInfo: 'k', + encryptedDataKey: new Uint8Array([1]), + }), + KeyringTraceFlag.WRAPPING_KEY_ENCRYPTED_DATA_KEY + ) + }, + }, + _cacheEntryHasExceededLimits: cacheEntryHasExceededLimits(), + getEncryptionMaterials: getEncryptionMaterials(cacheKeyHelpers), + } as any + const request = { suite, encryptionContext: {}, plaintextLength: 1 } + + await cmm.getEncryptionMaterials(request) + now += 100 + await cmm.getEncryptionMaterials(request) + expect(backingCalls).to.equal(1) + + now += 1 + await cmm.getEncryptionMaterials(request) + expect(backingCalls).to.equal(2) + }) +}) diff --git a/test-server/src/bridge.ts b/test-server/src/bridge.ts index 43aad7059..4f44ce881 100644 --- a/test-server/src/bridge.ts +++ b/test-server/src/bridge.ts @@ -18,7 +18,6 @@ import { buildEncrypt, CommitmentPolicy, getKmsClient, - getLocalCryptographicMaterialsCache, KeyringNode, KmsHierarchicalKeyRingNode, KmsKeyringNode, @@ -31,6 +30,8 @@ import { RawRsaKeyringNode, WrappingSuiteIdentifier, } from '@aws-crypto/client-node' +/* Not exported from the package index: builds a cache with a clock the test server controls. */ +import { localCryptographicMaterialsCache } from '@aws-crypto/cache-material/build/main/src/local_cryptographic_materials_cache' import { constants } from 'crypto' import { Readable } from 'stream' import { ClientError, ServerError } from './errors' @@ -50,7 +51,6 @@ import { MultiKeyringConfig, RawAesKeyringConfig, RawRsaKeyringConfig, - TestHooks, toSuiteId, } from './model' @@ -62,22 +62,19 @@ export interface CallCounts { decryptMaterials: number } -/* Test-only wrapper around a caching CMM's underlying CMM. - * Counts the caching CMM's calls to it, and can delay each call - * so that concurrent cache misses overlap. +/* Test-only wrapper around a caching CMM's underlying CMM, + * counting the caching CMM's calls to it. */ -class InstrumentedMaterialsManager implements NodeMaterialsManager { +class CountingMaterialsManager implements NodeMaterialsManager { constructor( private readonly inner: NodeMaterialsManager, - private readonly counts: CallCounts, - private readonly delayMilliseconds: number + private readonly counts: CallCounts ) {} async getEncryptionMaterials( request: Parameters[0] ) { this.counts.getEncryptionMaterials += 1 - await this.delay() return this.inner.getEncryptionMaterials(request) } @@ -85,22 +82,20 @@ class InstrumentedMaterialsManager implements NodeMaterialsManager { request: Parameters[0] ) { this.counts.decryptMaterials += 1 - await this.delay() return this.inner.decryptMaterials(request) } - - private async delay() { - if (this.delayMilliseconds > 0) { - await new Promise((resolve) => - setTimeout(resolve, this.delayMilliseconds) - ) - } - } } -interface Instrumentation { - counts: CallCounts - delayMilliseconds: number +/* Test-only state for one client: call counts, and the clock its caches use. + * The clock is the system time plus `clockOffsetMilliseconds`, which AdvanceClock moves forward. + */ +export class Instrumentation { + readonly counts: CallCounts = { + getEncryptionMaterials: 0, + decryptMaterials: 0, + } + clockOffsetMilliseconds = 0 + readonly clock = () => Date.now() + this.clockOffsetMilliseconds } export interface OperationResult { @@ -316,12 +311,15 @@ function buildCmm( case 'Caching': { const cfg = config as CachingCmmConfig return new NodeCachingMaterialsManager({ - backingMaterials: new InstrumentedMaterialsManager( + backingMaterials: new CountingMaterialsManager( buildCmm(cfg.underlyingCMM, instrumentation), - instrumentation.counts, - instrumentation.delayMilliseconds + instrumentation.counts + ), + cache: localCryptographicMaterialsCache( + 100, + 60 * 1000, + instrumentation.clock ), - cache: getLocalCryptographicMaterialsCache(100), maxAge: cfg.cacheLimitTtlSeconds * 1000, partition: cfg.partitionId, maxBytesEncrypted: cfg.limitBytes, @@ -341,7 +339,7 @@ export class EsdkClientBundle { constructor( private readonly client: Client, private readonly cmm: NodeMaterialsManager, - readonly callCounts: CallCounts + readonly instrumentation: Instrumentation ) {} async encrypt( @@ -441,16 +439,10 @@ async function collect(stream: NodeJS.ReadableStream): Promise { return Buffer.concat(chunks) } -export function buildClientBundle( - config: EsdkClientConfig, - testHooks: TestHooks = {} -): EsdkClientBundle { +export function buildClientBundle(config: EsdkClientConfig): EsdkClientBundle { const policy = commitmentPolicy(config.commitmentPolicy) - const callCounts = { getEncryptionMaterials: 0, decryptMaterials: 0 } - const cmm = buildCmm(config.cmm, { - counts: callCounts, - delayMilliseconds: testHooks.cachingCmmUnderlyingDelayMilliseconds || 0, - }) + const instrumentation = new Instrumentation() + const cmm = buildCmm(config.cmm, instrumentation) const client = buildClient({ commitmentPolicy: policy, maxEncryptedDataKeys: @@ -459,5 +451,5 @@ export function buildClientBundle( ? false : config.maxEncryptedDataKeys, }) - return new EsdkClientBundle(client, cmm, callCounts) + return new EsdkClientBundle(client, cmm, instrumentation) } diff --git a/test-server/src/model.ts b/test-server/src/model.ts index 2435680f1..7adfe3d19 100644 --- a/test-server/src/model.ts +++ b/test-server/src/model.ts @@ -109,11 +109,6 @@ export interface EsdkClientConfig { export interface CreateClientRequest { config?: EsdkClientConfig - testHooks?: TestHooks -} - -export interface TestHooks { - cachingCmmUnderlyingDelayMilliseconds?: number } export interface EncryptRequest { @@ -140,6 +135,23 @@ export interface GetCallCountsRequest { clientId?: string } +export interface EncryptConcurrentlyRequest { + clientId?: string + plaintexts?: Uint8Array[] + encryptionContext?: EncryptionContext + algorithmSuiteId?: string +} + +export interface DecryptConcurrentlyRequest { + clientId?: string + ciphertexts?: Uint8Array[] +} + +export interface AdvanceClockRequest { + clientId?: string + milliseconds?: number +} + /* Modeled ESDKAlgorithmSuiteId name -> library AlgorithmSuiteIdentifier. */ const SUITE_BY_MODEL_NAME: { [name: string]: AlgorithmSuiteIdentifier } = { ALG_AES_128_GCM_IV12_TAG16_NO_KDF: diff --git a/test-server/src/server.ts b/test-server/src/server.ts index 9b4409410..0ec0583f6 100644 --- a/test-server/src/server.ts +++ b/test-server/src/server.ts @@ -28,6 +28,9 @@ import { EncryptRequest, EncryptStreamRequest, GetCallCountsRequest, + EncryptConcurrentlyRequest, + DecryptConcurrentlyRequest, + AdvanceClockRequest, } from './model' const SMITHY_PROTOCOL = 'rpc-v2-cbor' @@ -63,7 +66,7 @@ async function createClient( if (!config) throw new ServerError('config is required') let bundle: EsdkClientBundle try { - bundle = buildClientBundle(config, request.testHooks) + bundle = buildClientBundle(config) } catch (err) { if (err instanceof ClientError || err instanceof ServerError) throw err throw new ServerError( @@ -155,14 +158,51 @@ async function dispatch( ) } case 'GetCallCounts': { - const { callCounts } = registry.resolve( + const { counts } = registry.resolve( (request as GetCallCountsRequest).clientId - ) + ).instrumentation return { - cachingCmmGetEncryptionMaterialsCalls: - callCounts.getEncryptionMaterials, - cachingCmmDecryptMaterialsCalls: callCounts.decryptMaterials, + cachingCmmGetEncryptionMaterialsCalls: counts.getEncryptionMaterials, + cachingCmmDecryptMaterialsCalls: counts.decryptMaterials, + } + } + case 'EncryptConcurrently': { + const req = request as EncryptConcurrentlyRequest + const bundle = registry.resolve(req.clientId) + if (!req.plaintexts) throw new ServerError('plaintexts is required') + // Start every encrypt before awaiting any, so they reach the caches together. + const results = await Promise.all( + req.plaintexts.map(async (plaintext) => + delegated( + bundle.encrypt( + plaintext, + req.encryptionContext, + req.algorithmSuiteId + ) + ) + ) + ) + return { ciphertexts: results.map(({ data }) => data) } + } + case 'DecryptConcurrently': { + const req = request as DecryptConcurrentlyRequest + const bundle = registry.resolve(req.clientId) + if (!req.ciphertexts) throw new ServerError('ciphertexts is required') + const results = await Promise.all( + req.ciphertexts.map(async (ciphertext) => + delegated(bundle.decrypt(ciphertext)) + ) + ) + return { plaintexts: results.map(({ data }) => data) } + } + case 'AdvanceClock': { + const req = request as AdvanceClockRequest + const { instrumentation } = registry.resolve(req.clientId) + if (typeof req.milliseconds !== 'number' || req.milliseconds < 0) { + throw new ServerError('milliseconds must be a non-negative number') } + instrumentation.clockOffsetMilliseconds += req.milliseconds + return {} } default: throw new ServerError(`unknown operation: ${operation}`) From 19a0fd1800596635b948ed88b492d92d9f8e6c53 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Oct 2026 22:25:34 +0000 Subject: [PATCH 4/4] fix(test-server): use the orchestrator at test-server-common in commons --- test-server/Makefile | 7 ++++--- test-server/test/server.test.ts | 35 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/test-server/Makefile b/test-server/Makefile index 89927c715..dfd6d6fbb 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -18,7 +18,8 @@ REPO_ROOT := $(abspath $(MAKEFILE_DIR)/..) COMMONS_CONFIGURATION := $(MAKEFILE_DIR)/server-config.json COMMONS_BRANCH ?= CLONE_DIR ?= $(MAKEFILE_DIR)/.commons-clone -CLONE_ORCH_DIR := $(CLONE_DIR)/esdk/test-server/orchestrator +CLONE_ORCH_DIR := $(CLONE_DIR)/test-server-common/orchestrator +CLONE_ROOT_DIR := $(CLONE_DIR)/esdk/test-server .PHONY: help build-server run-server start-server wait-for-server stop-server \ test test-server check-java clean @@ -88,10 +89,10 @@ test-server: check-java ## Run the complete cross-language TestServer via the co echo "ERROR: failed to clone $$url at branch $$branch; no Tests will run." >&2; exit 1; \ fi; \ if [ ! -d "$(CLONE_ORCH_DIR)" ]; then \ - echo "ERROR: branch $$branch of $$url has no orchestrator at esdk/test-server/orchestrator." >&2; exit 1; \ + echo "ERROR: branch $$branch of $$url has no orchestrator at test-server-common/orchestrator." >&2; exit 1; \ fi; \ echo "==> Delegating: context=language:javascript languageRepoRoot=$(REPO_ROOT)"; \ - cd "$(CLONE_ORCH_DIR)" && ./gradlew --console=plain run \ + cd "$(CLONE_ORCH_DIR)" && ./gradlew --console=plain run -Dtestserver.root="$(CLONE_ROOT_DIR)" \ --args="context=language:javascript languageRepoRoot=$(REPO_ROOT) commonsOrigin.url=$$url commonsOrigin.branch=$$branch commonsOrigin.reason=$$reason" clean: ## Remove build output, server scratch files, and the commons clone diff --git a/test-server/test/server.test.ts b/test-server/test/server.test.ts index 66efae140..5a337ba85 100644 --- a/test-server/test/server.test.ts +++ b/test-server/test/server.test.ts @@ -426,6 +426,41 @@ describe('client construction', () => { }) }) + it('AdvanceClock expires caching CMM entries', async () => { + const base = rawAesConfig() as { cmm: CborValue; commitmentPolicy: string } + const clientId = await createClient({ + commitmentPolicy: base.commitmentPolicy, + cmm: { Caching: { underlyingCMM: base.cmm, cacheLimitTtlSeconds: 60 } }, + }) + await operation('Encrypt', { clientId, plaintext: PLAINTEXT }) + const advanced = await operation('AdvanceClock', { + clientId, + milliseconds: 61000, + }) + expect(advanced.status).to.equal(200) + await operation('Encrypt', { clientId, plaintext: PLAINTEXT }) + + const counts = await operation('GetCallCounts', { clientId }) + expect(counts.body.cachingCmmGetEncryptionMaterialsCalls).to.equal(2) + }) + + it('EncryptConcurrently and DecryptConcurrently round-trip every message', async () => { + const clientId = await createClient(rawAesConfig()) + const plaintexts = [PLAINTEXT, Buffer.from('second'), Buffer.from('third')] + const encrypted = await operation('EncryptConcurrently', { + clientId, + plaintexts, + }) + expect(encrypted.status).to.equal(200) + const decrypted = await operation('DecryptConcurrently', { + clientId, + ciphertexts: encrypted.body.ciphertexts, + }) + expect( + (decrypted.body.plaintexts as Uint8Array[]).map((p) => Buffer.from(p)) + ).to.deep.equal(plaintexts) + }) + it('GetCallCounts reports zero for a client with no caching CMM', async () => { const clientId = await createClient(rawAesConfig()) await operation('Encrypt', { clientId, plaintext: PLAINTEXT })