From b6523a145a9adc0381f935711de1433f445b85b3 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 19 Oct 2026 10:35:28 +1100 Subject: [PATCH 1/4] feat(eql): let an EQL v4 text search query carry only the terms its operator reads eql_v4.query_text_search required hm, op and bf on every query value. bf is the bloom filter built from 3-grams, so text shorter than one 3-gram has none, and `col = 'Al'` or `col < 'Al'` could not be written against an eql_v4_text_search column. That is why Stack Encrypt could not produce the type. In v4 the search query domains (query_text_search and its _ore twin) now require at least one of their terms. Each extractor over them reads its term through eql_v4_internal.require_query_term, which raises invalid_parameter_value naming the domain, the term and the operators that compare it when the term is absent, so `=` on a value with no hm, `<` with no op and `@@` with no bf are refused rather than reading NULL and matching nothing. The extractors stay LANGUAGE sql, so the wrappers still inline and the eq_term/ord_term/match_term indexes still engage. v3 is cipherstash-client's format and does not change. The SQL source is rendered once per version and had no way to say "only in v4", so the renderer gains version blocks: lines between `-- {{#v3}}`/`-- {{/v3}}` or `-- {{#v4}}`/`-- {{/v4}}` render for that version only, and the directive lines for neither. The generator emits them for the two search query domains, and the helper sits in a v4 block of src/v3/scalars/functions.sql. release/cipherstash-encrypt.sql and its uninstaller are byte-identical to the build before this change. ADR-0002 gains a dated amendment recording the decision. --- .../eql-v4-text-search-optional-terms.md | 5 + ...-three-encodings-and-eql-v4-by-producer.md | 29 +++ packages/eql/AGENTS.md | 2 + .../eql/crates/eql-codegen/src/context.rs | 97 ++++++++- .../eql/crates/eql-codegen/src/generate.rs | 127 +++++++++++- packages/eql/crates/eql-codegen/src/render.rs | 189 ++++++++++++++++++ .../templates/functions/extractor.sql.j2 | 14 ++ .../eql-codegen/templates/query_types.sql.j2 | 22 ++ packages/eql/src/v3/scalars/functions.sql | 46 +++++ .../text/query_text_search_functions.sql | 25 +++ .../text/query_text_search_ore_functions.sql | 25 +++ .../src/v3/scalars/text/query_text_types.sql | 16 ++ .../sqlx/tests/v3_v4_coexistence_tests.rs | 99 +++++++++ 13 files changed, 684 insertions(+), 12 deletions(-) create mode 100644 .changeset/eql-v4-text-search-optional-terms.md diff --git a/.changeset/eql-v4-text-search-optional-terms.md b/.changeset/eql-v4-text-search-optional-terms.md new file mode 100644 index 000000000..36ac34674 --- /dev/null +++ b/.changeset/eql-v4-text-search-optional-terms.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**An EQL v4 text search query carries only the terms its operator reads.** `eql_v4.query_text_search` and `eql_v4.query_text_search_ore` now require at least one of their terms (`hm`, `op` or `ob`, `bf`) instead of all of them, so `col = $1` and `col < $1` work on text too short to cut a 3-gram, which has no match term. Each extractor over such a query raises (`invalid_parameter_value`) when the query lacks the term it reads, naming the domain, the term and the operators that compare it: `=` without `hm`, `<` without `op`, `@@` without `bf`. A missing term never matches no row silently, nor every row. Stored `eql_v4_text_search` values still carry every term. The `eql_v3` domains are unchanged: the v3 bundle is byte for byte what it was. diff --git a/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md b/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md index 8230b207b..e27627aa2 100644 --- a/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md +++ b/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md @@ -443,6 +443,35 @@ without loss, and refuses it otherwise. - **During the overlap**, expected to last a quarter or more, cipherstash-client takes fixes only. New kinds and domains are produced through Stack Encrypt and v4. An exception is a decision written down in its issue. +- *Amended 2026-10-19:* **a v4 search query carries only the terms its + operator reads.** The text search query domains (`eql_v4.query_text_search` + and `eql_v4.query_text_search_ore`, whose terms include the match term + `bf` beside others) require at least one of their terms rather than all + of them. Text shorter than one 3-gram has no match term, so a query that + had to carry every term could not express `=` or `<` on short text. + Stack Encrypt emits only the term of the index that answers the query's + operator. Every extractor over such an operand raises wherever it reads + the operand when the term it reads is absent, JSON `null` or an empty + array, naming the domain, the term and the operators that compare it, and + never the operand's contents, rather than reading it as `NULL` (no row) or + as an empty filter. The raise is at evaluation: a literal operand is + folded and raises at plan time, while a bound parameter under a generic + plan whose predicate is never reached returns no row and no error. A + stored search value carries every term, and text too short for a match + term stores the empty filter (`"bf": []`), the stored side's answer in + Stack Encrypt: `=` and `<` find it, and no `@@` query does, because a + query's filter is never empty (the extractor refuses `"bf": []`). That is + a claim about queries: `@@` between two stored values (a self-join + `a.name @@ b.name`, or `matches(col, )`) keeps + the shared guard, so two stored empty filters match each other there, as + they do in v3. No producer builds that predicate. A match + query of such text is refused, as it is for `TextMatch`, whose stored + value still refuses it too. The v3 domains are cipherstash-client's format + and keep requiring every term, byte for byte, so here the v4 SQL differs + from the v3 SQL. The one source holds both forms between version-block directive lines (`-- {{#v3}}` … `-- {{/v3}}`, + `-- {{#v4}}` … `-- {{/v4}}`), which the renderer keeps for their own + version and drops from the other, so the v3 bundle renders exactly as it + did before the block was written. ### Review diff --git a/packages/eql/AGENTS.md b/packages/eql/AGENTS.md index a4477f0df..45d907f66 100644 --- a/packages/eql/AGENTS.md +++ b/packages/eql/AGENTS.md @@ -54,6 +54,8 @@ The SQL source never names its schemas or its payload version literally. It writ `{{prefix}}` is the schema and the start of every other name: `{{prefix}}.eq_term`, `{{prefix}}_internal.hmac_256`, `public.{{prefix}}_text_eq`. `{{eql_version}}` appears in every domain CHECK: `VALUE->>'v' = '{{eql_version}}'`. This covers every file under `src/v3/` (hand-written and generated), `src/v3/version.template`, `tasks/pin_search_path_v3.sql` and `tasks/uninstall-v3.sql`. The generator emits the placeholders too: `SCHEMA`, `INTERNAL_SCHEMA`, `PUBLIC_TYPNAME_PREFIX` and `EQL_VERSION` in `crates/eql-codegen/src/consts.rs` are placeholder strings. The Rust bindings are the exception. They describe v3 payloads and keep `eql_domains::PUBLIC_TYPNAME_PREFIX`, the literal `eql_v3_`. +Where the two versions' SQL must differ, the source holds both forms in **version blocks**: lines between `-- {{#v3}}` and `-- {{/v3}}` render only for `eql_v3`, lines between `-- {{#v4}}` and `-- {{/v4}}` only for `eql_v4`, and the directive lines themselves render for neither, so a v4-only block leaves the v3 bundle byte for byte as it was. A directive is a whole line; blocks do not nest. The generator emits them through the `v3_only`/`v3_end`/`v4_only`/`v4_end` template globals. Today they hold the v4 search query operands' optional terms (`query_text_search`, `query_text_search_ore`: at least one of their terms, and each extractor raising through `{{prefix}}_internal.require_query_term`, which lives in a v4 block of `src/v3/scalars/functions.sql`). Diff `release/cipherstash-encrypt.sql` before and after any block you add. + `tasks/build.sh` renders the ordered source into `build/render/eql_v3/` and `build/render/eql_v4/` (gitignored, same relative paths) with `eql-codegen render`, runs the symbol-order and installer-completeness gates on the rendered files, and assembles `release/` from them. The renderer refuses three things: an unknown placeholder, an unterminated `{{`, and a **literal `eql_v3` or `eql_v4`** anywhere in the source, comments included. A literal name renders identically for both versions, so the v4 bundle would reach into v3. Write the placeholder in prose too. #### Files confined to one EQL version diff --git a/packages/eql/crates/eql-codegen/src/context.rs b/packages/eql/crates/eql-codegen/src/context.rs index a34cbd69b..392507933 100644 --- a/packages/eql/crates/eql-codegen/src/context.rs +++ b/packages/eql/crates/eql-codegen/src/context.rs @@ -2,6 +2,7 @@ use crate::consts::*; use crate::operator_surface::Operator; +use crate::render::{block_close, block_open, EqlVersion}; use eql_domains::{Domain, Term}; /// Build the minijinja environment with the embedded templates: one whole-file @@ -72,9 +73,70 @@ pub fn environment() -> minijinja::Environment<'static> { env.add_global("schema", SCHEMA); env.add_global("internal_schema", INTERNAL_SCHEMA); env.add_global("eql_version", EQL_VERSION); + // The renderer's version-block directives, for SQL that differs between + // `eql_v3` and `eql_v4` (`crate::render`, "Version blocks"). + env.add_global("v3_only", block_open(EqlVersion::V3)); + env.add_global("v3_end", block_close(EqlVersion::V3)); + env.add_global("v4_only", block_open(EqlVersion::V4)); + env.add_global("v4_end", block_close(EqlVersion::V4)); env } +/// Whether an EQL v4 query of `domain` carries only the terms its operator +/// reads (ADR-0002, amended 2026-10-19): a domain whose terms include the +/// match term beside others, `text_search` and `text_search_ore`. Text +/// shorter than one 3-gram has no match term, so a query that had to carry +/// every term could not express `=` or `<` on it. +/// +/// EQL v4 only. The v4 query domain then requires at least one of its terms, +/// and each extractor over it raises when the term it reads is absent +/// ([`required_term`]); the `eql_v3` domain is cipherstash-client's format and +/// keeps requiring every term. The v4 binding's query struct holds each term +/// as an `Option`. +pub fn v4_optional_query_terms(domain: &Domain) -> bool { + domain.terms.contains(&Term::Bloom) && Term::payload_terms(domain.terms).len() > 1 +} + +/// The term an extractor over a query operand reads, for an operand whose +/// terms are optional in EQL v4 ([`v4_optional_query_terms`]): the extractor +/// raises, naming the term and the operators that read it, when the operand +/// carries none, rather than answer `NULL` and match no row. +#[derive(serde::Serialize)] +pub struct RequiredTerm { + /// sql_str-escaped JSON key, e.g. `hm`. + pub key: String, + /// sql_str-escaped schema-qualified query domain, for the message. + pub domain: String, + /// sql_str-escaped operators the term answers on this domain, as prose: + /// `= and <>`, `<, <=, > and >=`, `@@`. + pub operators: String, +} + +/// The [`RequiredTerm`] for `term`'s extractor over `domain`'s query operand +/// `query_dom`, or `None` when the operand's terms are not optional. +pub fn required_term(domain: &Domain, term: Term, query_dom: &str) -> Option { + if !v4_optional_query_terms(domain) { + return None; + } + // The operators this extractor answers on this domain: an operator two + // terms support is read through the first (`=` on `text_search` reads + // `hm`, not `op`). + let ops: Vec<&str> = Term::operators_for_terms(domain.terms) + .into_iter() + .filter(|op| Term::extractor_for_operator(domain.terms, op) == Some(term.extractor())) + .collect(); + let operators = match ops.as_slice() { + [] => "no operator".to_string(), + [one] => (*one).to_string(), + [rest @ .., last] => format!("{} and {last}", rest.join(", ")), + }; + Some(RequiredTerm { + key: sql_str(term.json_key()), + domain: sql_str(query_dom), + operators: sql_str(&operators), + }) +} + /// One idempotent CREATE DOMAIN block, with SQL-required values precomputed. #[derive(serde::Serialize)] pub struct DomainBlock { @@ -91,6 +153,13 @@ pub struct DomainBlock { // query operand is index-terms-only). The template renders a // `NOT (VALUE ? k)` clause per key. pub forbidden_keys: Vec, + // sql_str-escaped term keys of a query twin whose terms are optional in + // EQL v4 (`v4_optional_query_terms`); empty otherwise. When set, `keys` + // holds the envelope keys alone, and the template renders the term keys + // as required for `eql_v3` and as "at least one" for `eql_v4`, each in + // its own version block, with a present `nonempty_array_keys` key still + // a non-empty array. + pub optional_term_keys: Vec, // sql_str-escaped one-line human description rendered as `COMMENT ON DOMAIN`. // Capability text is derived from the domain's terms (via // `Term::operators_for_terms`), so it can't drift from the generated CHECK / @@ -171,6 +240,8 @@ pub fn domain_block(family_name: &str, domain: &Domain) -> DomainBlock { .collect(), // Storage domains forbid nothing; the query twin forbids `c`. forbidden_keys: vec![], + // A stored value carries every term in both versions. + optional_term_keys: vec![], comment: scalar_domain_comment(family_name, domain), } } @@ -188,9 +259,13 @@ pub fn query_domain_block(family_name: &str, domain: &Domain) -> DomainBlock { .filter(|&&k| k != "c") .map(|k| sql_str(k)) .collect(); - for k in Term::term_json_keys(domain.terms) { - keys.push(sql_str(k)); - } + let term_keys = Term::term_json_keys(domain.terms).into_iter().map(sql_str); + let optional_term_keys = if v4_optional_query_terms(domain) { + term_keys.collect() + } else { + keys.extend(term_keys); + vec![] + }; DomainBlock { typname: sql_str(&name), @@ -201,6 +276,7 @@ pub fn query_domain_block(family_name: &str, domain: &Domain) -> DomainBlock { .map(sql_str) .collect(), forbidden_keys: vec![sql_str("c")], + optional_term_keys, comment: query_domain_comment(family_name, domain), } } @@ -223,6 +299,9 @@ pub enum FnEntry { ret: String, // e.g. eql_v3_internal.hmac_256 (selection STAYS in Rust) extractor: String, // e.g. eq_term ctor: String, // e.g. hmac_256 (called as {{ internal_schema }}.{{ ctor }}) + // Set on an extractor over a query operand whose terms are optional + // in EQL v4: the v4 body raises when the term is absent. + required: Option, }, Wrapper { op: String, // SQL operator used in the body, e.g. = @@ -260,10 +339,22 @@ pub struct FunctionsContext { /// the body's constructor call — so the declared return type and the call /// stay in lockstep. `Term::returns()` is intentionally not used. pub fn extractor_entry(term: Term) -> FnEntry { + extractor_entry_requiring(term, None) +} + +/// [`extractor_entry`] over a query operand: in EQL v4 it raises when the +/// operand lacks the term, if the operand's terms are optional there +/// ([`required_term`]). +pub fn query_extractor_entry(domain: &Domain, term: Term, query_dom: &str) -> FnEntry { + extractor_entry_requiring(term, required_term(domain, term, query_dom)) +} + +fn extractor_entry_requiring(term: Term, required: Option) -> FnEntry { FnEntry::Extractor { ret: format!("{INTERNAL_SCHEMA}.{}", term.ctor()), extractor: term.extractor().to_string(), ctor: term.ctor().to_string(), + required, } } diff --git a/packages/eql/crates/eql-codegen/src/generate.rs b/packages/eql/crates/eql-codegen/src/generate.rs index b6df602af..ce35b1cb7 100644 --- a/packages/eql/crates/eql-codegen/src/generate.rs +++ b/packages/eql/crates/eql-codegen/src/generate.rs @@ -258,8 +258,8 @@ fn query_types_path(family_name: &str) -> String { pub fn render_query_functions_file(family_name: &str, domain: &Domain) -> String { use crate::consts::sql_str; use crate::context::{ - domain_name, environment, extractor_entry, query_domain_name, wrapper_entry, - FunctionsContext, + domain_name, environment, query_domain_name, query_extractor_entry, + v4_optional_query_terms, wrapper_entry, FunctionsContext, }; let name = domain.full_name(family_name); let query_name = domain.query_name(family_name); @@ -269,9 +269,10 @@ pub fn render_query_functions_file(family_name: &str, domain: &Domain) -> String let mut entries = Vec::new(); // Extractor overloads on the query domain (the template renders `a {{ dom }}` - // with dom = the query domain). + // with dom = the query domain). Over an operand whose terms are optional + // in EQL v4, each raises there when its term is absent. for term in Term::extractor_terms(domain.terms) { - entries.push(extractor_entry(term)); + entries.push(query_extractor_entry(domain, term, &query_dom)); } // Comparison wrappers: (storage, query) and its (query, storage) commutator, // for supported operators only (a query operand is never sent for a blocked @@ -307,12 +308,18 @@ pub fn render_query_functions_file(family_name: &str, domain: &Domain) -> String )); } + let mut requires = vec![ + V3_SCHEMA.to_string(), + query_types_path(family_name), + scalar_path(family_name, &format!("{name}_functions.sql")), + ]; + // The v4 extractors over an operand with optional terms raise through the + // shared helper there. + if v4_optional_query_terms(domain) { + requires.push(V3_SCALARS_BLOCKER.to_string()); + } let ctx = FunctionsContext { - requires: vec![ - V3_SCHEMA.to_string(), - query_types_path(family_name), - scalar_path(family_name, &format!("{name}_functions.sql")), - ], + requires, family_name: family_name.to_string(), name: query_name, domain_lit: sql_str(&query_dom), @@ -2529,4 +2536,106 @@ mod tests { ); } } + + /// The CHECK of `name`'s CREATE DOMAIN in `sql`, rendered as `prefix`. + fn domain_check(sql: &str, prefix: &str, name: &str) -> String { + let start = sql + .find(&format!("CREATE DOMAIN {prefix}.{name} AS")) + .unwrap_or_else(|| panic!("{prefix}.{name}")); + let end = start + sql[start..].find(");").expect("CHECK closes"); + sql[start..end].to_string() + } + + /// EQL v4's search query operands carry only the terms their operator + /// reads (ADR-0002, amended 2026-10-19); eql_v3's keep requiring every + /// term, as before. + #[test] + fn a_v4_search_query_operand_needs_one_term_and_v3_needs_all() { + use crate::render::{render_str, EqlVersion}; + let text = spec("text"); + let types = super::render_query_types_file(text); + let v3 = render_str(&types, EqlVersion::V3).unwrap(); + let v4 = render_str(&types, EqlVersion::V4).unwrap(); + + let search3 = domain_check(&v3, "eql_v3", "query_text_search"); + for key in ["hm", "op", "bf"] { + assert!( + search3.contains(&format!("AND VALUE ? '{key}'")), + "{search3}" + ); + } + let search4 = domain_check(&v4, "eql_v4", "query_text_search"); + assert!( + search4.contains("AND (VALUE ? 'hm' OR VALUE ? 'op' OR VALUE ? 'bf')"), + "{search4}" + ); + assert!(!search4.contains("AND VALUE ? 'hm'"), "{search4}"); + assert!( + search4.contains("AND VALUE ? 'v'") && search4.contains("NOT (VALUE ? 'c')"), + "{search4}" + ); + // The ORE twin: a present `ob` is still a non-empty array. + let ore3 = domain_check(&v3, "eql_v3", "query_text_search_ore"); + assert!(ore3.contains("AND VALUE ? 'ob'"), "{ore3}"); + assert!( + ore3.contains("AND jsonb_typeof(VALUE -> 'ob') = 'array'"), + "{ore3}" + ); + let ore4 = domain_check(&v4, "eql_v4", "query_text_search_ore"); + assert!( + ore4.contains("AND (VALUE ? 'hm' OR VALUE ? 'ob' OR VALUE ? 'bf')") + && ore4 + .contains("AND (NOT (VALUE ? 'ob') OR (jsonb_typeof(VALUE -> 'ob') = 'array'"), + "{ore4}" + ); + // Every other query operand is the same in both versions. + for other in ["query_text_eq", "query_text_match", "query_text_ord"] { + assert_eq!( + domain_check(&v3, "eql_v3", other) + .replace("eql_v3", "eql_vN") + .replace("'3'", "'N'"), + domain_check(&v4, "eql_v4", other) + .replace("eql_v4", "eql_vN") + .replace("'4'", "'N'"), + "{other}" + ); + } + + // Each v4 extractor over the operand raises without its term; v3's + // reads the term as before. + let functions = super::render_query_functions_file("text", domain(text, "search")); + let f3 = render_str(&functions, EqlVersion::V3).unwrap(); + let f4 = render_str(&functions, EqlVersion::V4).unwrap(); + assert!(!f3.contains("require_query_term"), "{f3}"); + assert!(f3.contains("AS $$ SELECT eql_v3_internal.hmac_256(a::jsonb) $$;")); + for (ctor, key, ops) in [ + ("hmac_256", "hm", "= and <>"), + ("ope_cllw", "op", "<, <=, > and >="), + ("bloom_filter", "bf", "@@"), + ] { + let body = format!( + "SELECT eql_v4_internal.{ctor}(eql_v4_internal.require_query_term(a::jsonb, \ + 'eql_v4.query_text_search', '{key}', '{ops}'))" + ); + assert!(f4.contains(&body), "{body} in {f4}"); + } + // A query operand whose terms are not optional reads them directly, + // in one form for both versions. + let eq = super::render_query_functions_file("text", domain(text, "eq")); + assert!(!eq.contains("require_query_term") && !eq.contains("{{#v")); + } + + #[test] + fn only_the_search_domains_have_optional_v4_query_terms() { + let optional: Vec = CATALOG + .iter() + .flat_map(|f| { + f.domains + .iter() + .filter(|d| d.is_scalar() && crate::context::v4_optional_query_terms(d)) + .map(move |d| d.full_name(f.name)) + }) + .collect(); + assert_eq!(optional, ["text_search_ore", "text_search"]); + } } diff --git a/packages/eql/crates/eql-codegen/src/render.rs b/packages/eql/crates/eql-codegen/src/render.rs index 6f39f3b01..2441ce360 100644 --- a/packages/eql/crates/eql-codegen/src/render.rs +++ b/packages/eql/crates/eql-codegen/src/render.rs @@ -25,6 +25,26 @@ //! `between`, ADR-0009 decision 7) shares the one source tree: the build //! renders a file only for the version it belongs to ([`belongs_to`]), and //! [`render_str`] refuses to render it as any other. +//! # Version blocks +//! +//! Where one version's SQL must differ from the other's, the source holds both +//! forms, each between two directive lines (ADR-0002, amended 2026-10-19): +//! +//! ```sql +//! -- {{#v3}} +//! AND VALUE ? 'hm' +//! -- {{/v3}} +//! -- {{#v4}} +//! AND (VALUE ? 'hm' OR VALUE ? 'op') +//! -- {{/v4}} +//! ``` +//! +//! A block's lines are kept in its own version's output and dropped from the +//! other's, and the directive lines are dropped from both, so a source with a +//! block renders, for the version the block does not name, exactly as it did +//! before the block was written. A directive is a whole line (leading +//! whitespace allowed); blocks do not nest, a close must name the open block's +//! version, and a block left open at the end of the file is an error. use std::fmt; use std::fs; @@ -42,6 +62,47 @@ pub const EQL_VERSION: &str = "{{eql_version}}"; /// to every version. The number is the payload version ([`EqlVersion::number`]), /// not the schema name, because the source may not name a schema literally. pub const ONLY_DIRECTIVE: &str = "-- ONLY EQL VERSION:"; +/// The directive line that opens a block kept only in `version`'s output: +/// `-- {{#v4}}`. See [the module docs](self#version-blocks). +pub fn block_open(version: EqlVersion) -> String { + format!("-- {{{{#v{}}}}}", version.number()) +} + +/// The directive line that closes a block [`block_open`] opened: `-- {{/v4}}`. +pub fn block_close(version: EqlVersion) -> String { + format!("-- {{{{/v{}}}}}", version.number()) +} + +/// A version-block directive line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Directive { + Open(EqlVersion), + Close(EqlVersion), +} + +/// The directive `line` is, if it is one: `-- {{#vN}}` or `-- {{/vN}}` for a +/// version this renderer knows, alone on the line. +fn directive(line: &str) -> Option { + let inner = line + .trim() + .strip_prefix("--")? + .trim_start() + .strip_prefix("{{")? + .strip_suffix("}}")?; + let (open, name) = if let Some(name) = inner.strip_prefix('#') { + (true, name) + } else { + (false, inner.strip_prefix('/')?) + }; + let version = EqlVersion::ALL + .into_iter() + .find(|v| name.strip_prefix('v') == Some(v.number()))?; + Some(if open { + Directive::Open(version) + } else { + Directive::Close(version) + }) +} /// An EQL version the source can be rendered as. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -103,6 +164,13 @@ pub enum RenderError { only: EqlVersion, version: EqlVersion, }, + /// A version block opened inside another; blocks do not nest. + NestedBlock { line: usize }, + /// A block close with no open block, or naming another version than the + /// open one. + UnmatchedClose { line: usize }, + /// A block still open at the end of the source; `line` is its opening. + UnclosedBlock { line: usize }, } impl fmt::Display for RenderError { @@ -139,6 +207,16 @@ impl fmt::Display for RenderError { only.prefix(), version.prefix() ), + RenderError::NestedBlock { line } => { + write!(f, "line {line}: a version block opened inside another") + } + RenderError::UnmatchedClose { line } => write!( + f, + "line {line}: a version block close with no open block of that version" + ), + RenderError::UnclosedBlock { line } => { + write!(f, "line {line}: a version block opened here is never closed") + } } } } @@ -234,6 +312,8 @@ pub fn render_str(src: &str, version: EqlVersion) -> Result } } let mut out = String::with_capacity(src.len()); + // The open version block, and the line that opened it. + let mut block: Option<(EqlVersion, usize)> = None; for (i, line) in src.split_inclusive('\n').enumerate() { let lineno = i + 1; for v in EqlVersion::ALL { @@ -244,6 +324,24 @@ pub fn render_str(src: &str, version: EqlVersion) -> Result }); } } + match (directive(line), block) { + (Some(Directive::Open(_)), Some(_)) => { + return Err(RenderError::NestedBlock { line: lineno }) + } + (Some(Directive::Open(v)), None) => { + block = Some((v, lineno)); + continue; + } + (Some(Directive::Close(v)), Some((open, _))) if v == open => { + block = None; + continue; + } + (Some(Directive::Close(_)), _) => { + return Err(RenderError::UnmatchedClose { line: lineno }) + } + (None, Some((v, _))) if v != version => continue, + (None, _) => {} + } let mut rest = line; while let Some(open) = rest.find("{{") { out.push_str(&rest[..open]); @@ -265,6 +363,9 @@ pub fn render_str(src: &str, version: EqlVersion) -> Result } out.push_str(rest); } + if let Some((_, line)) = block { + return Err(RenderError::UnclosedBlock { line }); + } Ok(out) } @@ -553,6 +654,94 @@ mod tests { ); } + #[test] + fn a_version_block_renders_in_its_own_version_only() { + let src = format!( + "a {{{{prefix}}}}\n{}\n three\n{}\n {}\n four {{{{eql_version}}}}\n {}\nz\n", + block_open(EqlVersion::V3), + block_close(EqlVersion::V3), + block_open(EqlVersion::V4), + block_close(EqlVersion::V4), + ); + assert_eq!( + render_str(&src, EqlVersion::V3).unwrap(), + "a eql_v3\n three\nz\n" + ); + assert_eq!( + render_str(&src, EqlVersion::V4).unwrap(), + "a eql_v4\n four 4\nz\n" + ); + } + + /// A block written for one version leaves the other's output exactly as + /// the source without the block renders it. + #[test] + fn a_block_for_one_version_leaves_the_other_unchanged() { + let plain = "SELECT {{prefix}}.f();\nSELECT 2;\n"; + let with = "SELECT {{prefix}}.f();\n-- {{#v4}}\nSELECT 'v4 only';\n-- {{/v4}}\nSELECT 2;\n"; + assert_eq!( + render_str(with, EqlVersion::V3).unwrap(), + render_str(plain, EqlVersion::V3).unwrap() + ); + assert_eq!( + render_str(with, EqlVersion::V4).unwrap(), + "SELECT eql_v4.f();\nSELECT 'v4 only';\nSELECT 2;\n" + ); + } + + #[test] + fn rejects_malformed_blocks() { + for (src, err) in [ + ( + "-- {{#v3}}\n-- {{#v4}}\n-- {{/v4}}\n-- {{/v3}}\n", + RenderError::NestedBlock { line: 2 }, + ), + ("x\n-- {{/v3}}\n", RenderError::UnmatchedClose { line: 2 }), + ( + "-- {{#v3}}\n-- {{/v4}}\n", + RenderError::UnmatchedClose { line: 2 }, + ), + ("x\n-- {{#v4}}\ny\n", RenderError::UnclosedBlock { line: 2 }), + ] { + assert_eq!(render_str(src, EqlVersion::V4), Err(err), "{src:?}"); + } + // A directive naming a version the renderer does not know, or one + // that is not alone on its line, is an unknown placeholder. + for src in ["-- {{#v5}}\n", "SELECT 1; -- {{#v4}}\n"] { + assert!( + matches!( + render_str(src, EqlVersion::V4), + Err(RenderError::UnknownPlaceholder { line: 1, .. }) + ), + "{src:?}" + ); + } + // A literal name inside the other version's block is still refused. + assert!(matches!( + render_str( + "-- {{#v4}}\nSELECT eql_v3.f();\n-- {{/v4}}\n", + EqlVersion::V3 + ), + Err(RenderError::LiteralName { line: 2, .. }) + )); + } + + #[test] + fn a_directive_may_be_indented() { + assert_eq!( + directive(" -- {{#v4}}\n"), + Some(Directive::Open(EqlVersion::V4)) + ); + assert_eq!( + directive("--{{/v3}}"), + Some(Directive::Close(EqlVersion::V3)) + ); + assert_eq!(block_open(EqlVersion::V4), "-- {{#v4}}"); + assert_eq!(block_close(EqlVersion::V3), "-- {{/v3}}"); + assert_eq!(directive("-- {{prefix}}"), None); + assert_eq!(directive("-- {{#v4}} trailing"), None); + } + #[test] fn parses_versions() { assert_eq!("3".parse(), Ok(EqlVersion::V3)); diff --git a/packages/eql/crates/eql-codegen/templates/functions/extractor.sql.j2 b/packages/eql/crates/eql-codegen/templates/functions/extractor.sql.j2 index 9f3c60286..f4596433c 100644 --- a/packages/eql/crates/eql-codegen/templates/functions/extractor.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/functions/extractor.sql.j2 @@ -1,7 +1,21 @@ --! @brief Index extractor for {{ dom }}. --! @param a {{ dom }} --! @return {{ e.ret }} +{%- if e.required %} +{{ v4_only }} +--! @throws When the operand carries no `{{ e.required.key }}` term, the term {{ e.required.operators }} compare. +{{ v4_end }} +{%- endif %} CREATE FUNCTION {{ schema }}.{{ e.extractor }}(a {{ dom }}) RETURNS {{ e.ret }} LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +{%- if e.required %} +{{ v3_only }} AS $$ SELECT {{ internal_schema }}.{{ e.ctor }}(a::jsonb) $$; +{{ v3_end }} +{{ v4_only }} +AS $$ SELECT {{ internal_schema }}.{{ e.ctor }}({{ internal_schema }}.require_query_term(a::jsonb, '{{ e.required.domain }}', '{{ e.required.key }}', '{{ e.required.operators }}')) $$; +{{ v4_end }} +{%- else %} +AS $$ SELECT {{ internal_schema }}.{{ e.ctor }}(a::jsonb) $$; +{%- endif %} diff --git a/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 b/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 index eb93b0266..db992f483 100644 --- a/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 @@ -26,13 +26,35 @@ BEGIN {%- for k in d.keys %} AND VALUE ? '{{ k }}' {%- endfor %} + {%- if d.optional_term_keys %} +{{ v3_only }} + {%- for k in d.optional_term_keys %} + AND VALUE ? '{{ k }}' + {%- endfor %} +{{ v3_end }} +{{ v4_only }} + AND ({% for k in d.optional_term_keys %}{% if not loop.first %} OR {% endif %}VALUE ? '{{ k }}'{% endfor %}) +{{ v4_end }} + {%- endif %} {%- for k in d.forbidden_keys %} AND NOT (VALUE ? '{{ k }}') {%- endfor %} + {%- if d.optional_term_keys and d.nonempty_array_keys %} +{{ v3_only }} + {%- endif %} {%- for k in d.nonempty_array_keys %} AND jsonb_typeof(VALUE -> '{{ k }}') = 'array' AND jsonb_array_length(VALUE -> '{{ k }}') > 0 {%- endfor %} + {%- if d.optional_term_keys and d.nonempty_array_keys %} +{{ v3_end }} +{{ v4_only }} + {%- for k in d.nonempty_array_keys %} + AND (NOT (VALUE ? '{{ k }}') OR (jsonb_typeof(VALUE -> '{{ k }}') = 'array' + AND jsonb_array_length(VALUE -> '{{ k }}') > 0)) + {%- endfor %} +{{ v4_end }} + {%- endif %} AND VALUE->>'v' = '{{ eql_version }}' ); END IF; diff --git a/packages/eql/src/v3/scalars/functions.sql b/packages/eql/src/v3/scalars/functions.sql index 1376a4719..affe0d755 100644 --- a/packages/eql/src/v3/scalars/functions.sql +++ b/packages/eql/src/v3/scalars/functions.sql @@ -58,3 +58,49 @@ BEGIN RAISE EXCEPTION 'operator % is not supported for %', operator_name, type_name; END; $$ LANGUAGE plpgsql; +-- {{#v4}} + +--! @brief Return a query operand unchanged when it carries the named term, +--! and raise when it does not. +--! +--! An EQL v4 search-domain query operand ({{prefix}}.query_text_search, +--! {{prefix}}.query_text_search_ore) carries only the terms its operator +--! reads, so each extractor over it reads its term through this helper. A +--! missing term raises here rather than reading as NULL, which would compare +--! as no row. +--! +--! A term whose value is JSON `null` is missing, and so is an empty array: a +--! `bf` with no positions is the filter of text too short for a 3-gram, which +--! a stored filter of the same text would contain, so a query carrying it +--! would match exactly the stored values no match query should find. +--! +--! The raise happens where the operand is evaluated, not where the statement +--! is planned. A literal operand is folded at plan time and raises even over +--! an empty table; a bound parameter under a generic plan raises wherever the +--! extractor reads it (an index scan always does, to build its key), and a +--! plan that never reaches the predicate returns no row and no error. +--! @internal +--! @param val jsonb The query operand. +--! @param domain_name text The operand's domain, for the message. +--! @param term text The JSON key of the term the extractor reads (hm, op, ob, bf). +--! @param operators text The operators that read the term, for the message. +--! @return jsonb `val`, unchanged. +--! @throws invalid_parameter_value When `val` has no `term` key, or its value +--! is JSON `null` or an empty array. The message names the domain, the +--! term and the operators, never the operand. +CREATE FUNCTION {{prefix}}_internal.require_query_term(val jsonb, domain_name text, term text, operators text) +RETURNS jsonb +IMMUTABLE STRICT PARALLEL SAFE +SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + IF NOT (val ? term) + OR jsonb_typeof(val -> term) = 'null' + OR (jsonb_typeof(val -> term) = 'array' AND jsonb_array_length(val -> term) = 0) THEN + RAISE EXCEPTION '% value has no "%" term, the term % compare', domain_name, term, operators + USING ERRCODE = 'invalid_parameter_value'; + END IF; + RETURN val; +END; +$$ LANGUAGE plpgsql; +-- {{/v4}} diff --git a/packages/eql/src/v3/scalars/text/query_text_search_functions.sql b/packages/eql/src/v3/scalars/text/query_text_search_functions.sql index 0bc6d563b..4b63185a4 100644 --- a/packages/eql/src/v3/scalars/text/query_text_search_functions.sql +++ b/packages/eql/src/v3/scalars/text/query_text_search_functions.sql @@ -2,6 +2,7 @@ -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/scalars/text/query_text_types.sql -- REQUIRE: src/v3/scalars/text/text_search_functions.sql +-- REQUIRE: src/v3/scalars/functions.sql --! @file encrypted_domain/text/query_text_search_functions.sql --! @brief Functions for {{prefix}}.query_text_search. @@ -9,26 +10,50 @@ --! @brief Index extractor for {{prefix}}.query_text_search. --! @param a {{prefix}}.query_text_search --! @return {{prefix}}_internal.hmac_256 +-- {{#v4}} +--! @throws When the operand carries no `hm` term, the term = and <> compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.eq_term(a {{prefix}}.query_text_search) RETURNS {{prefix}}_internal.hmac_256 LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.hmac_256(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.hmac_256({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search', 'hm', '= and <>')) $$; +-- {{/v4}} --! @brief Index extractor for {{prefix}}.query_text_search. --! @param a {{prefix}}.query_text_search --! @return {{prefix}}_internal.ope_cllw +-- {{#v4}} +--! @throws When the operand carries no `op` term, the term <, <=, > and >= compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.ord_term(a {{prefix}}.query_text_search) RETURNS {{prefix}}_internal.ope_cllw LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.ope_cllw(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.ope_cllw({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search', 'op', '<, <=, > and >=')) $$; +-- {{/v4}} --! @brief Index extractor for {{prefix}}.query_text_search. --! @param a {{prefix}}.query_text_search --! @return {{prefix}}_internal.bloom_filter +-- {{#v4}} +--! @throws When the operand carries no `bf` term, the term @@ compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.match_term(a {{prefix}}.query_text_search) RETURNS {{prefix}}_internal.bloom_filter LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.bloom_filter(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.bloom_filter({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search', 'bf', '@@')) $$; +-- {{/v4}} --! @brief Operator wrapper for {{prefix}}.query_text_search. --! @param a public.{{prefix}}_text_search diff --git a/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql b/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql index 4f1f4e94a..44295168d 100644 --- a/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql +++ b/packages/eql/src/v3/scalars/text/query_text_search_ore_functions.sql @@ -2,6 +2,7 @@ -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/scalars/text/query_text_types.sql -- REQUIRE: src/v3/scalars/text/text_search_ore_functions.sql +-- REQUIRE: src/v3/scalars/functions.sql --! @file encrypted_domain/text/query_text_search_ore_functions.sql --! @brief Functions for {{prefix}}.query_text_search_ore. @@ -9,26 +10,50 @@ --! @brief Index extractor for {{prefix}}.query_text_search_ore. --! @param a {{prefix}}.query_text_search_ore --! @return {{prefix}}_internal.hmac_256 +-- {{#v4}} +--! @throws When the operand carries no `hm` term, the term = and <> compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.eq_term(a {{prefix}}.query_text_search_ore) RETURNS {{prefix}}_internal.hmac_256 LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.hmac_256(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.hmac_256({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search_ore', 'hm', '= and <>')) $$; +-- {{/v4}} --! @brief Index extractor for {{prefix}}.query_text_search_ore. --! @param a {{prefix}}.query_text_search_ore --! @return {{prefix}}_internal.ore_block_256 +-- {{#v4}} +--! @throws When the operand carries no `ob` term, the term <, <=, > and >= compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.ord_term_ore(a {{prefix}}.query_text_search_ore) RETURNS {{prefix}}_internal.ore_block_256 LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.ore_block_256(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.ore_block_256({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search_ore', 'ob', '<, <=, > and >=')) $$; +-- {{/v4}} --! @brief Index extractor for {{prefix}}.query_text_search_ore. --! @param a {{prefix}}.query_text_search_ore --! @return {{prefix}}_internal.bloom_filter +-- {{#v4}} +--! @throws When the operand carries no `bf` term, the term @@ compare. +-- {{/v4}} CREATE FUNCTION {{prefix}}.match_term(a {{prefix}}.query_text_search_ore) RETURNS {{prefix}}_internal.bloom_filter LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +-- {{#v3}} AS $$ SELECT {{prefix}}_internal.bloom_filter(a::jsonb) $$; +-- {{/v3}} +-- {{#v4}} +AS $$ SELECT {{prefix}}_internal.bloom_filter({{prefix}}_internal.require_query_term(a::jsonb, '{{prefix}}.query_text_search_ore', 'bf', '@@')) $$; +-- {{/v4}} --! @brief Operator wrapper for {{prefix}}.query_text_search_ore. --! @param a public.{{prefix}}_text_search_ore diff --git a/packages/eql/src/v3/scalars/text/query_text_types.sql b/packages/eql/src/v3/scalars/text/query_text_types.sql index 9360a9fbf..f6f8a9b1d 100644 --- a/packages/eql/src/v3/scalars/text/query_text_types.sql +++ b/packages/eql/src/v3/scalars/text/query_text_types.sql @@ -119,12 +119,23 @@ BEGIN jsonb_typeof(VALUE) = 'object' AND VALUE ? 'v' AND VALUE ? 'i' +-- {{#v3}} AND VALUE ? 'hm' AND VALUE ? 'ob' AND VALUE ? 'bf' +-- {{/v3}} +-- {{#v4}} + AND (VALUE ? 'hm' OR VALUE ? 'ob' OR VALUE ? 'bf') +-- {{/v4}} AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND (NOT (VALUE ? 'ob') OR (jsonb_typeof(VALUE -> 'ob') = 'array' + AND jsonb_array_length(VALUE -> 'ob') > 0)) +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -141,9 +152,14 @@ BEGIN jsonb_typeof(VALUE) = 'object' AND VALUE ? 'v' AND VALUE ? 'i' +-- {{#v3}} AND VALUE ? 'hm' AND VALUE ? 'op' AND VALUE ? 'bf' +-- {{/v3}} +-- {{#v4}} + AND (VALUE ? 'hm' OR VALUE ? 'op' OR VALUE ? 'bf') +-- {{/v4}} AND NOT (VALUE ? 'c') AND VALUE->>'v' = '{{eql_version}}' ); diff --git a/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs b/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs index 3e93e86d0..fb4f38f38 100644 --- a/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs +++ b/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs @@ -322,3 +322,102 @@ async fn eq_term_across_versions_compares_text(pool: PgPool) -> Result<()> { ); Ok(()) } + +/// SQLSTATE `invalid_parameter_value`. +const INVALID_PARAMETER_VALUE: &str = "22023"; + +/// A text search query operand of `version` built from the first fixture row +/// whose filter is not empty, minus the ciphertext, the block ORE term and +/// the keys in `drop`. +fn search_operand(version: u8, drop: &[&str]) -> String { + let mut operand = format!( + "jsonb_set((SELECT payload - 'c' - 'ob' FROM fixtures.eql_v3_text \ + WHERE jsonb_array_length(payload -> 'bf') > 0 ORDER BY id LIMIT 1), '{{v}}', '{version}')" + ); + for key in drop { + operand = format!("({operand} - '{key}')"); + } + format!("({operand})::eql_v{version}.query_text_search") +} + +/// The two versions part on the text search query operand (ADR-0002, +/// amended 2026-10-19). v3's is cipherstash-client's and needs `hm`, `op` +/// and `bf`. v4's needs any one of them, since Stack Encrypt sends only the +/// term its operator reads, and each v4 operator refuses an operand without +/// its term instead of comparing a NULL and matching no row. +#[sqlx::test(fixtures(path = "../fixtures", scripts("eql_v3_text")))] +async fn a_v4_search_query_needs_one_term_and_a_v3_one_needs_all(pool: PgPool) -> Result<()> { + install_v4(&pool).await?; + + let one_term: [&[&str]; 3] = [&["op", "bf"], &["hm", "bf"], &["hm", "op"]]; + sqlx::query(&format!("SELECT {}", search_operand(3, &[]))) + .execute(&pool) + .await?; + for drop in one_term { + let err = sqlx::query(&format!("SELECT {}", search_operand(3, drop))) + .execute(&pool) + .await + .expect_err("a v3 search query needs every term"); + assert_eq!( + sqlstate(&err).as_deref(), + Some(CHECK_VIOLATION), + "v3 without {drop:?}: {err}" + ); + } + for drop in std::iter::once(&[][..]).chain(one_term) { + sqlx::query(&format!("SELECT {}", search_operand(4, drop))) + .execute(&pool) + .await + .unwrap_or_else(|e| panic!("v4 without {drop:?}: {e}")); + } + let err = sqlx::query(&format!( + "SELECT {}", + search_operand(4, &["hm", "op", "bf"]) + )) + .execute(&pool) + .await + .expect_err("a v4 search query needs a term"); + assert_eq!(sqlstate(&err).as_deref(), Some(CHECK_VIOLATION), "{err}"); + + sqlx::raw_sql( + "CREATE TABLE t4s (c public.eql_v4_text_search); + INSERT INTO t4s SELECT jsonb_set(payload - 'ob', '{v}', '4')::public.eql_v4_text_search + FROM fixtures.eql_v3_text WHERE jsonb_array_length(payload -> 'bf') > 0;", + ) + .execute(&pool) + .await?; + for (op, term, rest) in [ + ("=", "hm", ["op", "bf"]), + ("<=", "op", ["hm", "bf"]), + ("@@", "bf", ["hm", "op"]), + ] { + // With its term, the operator answers: the operand's own row. + let n: i64 = sqlx::query_scalar(&format!( + "SELECT count(*) FROM t4s WHERE c {op} {}", + search_operand(4, &rest) + )) + .fetch_one(&pool) + .await?; + assert!(n >= 1, "c {op} with only {term} finds its own row"); + // Without it, the operator refuses, naming the term. + let err = sqlx::query(&format!( + "SELECT count(*) FROM t4s WHERE c {op} {}", + search_operand(4, &[term]) + )) + .execute(&pool) + .await + .expect_err("an operator refuses an operand without its term"); + assert_eq!( + sqlstate(&err).as_deref(), + Some(INVALID_PARAMETER_VALUE), + "c {op} without {term}: {err}" + ); + assert!( + err.to_string().contains(&format!( + "eql_v4.query_text_search value has no \"{term}\" term" + )), + "c {op} without {term}: {err}" + ); + } + Ok(()) +} From 6a1b2c0aebac7daa37ae9d019b5c0238b2860f93 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 19 Oct 2026 10:49:00 +1100 Subject: [PATCH 2/4] feat(stack-encrypt)!: hand a target's query the index that answers its operator TargetResolver::query took a type name, a label and a value, so a resolver had to build the type's whole query twin, every term of it, for every operator. EQL v4's TextSearch needs to emit only the term its operator reads: a match term cannot be made from text shorter than one 3-gram, and `=` or `<` on such text must not be refused for want of one. query_term already knows the answering index (OpSpec::answered_by over the type's indexes). It now passes it through record::query to TargetResolver::query as an &IndexSpec after the type's name. A resolver whose twins carry every term ignores it. eql-bindings' encryption::targets::query takes it the same way and refuses an index the type does not carry (TargetError::NoIndex) before deriving anything. The Go guest forwards it; an index the type lacks can only come from an engine bug, so the guest reports it as internal. BREAKING CHANGE: TargetResolver::query and dynamic::record::query take an &IndexSpec; eql_bindings::encryption::targets::query takes one after the name. --- languages/golang/encrypt/guest/src/targets.rs | 14 +++- packages/eql/crates/eql-bindings/CHANGELOG.md | 5 ++ .../eql-bindings/src/encryption/targets.rs | 57 +++++++++++++- .../eql/tests/encryption/tests/common/mod.rs | 14 ++++ .../tests/encryption/tests/heap_hygiene.rs | 76 ++++++++++++++----- .../eql/tests/encryption/tests/targets.rs | 45 ++++++++--- .../tests/encryption/tests/typed_queries.rs | 3 +- .../encryption/tests/v4_date_time_numeric.rs | 4 +- .../eql/tests/encryption/tests/v4_order.rs | 26 +++++-- .../eql/tests/encryption/tests/v4_types.rs | 34 +++++++-- packages/stack-encrypt/CHANGELOG.md | 9 +++ .../fuzz/fuzz_targets/check_record.rs | 1 + packages/stack-encrypt/src/dynamic/query.rs | 10 ++- packages/stack-encrypt/src/dynamic/record.rs | 67 +++++++++++++--- packages/stack-encrypt/src/dynamic/target.rs | 14 ++++ 15 files changed, 314 insertions(+), 65 deletions(-) diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index b863f0669..b19b3d6db 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -79,11 +79,13 @@ impl TargetResolver for EqlTargets { fn query<'a, K: 'static>( &self, name: &str, + index: &stack_encrypt::target::IndexSpec, keyset: &'a stack_encrypt::KeysetCipher<'_, K>, label: &stack_encrypt::Label, plaintext: stack_encrypt::dynamic::ValueFrame, ) -> Result, K>, TargetError> { - eql_bindings::encryption::targets::query(name, keyset, label, plaintext).map_err(convert) + eql_bindings::encryption::targets::query(name, index, keyset, label, plaintext) + .map_err(convert) } } @@ -252,6 +254,16 @@ mod tests { }), TargetError::Plaintext { target, expected: Some(stack_encrypt::dynamic::ValueKind::String), .. } if target == "TextEq" )); + // The engine asks a type's query only through an index the type + // carries, so one it does not is the engine's failure, not the + // caller's input. + assert!(matches!( + convert(Eql::NoIndex { + name: "TextEq", + index: "ope", + }), + TargetError::Other(_) + )); } /// serde_json quotes the value it refused, and a stored EQL value holds diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index bf1e1217b..07fe98214 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -101,6 +101,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 orderable plaintext. Their query twins carry one bound, so the engine answers `<`, `<=`, `>` and `>=` (and `=` on text) through them, not `BETWEEN`. +- **`encryption::targets::query` takes the index that answers the query's + operator** (`targets` feature), an `&IndexSpec` after the name, as Stack + Encrypt's `TargetResolver::query` hands it over. An index the type does + not carry is refused with `TargetError::NoIndex`, before anything is + derived. - **`Target::match_options`** (`targets` feature): the options a type's match index is built under, `EqlMatch`'s, for a type that carries one (`TextMatch`), and `None` otherwise. Stack Encrypt answers `matches` diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 5703c783e..2f4737605 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -90,7 +90,7 @@ use std::fmt; use serde::{de::DeserializeOwned, Serialize}; use stack_encrypt::dynamic::{ValueFrame, ValueKind}; use stack_encrypt::kms::MaybeSend; -use stack_encrypt::target::ExpectedContext; +use stack_encrypt::target::{ExpectedContext, IndexSpec}; use stack_encrypt::{ Decrypt, DecryptInto, Encrypt, EncryptFrom, Error, KeysetCipher, Label, NonEmpty, Pending, StackCipher, @@ -200,6 +200,15 @@ pub enum TargetError { /// The type's name. name: &'static str, }, + /// A query asked through an index the type does not carry + /// ([`Target::indexes`]), so it has no term to derive. + #[error("{name} carries no {index} index, so no query is answered through it")] + NoIndex { + /// The type's name. + name: &'static str, + /// The index asked for, by its `IndexSpec::key()`. + index: &'static str, + }, /// The field's context is not an EQL column: an [`Identifier`] is a /// two-segment label, table then column, and this label is not one. #[error("{label:?} is not an EQL column identifier: expected two segments, table and column")] @@ -336,23 +345,44 @@ pub fn decrypt<'a, K: 'static>( /// matches stored values of the type under `context`'s column, as JSON /// bytes. A query derives no data key, so the pending settles without I/O. /// +/// `index` is the type's index that answers the query's operator, one of +/// [`Target::indexes`]; Stack Encrypt passes the one its +/// `OpSpec::answered_by` picks. +/// /// # Errors /// -/// As [`encrypt`]. +/// As [`encrypt`], and [`TargetError::NoIndex`] for an index the type does +/// not carry. pub fn query<'a, K: 'static>( name: &str, + index: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, context: &Label, plaintext: ValueFrame, ) -> Result, K>, TargetError> { producible(name)?; - if let Some(error) = no_query(target(name)) { + let Some(target) = target(name) else { + return Err(refuse(name)); + }; + if let Some(error) = no_query(Some(target)) { return Err(error); } + carries(target, index)?; let column = Identifier::from_label(context)?; query_named(name, keyset, column, plaintext) } +/// Refuse a query through an index the type does not carry. +fn carries(target: &'static Target, index: &IndexSpec) -> Result<(), TargetError> { + match target.indexes.iter().find(|key| **key == index.key()) { + Some(_) => Ok(()), + None => Err(TargetError::NoIndex { + name: target.name, + index: index.key(), + }), + } +} + /// Refuse a name the dispatch has no arm for, as the table explains it. fn producible(name: &str) -> Result<(), TargetError> { match target(name) { @@ -640,6 +670,27 @@ mod tests { assert!(matches!(refuse_query("Nope"), TargetError::Unknown { .. })); } + #[test] + fn a_query_through_an_index_the_type_lacks_is_refused() { + let text_eq = target("TextEq").unwrap(); + assert!(carries(text_eq, &IndexSpec::Equality).is_ok()); + let error = carries(text_eq, &IndexSpec::Ope).unwrap_err(); + assert!( + matches!( + error, + TargetError::NoIndex { + name: "TextEq", + index: "ope" + } + ), + "{error:?}" + ); + assert_eq!( + error.to_string(), + "TextEq carries no ope index, so no query is answered through it" + ); + } + #[test] fn refusals_name_the_type_and_its_reason() { assert!(matches!(refuse("Nope"), TargetError::Unknown { name } if name == "Nope")); diff --git a/packages/eql/tests/encryption/tests/common/mod.rs b/packages/eql/tests/encryption/tests/common/mod.rs index 22540a804..634369030 100644 --- a/packages/eql/tests/encryption/tests/common/mod.rs +++ b/packages/eql/tests/encryption/tests/common/mod.rs @@ -71,3 +71,17 @@ pub async fn cipher() -> (StackCipher, Arc>) { calls.lock().unwrap().loads = 0; (cipher, calls) } + +/// The index a query on the named type is asked through when the test does +/// not care which: the first the type carries. Equality for a name with no +/// row or no index, which a test of a refusal names; the refusal comes +/// before the index is read. +// Not every suite that includes this module queries by name. +#[allow(dead_code)] +pub fn index_of(name: &str) -> stack_encrypt::target::IndexSpec { + use stack_encrypt::target::IndexSpec; + eql_bindings::encryption::targets::target(name) + .and_then(|target| target.indexes.first()) + .and_then(|key| IndexSpec::parse(key)) + .unwrap_or(IndexSpec::Equality) +} diff --git a/packages/eql/tests/encryption/tests/heap_hygiene.rs b/packages/eql/tests/encryption/tests/heap_hygiene.rs index 32b553e3b..ef7818cbe 100644 --- a/packages/eql/tests/encryption/tests/heap_hygiene.rs +++ b/packages/eql/tests/encryption/tests/heap_hygiene.rs @@ -11,9 +11,10 @@ //! //! One free is excluded by name: vitaminc-encrypt 0.6.0's aws-lc backend //! (`backend::aws_lc::CipherKey::seal`, native builds only) frees an unwiped -//! plaintext buffer on every seal. That is vitaminc's to fix; the spy -//! recognises those frees by their backtrace, so a build without symbols -//! fails rather than passes. +//! plaintext buffer on every seal; that is vitaminc's to fix. It is counted +//! and printed, never silent, and every other free on those paths is still +//! watched. The spy recognises these frees by their backtrace, so a build +//! without symbols fails rather than passes. //! //! Unix only: the spy scans a freed block with libc's `memmem`, not as a @@ -42,12 +43,16 @@ thread_local! { /// threads, and the findings below are one set. static GATE: Mutex<()> = Mutex::new(()); -/// The marked frees seen, as backtraces; and how many were vitaminc's. +/// The marked frees seen, as backtraces; and how many each exclusion took. static FOUND: Mutex> = Mutex::new(Vec::new()); -static UPSTREAM: Mutex = Mutex::new(0); +static EXCLUDED: Mutex<[usize; 1]> = Mutex::new([0; 1]); -/// The vitaminc frame whose free is excluded (see the module docs). -const UPSTREAM_SEAL: &str = "vitaminc_encrypt::backend::aws_lc::CipherKey::seal"; +/// The frames whose frees are excluded, each with what it is (see the module +/// docs): vitaminc's aws-lc seal. +const EXCLUSIONS: [(&str, &str); 1] = [( + "vitaminc_encrypt::backend::aws_lc::CipherKey::seal", + "vitaminc's aws-lc seal", +)]; /// Whether the `len` bytes at `ptr` hold [`MARK`]. A freed block can hold /// bytes Rust never initialised (a `Vec`'s spare capacity, a struct's @@ -87,8 +92,12 @@ unsafe impl GlobalAlloc for Spy { // Symbol demangling differs by platform: Linux writes // `<…::CipherKey>::seal`, macOS `…::CipherKey::seal`. Match // with the angle brackets dropped. - if trace.replace(['<', '>'], "").contains(UPSTREAM_SEAL) { - *UPSTREAM.lock().unwrap() += 1; + let plain = trace.replace(['<', '>'], ""); + if let Some(at) = EXCLUSIONS + .iter() + .position(|(frame, _)| plain.contains(frame)) + { + EXCLUDED.lock().unwrap()[at] += 1; } else { FOUND .lock() @@ -110,14 +119,16 @@ static SPY: Spy = Spy; fn watched(what: &str, f: impl FnOnce() -> T) -> T { let _gate = GATE.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); FOUND.lock().unwrap().clear(); - *UPSTREAM.lock().unwrap() = 0; + *EXCLUDED.lock().unwrap() = [0; 1]; WATCHING.with(|on| on.set(true)); let out = f(); WATCHING.with(|on| on.set(false)); let found = std::mem::take(&mut *FOUND.lock().unwrap()); - let upstream = std::mem::take(&mut *UPSTREAM.lock().unwrap()); - if upstream > 0 { - eprintln!("{what}: skipped {upstream} unwiped frees in vitaminc's aws-lc seal"); + let excluded = std::mem::take(&mut *EXCLUDED.lock().unwrap()); + for ((_, name), skipped) in EXCLUSIONS.iter().zip(excluded) { + if skipped > 0 { + eprintln!("{what}: skipped {skipped} unwiped frees in {name}"); + } } assert!( found.is_empty(), @@ -136,14 +147,37 @@ fn text(value: &str) -> ValueFrame { ValueFrame::of(value).unwrap() } +/// Every index the target carries, each of which a query may be asked +/// through. A type whose query is built per index (`TextSearch`) runs a +/// different projection, and its conversion into the query twin, for each. +fn indexes_of(target: &targets::Target) -> Vec { + target + .indexes + .iter() + .map(|key| stack_encrypt::target::IndexSpec::parse(key).expect("an engine index key")) + .collect() +} + fn runtime() -> tokio::runtime::Runtime { tokio::runtime::Builder::new_current_thread() .build() .unwrap() } -/// The producible EQL types over text, each with whether it has a query. -const TEXT_TARGETS: [(&str, bool); 3] = [("Text", false), ("TextEq", true), ("TextMatch", true)]; +/// The producible EQL types over text, read from the target table so a type +/// that becomes producible is watched without an edit here. +fn text_targets() -> Vec<&'static targets::Target> { + let found: Vec<_> = targets::targets() + .iter() + .filter(|target| target.family == "text" && target.producible) + .collect(); + assert!( + found.iter().any(|target| target.query.is_none()) + && found.iter().any(|target| target.query.is_some()), + "the table lists a storage-only and a queried text type" + ); + found +} #[test] fn a_text_target_leaves_no_plaintext_on_the_heap() { @@ -157,7 +191,8 @@ fn a_text_target_leaves_no_plaintext_on_the_heap() { }); let keyset = cipher.default_keyset(); let label = Label::new(["users", "email"]).unwrap(); - for (name, has_query) in TEXT_TARGETS { + for target in text_targets() { + let name = target.name; let plaintext = text(&marked("stored")); let stored = watched(&format!("{name} encrypt"), || { rt.block_on(async { @@ -189,11 +224,14 @@ fn a_text_target_leaves_no_plaintext_on_the_heap() { }); drop(opened); }); - if has_query { + if target.query.is_none() { + continue; + } + for index in indexes_of(target) { let query = text(&marked("query")); - watched(&format!("{name} query"), || { + watched(&format!("{name} query through {}", index.key()), || { rt.block_on(async { - targets::query(name, &keyset, &label, query) + targets::query(name, &index, &keyset, &label, query) .unwrap() .await .unwrap() diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index ad3020dba..43091ada3 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -184,10 +184,16 @@ mod given_text_eq { async fn query_bytes_are_the_typed_query_twin() { let (cipher, calls) = common::cipher().await; let keyset = cipher.default_keyset(); - let by_name = targets::query("TextEq", &keyset, &email(), text("alice@example.com")) - .unwrap() - .await - .unwrap(); + let by_name = targets::query( + "TextEq", + &common::index_of("TextEq"), + &keyset, + &email(), + text("alice@example.com"), + ) + .unwrap() + .await + .unwrap(); let typed: TextEqQuery = keyset .encrypt_as(&"alice@example.com".to_owned(), column()) .await @@ -255,6 +261,7 @@ mod given_text_eq { ); let error = refused(targets::query( "TextEq", + &common::index_of("TextEq"), &keyset, &email(), frame(None::), @@ -354,7 +361,13 @@ mod given_a_name_the_engine_cannot_produce { name, ); unproducible( - refused(targets::query(name, &keyset, &email(), text("x"))), + refused(targets::query( + name, + &common::index_of(name), + &keyset, + &email(), + text("x"), + )), name, ); unproducible( @@ -404,7 +417,13 @@ mod given_an_unknown_name { &email(), text("x"), ))); - unknown(refused(targets::query(name, &keyset, &email(), text("x")))); + unknown(refused(targets::query( + name, + &common::index_of(name), + &keyset, + &email(), + text("x"), + ))); unknown(refused(targets::decrypt(name, &cipher, &email(), b"{}"))); } assert_eq!( @@ -687,10 +706,16 @@ mod the_cross_language_fixture { async fn the_text_eq_query_fixture_is_what_the_dispatch_derives() { let (cipher, calls) = common::cipher().await; let keyset = cipher.default_keyset(); - let query = targets::query("TextEq", &keyset, &email(), text(PLAINTEXT)) - .unwrap() - .await - .unwrap(); + let query = targets::query( + "TextEq", + &common::index_of("TextEq"), + &keyset, + &email(), + text(PLAINTEXT), + ) + .unwrap() + .await + .unwrap(); assert!( calls.lock().unwrap().generate.is_empty(), "a query mints nothing" diff --git a/packages/eql/tests/encryption/tests/typed_queries.rs b/packages/eql/tests/encryption/tests/typed_queries.rs index 3ee606359..26f4f7b05 100644 --- a/packages/eql/tests/encryption/tests/typed_queries.rs +++ b/packages/eql/tests/encryption/tests/typed_queries.rs @@ -90,11 +90,12 @@ impl TargetResolver for Eql { fn query<'a, K: 'static>( &self, name: &str, + index: &stack_encrypt::target::IndexSpec, keyset: &'a KeysetCipher<'_, K>, label: &Label, plaintext: ValueFrame, ) -> Result, K>, TargetError> { - eql_targets::query(name, keyset, label, plaintext).map_err(convert) + eql_targets::query(name, index, keyset, label, plaintext).map_err(convert) } } diff --git a/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs b/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs index 8026446eb..466e7377c 100644 --- a/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs +++ b/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs @@ -91,7 +91,7 @@ async fn hm( name: &str, value: ValueFrame, ) -> String { - let bytes = targets::query(name, keyset, &label(), value) + let bytes = targets::query(name, &common::index_of(name), keyset, &label(), value) .unwrap() .await .unwrap(); @@ -382,7 +382,7 @@ async fn the_ledger_query_fixture_is_what_the_dispatch_derives() { let keyset = &keyset; async move { let label = Label::new(["profiles", column]).unwrap(); - let bytes = targets::query(name, keyset, &label, value) + let bytes = targets::query(name, &common::index_of(name), keyset, &label, value) .unwrap() .await .unwrap(); diff --git a/packages/eql/tests/encryption/tests/v4_order.rs b/packages/eql/tests/encryption/tests/v4_order.rs index f9f76da38..89852221f 100644 --- a/packages/eql/tests/encryption/tests/v4_order.rs +++ b/packages/eql/tests/encryption/tests/v4_order.rs @@ -66,7 +66,7 @@ async fn query( name: &str, value: ValueFrame, ) -> serde_json::Value { - let bytes = targets::query(name, keyset, &label(), value) + let bytes = targets::query(name, &common::index_of(name), keyset, &label(), value) .unwrap() .await .unwrap(); @@ -340,7 +340,13 @@ async fn text_with_no_fold_is_refused() { Err(error) => panic!("{name}: refused before the plan ran: {error}"), }; assert!(stored.is_err(), "{name}: stored text with no fold"); - let queried = match targets::query(name, &keyset, &label(), frame(unassigned)) { + let queried = match targets::query( + name, + &common::index_of(name), + &keyset, + &label(), + frame(unassigned), + ) { Ok(pending) => pending.await, Err(error) => panic!("{name}: refused before the plan ran: {error}"), }; @@ -406,7 +412,7 @@ async fn the_ordered_query_fixture_is_what_the_dispatch_derives() { let keyset = &keyset; async move { let label = Label::new(["rankings", column]).unwrap(); - let bytes = targets::query(name, keyset, &label, value) + let bytes = targets::query(name, &common::index_of(name), keyset, &label, value) .unwrap() .await .unwrap(); @@ -682,10 +688,16 @@ fn postgres_compares_and_orders_ope_rows() { let probes: Vec = runtime.block_on(async { let mut probes = Vec::new(); for ((column, _, _), (name, value)) in COLUMNS.iter().zip(frames(&rows[1].1)) { - let bytes = targets::query(name, &keyset, &under(column), value) - .unwrap() - .await - .unwrap(); + let bytes = targets::query( + name, + &common::index_of(name), + &keyset, + &under(column), + value, + ) + .unwrap() + .await + .unwrap(); probes.push(String::from_utf8(bytes).unwrap()); } probes diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index 84f0c32a6..90c41e932 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -210,7 +210,7 @@ async fn eq_terms( value: ValueFrame, ) -> (serde_json::Value, serde_json::Value) { let stored: serde_json::Value = by_name(keyset, name, value.clone()).await; - let query = targets::query(name, keyset, &label(), value) + let query = targets::query(name, &common::index_of(name), keyset, &label(), value) .unwrap() .await .unwrap(); @@ -431,17 +431,29 @@ async fn every_producible_type_dispatches_by_name() { match target.query { None => { - let error = refused(targets::query(name, &keyset, &label(), sample(kind))); + let error = refused(targets::query( + name, + &common::index_of(name), + &keyset, + &label(), + sample(kind), + )); assert!( matches!(error, TargetError::NoQuery { .. }), "{name}: {error}" ); } Some(_) => { - let query = targets::query(name, &keyset, &label(), sample(kind)) - .unwrap() - .await - .unwrap(); + let query = targets::query( + name, + &common::index_of(name), + &keyset, + &label(), + sample(kind), + ) + .unwrap() + .await + .unwrap(); let query: serde_json::Value = serde_json::from_slice(&query).unwrap(); assert!( query.get("c").is_none(), @@ -481,7 +493,13 @@ async fn storage_only_text_round_trips_through_both_entry_points() { let opened: String = cipher.decrypt_as(named, column().into()).await.unwrap(); assert_eq!(opened, "note"); // A storage-only type answers no query. - let error = refused(targets::query("Text", &keyset, &label(), frame("note"))); + let error = refused(targets::query( + "Text", + &common::index_of("Text"), + &keyset, + &label(), + frame("note"), + )); assert!( matches!(error, TargetError::NoQuery { name: "Text" }), "{error}" @@ -1258,7 +1276,7 @@ async fn the_v4_query_fixture_is_what_the_dispatch_derives() { let keyset = &keyset; async move { let label = Label::new(["profiles", column]).unwrap(); - let bytes = targets::query(name, keyset, &label, value) + let bytes = targets::query(name, &common::index_of(name), keyset, &label, value) .unwrap() .await .unwrap(); diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 7faa5630e..6e0649ac3 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -196,6 +196,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Breaking +- **A target's query is handed the index that answers its operator** + (`dynamic` feature). `TargetResolver::query` and `record::query` take an + `&IndexSpec` after the type's name and the field respectively; + `query_term` passes the index `OpSpec::answered_by` picks over the type's + indexes. A type whose query carries every term may ignore it. One whose + terms are optional derives that index's term alone, so EQL v4's + `TextSearch` answers `=` and `<` on text too short to make a match term, + and makes one only for `matches`. A resolver adds the parameter; a caller + of `record::query` passes the index it queries through. - **A ciphertext that opens as the wrong type is `Error::OpenedType`, not `Error::Aead`** (#1118). A typed decrypt of a leaf that authenticates but holds another type (an integer opened as `String`, a value too wide for diff --git a/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs b/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs index ad9265db0..402c6e441 100644 --- a/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs +++ b/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs @@ -104,6 +104,7 @@ impl TargetResolver for Fixed { fn query<'a, K: 'static>( &self, _: &str, + _: &stack_encrypt::target::IndexSpec, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index bb835c22e..fe1726039 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -767,8 +767,12 @@ where return Err(Error::Internal); }; let domain = target.query_sql_domain; - return Ok(super::record::query(cipher, plan, field, value, resolver)? - .map(move |json| made(domain, Operand::One(json)))); + // The answering index goes to the resolver: a type whose query + // carries only its operator's term derives that index's alone. + return Ok( + super::record::query(cipher, plan, field, &answer.index, value, resolver)? + .map(move |json| made(domain, Operand::One(json))), + ); } let context = answer.field.context(); let term = |value: &ValueFrame| { @@ -2362,6 +2366,7 @@ mod tests { fn query<'a, K: 'static>( &self, _: &str, + _: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, @@ -2538,6 +2543,7 @@ mod tests { fn query<'a, K: 'static>( &self, _: &str, + _: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index 262e1c53c..e39a91b35 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -1447,6 +1447,12 @@ pub fn encrypt_with<'a, K: 'static>( /// settles without I/O; it is a [`Pending`] all the same, for one shape at /// the call site. /// +/// `index` is the type's index that answers the query's operator, handed to +/// the resolver ([`TargetResolver::query`]): a type whose query carries only +/// the term its operator reads derives that index's term alone. +/// [`query_term`](super::query_term()) passes the index +/// [`OpSpec::answered_by`](super::OpSpec::answered_by) picks. +/// /// # Errors /// /// [`Error::Plan`] if the plan has no field `field` or it is not a target @@ -1457,6 +1463,7 @@ pub fn query<'a, K: 'static>( cipher: &'a KeysetCipher<'_, K>, plan: &Plan, field: &str, + index: &IndexSpec, value: impl Encrypt, resolver: &(impl TargetResolver + ?Sized), ) -> Result, K>, Error> { @@ -1478,7 +1485,7 @@ pub fn query<'a, K: 'static>( })?; check_field(&value, field)?; resolver - .query(name, cipher, field.label(), value) + .query(name, index, cipher, field.label(), value) .map_err(|error| Error::Target(name_target(error, &field.name))) } @@ -5533,6 +5540,7 @@ mod tests { fn query<'a, K: 'static>( &self, name: &str, + _: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, label: &Label, plaintext: ValueFrame, @@ -6249,16 +6257,30 @@ mod tests { let cipher = cipher().await; let keyset = cipher.default_keyset(); let plan = mixed_plan(); - let probe = query(&keyset, &plan, "email", s("a@x"), &FakeEql) - .expect("a target field") - .await - .expect("derives"); + let probe = query( + &keyset, + &plan, + "email", + &IndexSpec::Equality, + s("a@x"), + &FakeEql, + ) + .expect("a target field") + .await + .expect("derives"); let probe: serde_json::Value = serde_json::from_slice(&probe).unwrap(); assert_eq!(probe["i"], "users/email"); - let again = query(&keyset, &plan, "email", s("a@x"), &FakeEql) - .unwrap() - .await - .unwrap(); + let again = query( + &keyset, + &plan, + "email", + &IndexSpec::Equality, + s("a@x"), + &FakeEql, + ) + .unwrap() + .await + .unwrap(); assert_eq!( serde_json::from_slice::(&again).unwrap()["hm"], probe["hm"], @@ -6269,17 +6291,38 @@ mod tests { // bare build: each refused before the resolver runs. The first // two name the field asked for and why. for (field, reason) in [("age", Reason::NotATarget), ("nope", Reason::NoSuchField)] { - let error = refused(query(&keyset, &plan, field, s("x"), &FakeEql)); + let error = refused(query( + &keyset, + &plan, + field, + &IndexSpec::Equality, + s("x"), + &FakeEql, + )); assert!(matches!(error, Error::Plan { .. }), "{error:?}"); assert_eq!(error.field(), Some(field), "{error}"); assert_eq!(error.reason(), Some(reason), "{error}"); } assert!(matches!( - refused(query(&keyset, &plan, "email", Tree::UInt32(1), &FakeEql)), + refused(query( + &keyset, + &plan, + "email", + &IndexSpec::Equality, + Tree::UInt32(1), + &FakeEql + )), Error::Source { .. } )); assert!(matches!( - refused(query(&keyset, &plan, "email", s("x"), &NoTargets)), + refused(query( + &keyset, + &plan, + "email", + &IndexSpec::Equality, + s("x"), + &NoTargets + )), Error::Target(TargetError::NoTargets { .. }) )); } diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index a15a57faf..e34902c2d 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -31,6 +31,7 @@ use super::query::target_operators; use super::{OpSpec, ValueFrame, ValueKind}; use crate::sem::MatchOptions; +use crate::target::IndexSpec; use crate::{KeysetCipher, Label, Pending, StackCipher}; /// One EQL type a plan may name as a target, as the host describes it. @@ -518,12 +519,23 @@ pub trait TargetResolver { /// Run the named type's query twin over one value: the operand that /// matches stored values under `label`'s column, as JSON bytes. /// + /// `index` is the type's index that answers the query's operator, one of + /// the descriptor's [`indexes`](TargetDescriptor::indexes): + /// [`query_term`](super::query_term()) passes the one + /// [`OpSpec::answered_by`] picks. A twin that carries every term of the + /// type may ignore it. A twin whose terms are optional (EQL v4's + /// `TextSearchQuery`) carries that index's term alone, so a query whose + /// operator needs one term is not refused for want of another: a match + /// term, which text shorter than one token cannot make, is derived only + /// for `matches`. + /// /// # Errors /// /// As [`encrypt`](Self::encrypt). fn query<'a, K: 'static>( &self, name: &str, + index: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, label: &Label, plaintext: ValueFrame, @@ -573,6 +585,7 @@ impl TargetResolver for NoTargets { fn query<'a, K: 'static>( &self, name: &str, + _: &IndexSpec, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, @@ -650,6 +663,7 @@ mod tests { fn query<'a, K: 'static>( &self, _: &str, + _: &IndexSpec, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, From 73583d2dd2825affa82d1ddcc79f6d382830a35b Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 19 Oct 2026 11:11:09 +1100 Subject: [PATCH 3/4] feat(eql-bindings): produce TextSearch, its query carrying only its operator's term TextSearch was unproducible because its query twin carried hm, op and bf on every query, and text shorter than one 3-gram has no bf: `=` or `<` on a two-letter name would have been refused. EQL v4's search query domain now takes any one term, so the twin can carry just the one its operator reads. In v4, TextSearchQuery (and the unproducible TextSearchOreQuery) hold each term as an Option, skipped in the JSON when None. The generator emits one projection per term, TextSearchEqualityQuery, TextSearchOrderQuery and TextSearchMatchQuery, each deriving EncryptFrom over the envelope and its one term and converting into TextSearchQuery. The by-name query dispatch, which now receives the answering index, runs the projection for that index, so a match term is derived only for `matches`. The twin itself carries no derive. The stored TextSearch carries all three terms, the same bytes TextEq, TextOrd and TextMatch store, so text too short for a 3-gram is refused on write with EmptyTermText, exactly as TextMatch refuses it: a stored row with an empty filter would be one no match query finds. TextSearch joins ENCRYPTION_DOMAINS, the regenerated target table marks it producible, and the Go encrypt/eql table follows. The engine answers eq, lt, lte, gt, gte and matches on it. Tests: each index's query carries its term alone and equal to the stored one; "Al" is queried by eq and ope and refused for match; the typed projections serialize to what the dispatch writes; and, against an EQL v4 database, `=`, `<>`, `<`, `>=`, `@@` and the function forms find the plaintexts' rows, a one-term operand is a query_text_search and a termless one is not, and each operator and extractor refuses an operand missing its term with invalid_parameter_value naming the term and never the operand. --- .../eql-v4-text-search-optional-terms.md | 2 +- languages/golang/CHANGELOG.md | 9 + languages/golang/cmd/stashgen/README.md | 5 +- languages/golang/encrypt/README.md | 9 +- languages/golang/encrypt/eql/eql_gen.go | 4 +- languages/golang/encrypt/eql/eql_test.go | 2 +- languages/golang/encrypt/guest/src/targets.rs | 1 + .../golang/encrypt/guest/tests/native_ops.rs | 14 + languages/golang/encrypt/query_test.go | 10 + packages/eql/crates/eql-bindings/CHANGELOG.md | 32 + packages/eql/crates/eql-bindings/README.md | 21 +- .../eql/crates/eql-bindings/src/encryption.rs | 56 +- .../eql-bindings/src/encryption/targets.rs | 48 +- .../eql/crates/eql-bindings/src/v4/mod.rs | 57 +- .../eql/crates/eql-bindings/src/v4/targets.rs | 43 +- .../eql/crates/eql-bindings/src/v4/text.rs | 113 +++- .../eql/crates/eql-codegen/src/bindings.rs | 292 +++++++-- .../eql/crates/eql-codegen/src/targets.rs | 128 +++- packages/eql/mise.toml | 2 +- .../eql/tests/encryption/tests/targets.rs | 4 +- .../tests/encryption/tests/typed_queries.rs | 4 +- .../eql/tests/encryption/tests/v4_order.rs | 7 +- .../tests/encryption/tests/v4_text_search.rs | 582 ++++++++++++++++++ .../eql/tests/encryption/tests/v4_types.rs | 60 +- 24 files changed, 1367 insertions(+), 138 deletions(-) create mode 100644 packages/eql/tests/encryption/tests/v4_text_search.rs diff --git a/.changeset/eql-v4-text-search-optional-terms.md b/.changeset/eql-v4-text-search-optional-terms.md index 36ac34674..eb7d1e9dc 100644 --- a/.changeset/eql-v4-text-search-optional-terms.md +++ b/.changeset/eql-v4-text-search-optional-terms.md @@ -2,4 +2,4 @@ '@cipherstash/eql': minor --- -**An EQL v4 text search query carries only the terms its operator reads.** `eql_v4.query_text_search` and `eql_v4.query_text_search_ore` now require at least one of their terms (`hm`, `op` or `ob`, `bf`) instead of all of them, so `col = $1` and `col < $1` work on text too short to cut a 3-gram, which has no match term. Each extractor over such a query raises (`invalid_parameter_value`) when the query lacks the term it reads, naming the domain, the term and the operators that compare it: `=` without `hm`, `<` without `op`, `@@` without `bf`. A missing term never matches no row silently, nor every row. Stored `eql_v4_text_search` values still carry every term. The `eql_v3` domains are unchanged: the v3 bundle is byte for byte what it was. +**An EQL v4 text search query carries only the terms its operator reads.** `eql_v4.query_text_search` and `eql_v4.query_text_search_ore` now require at least one of their terms (`hm`, `op` or `ob`, `bf`) instead of all of them, so `col = $1` and `col < $1` work on text too short to cut a 3-gram, which has no match term. Each extractor over such a query raises (`invalid_parameter_value`) wherever it reads the operand when the query lacks the term it reads, or holds it as JSON `null` or an empty array, naming the domain, the term and the operators that compare it: `=` without `hm`, `<` without `op`, `@@` without `bf`. So a missing term raises rather than matching no row silently, or every row; the raise is at evaluation, so a generic plan that never reaches the predicate returns no row and no error. Stored `eql_v4_text_search` values still carry every term. The `eql_v3` domains are unchanged: the v3 bundle is byte for byte what it was. With it, **Stack Encrypt produces `TextSearch`** (`eql-bindings`, `stack-encrypt` feature): `TextSearchQuery`'s terms are `Option`s, built per index through `TextSearchEqualityQuery`, `TextSearchOrderQuery` and `TextSearchMatchQuery`, and the stored value carries every term: text too short for a 3-gram stores the empty filter (`"bf": []`, the new `v4::terms::SearchBloomFilter` field type), so `=` and `<` find it and no `@@` query does, where `TextMatch` still refuses it. `TextSearchQuery`'s `Option` terms and `TextSearch`'s `SearchBloomFilter` change the fields of the unreleased `v4` module only; no published `eql-bindings` has it. diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 2a78111e1..a47152c6a 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -116,6 +116,15 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. that orders by ADR-0002's order key: floats as PostgreSQL's `float8` sorts them, text by its fold. No `Between` yet: it needs EQL v4's range query domains. +- **`TextSearch`**: a `string` field tagged `encrypt_into=TextSearch` gets + `Eq`, `Lt`, `Lte`, `Gt`, `Gte` and `Matches`, one column for all of them. + The stored value keeps all three terms. Text shorter than three + characters, an empty string included, is stored with an empty match + filter, where `TextMatch` refuses it with `ErrTerm`: `Eq`, `Lt`, `Lte`, + `Gt` and `Gte` find the row, and no `Matches` query does. Each query + carries only the term its operator reads, so `Eq` and `Lt` take a + two-letter value and `Matches` refuses one (`ErrTerm`). Needs the EQL v4 + SQL from this release, whose `eql_v4.query_text_search` accepts one term. - **Ordering methods on text.** A string field with `index=ore` or `index=ope` gets `Lt`, `Lte`, `Gt`, `Gte` and `Between`. Text orders by its folded form (ADR-0002: case and accents folded, under Unicode 16), so diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 89e095cae..84cb034aa 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -158,7 +158,7 @@ The engine produces these EQL types today, each from one Go field type: | `encrypt_into` | Go field | |---|---| -| `TextEq`, `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe` | `string` | +| `TextEq`, `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe`, `TextSearch` | `string` | | `Smallint`, `SmallintEq`, `SmallintOrd`, `SmallintOrdOpe` | `int16`, or any Go integer type | | `Integer`, `IntegerEq`, `IntegerOrd`, `IntegerOrdOpe` | `int32`, or any Go integer type | | `Bigint`, `BigintEq`, `BigintOrd`, `BigintOrdOpe` | `int64`, or any Go integer type | @@ -169,12 +169,13 @@ The engine produces these EQL types today, each from one Go field type: | `Timestamp`, `TimestampEq` | `time.Time` | | `Numeric`, `NumericEq` | `encrypt.Decimal` | -The `Eq` types answer `Eq`; the `Ord` and `OrdOpe` types answer `Lt`, `Lte`, `Gt` and `Gte`, and on text `Eq` too; `Text`, `Smallint`, `Integer`, `Bigint`, `Boolean`, `Real`, `Double`, `Date`, `Timestamp` and `Numeric` are stored and read only, with no query method. +The `Eq` types answer `Eq`; the `Ord` and `OrdOpe` types answer `Lt`, `Lte`, `Gt` and `Gte`, and on text `Eq` too; `TextMatch` answers `Matches`, and `TextSearch` all of `Eq`, `Lt`, `Lte`, `Gt`, `Gte` and `Matches`, each query carrying only the term its operator reads; `Text`, `Smallint`, `Integer`, `Bigint`, `Boolean`, `Real`, `Double`, `Date`, `Timestamp` and `Numeric` are stored and read only, with no query method. The ordered date, timestamp and numeric types are refused: the engine derives no order term for these kinds yet. An ordered type has no `Between` method yet, because a `BETWEEN` operand needs EQL v4's range query domains. Any built-in Go integer field takes an integer type (ADR-0002) and seals at the type's width: an `int16` field always fits an `Integer`, and an `int64` field is range checked value by value, `Encrypt` refusing a value that does not fit with `encrypt.ErrEncoding`. `encrypt.Int128` and `encrypt.Uint128` are not converted. A float's width must match: a `float64` field is not a `Real`. A `TextMatch` field refuses text shorter than three characters, such as an empty string or a two-letter code, because it has no 3-grams to index; its query refuses such text too. +A `TextSearch` field stores such text with an empty match filter, so its `Eq`, `Lt`, `Lte`, `Gt` and `Gte` queries find it and no `Matches` query does; those queries take such text too, and its `Matches` query refuses it. Put the column on the left of `@@` (`WHERE name @@ $1::eql_v4.query_text_match`): the other operand order finds nothing until #1171 is fixed. `encrypt_into` with any other type is refused with the type's name and the reason it is not produced yet. Separate columns work today for four indexes: `equality`, `match`, `ore` and `ope`. diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index e7ff591eb..0b55d690e 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -488,6 +488,7 @@ The EQL types the engine produces, and the Go field types each takes: | `TextEq` | `string` | `=`, `<>` | | `TextMatch` | `string` | `@@` (substring match, 3-grams, case-insensitive) | | `TextOrd`, `TextOrdOpe` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `TextSearch` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY`, `@@` | | `Text` | `string` | none: stored and read only | | `Smallint` | `int16`, or any Go integer type (see below) | none | | `SmallintEq` | the same | `=`, `<>` | @@ -523,7 +524,13 @@ with `ErrEncoding`, naming the field's kind and not the value. exactly their own width. A `TextMatch` value must be at least three characters long: shorter text has -no 3-grams to index and is refused. The number equality types compare as +no 3-grams to index and is refused. A `TextSearch` value of such text, an +empty string included, is stored with an empty match filter: `Eq`, `Lt`, +`Lte`, `Gt` and `Gte` find it through its other terms, and no `Matches` +query does. A `TextSearch` query carries only the term its operator reads, +so `Eq` and `Lt` on two-letter text work and `Matches` on it is refused; EQL +v4 raises if a query lacks the term its operator compares. The number +equality types compare as PostgreSQL compares the plain types: an integer equals itself whatever its width, and for floats `-0.0 = 0.0` and `NaN = NaN`; the stored value still decrypts to exactly what was written. The same holds for the date, diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index 8d23c8332..1166b43ef 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -46,7 +46,7 @@ var Types = []Type{ {Name: "TextOrd", GoName: "TextOrd", Family: "text", Suffix: "Ord", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord", Indexes: []string{"eq", "ope"}, Query: "TextOrdQuery", QuerySQLDomain: "eql_v4.query_text_ord", Producible: true, Reason: ""}, {Name: "TextOrdOpe", GoName: "TextOrdOpe", Family: "text", Suffix: "OrdOpe", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ope", Indexes: []string{"eq", "ope"}, Query: "TextOrdOpeQuery", QuerySQLDomain: "eql_v4.query_text_ord_ope", Producible: true, Reason: ""}, {Name: "TextSearchOre", GoName: "TextSearchOre", Family: "text", Suffix: "SearchOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_search_ore", Indexes: []string{"eq", "ore", "match"}, Query: "TextSearchOreQuery", QuerySQLDomain: "eql_v4.query_text_search_ore", Producible: false, Reason: "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)"}, - {Name: "TextSearch", GoName: "TextSearch", Family: "text", Suffix: "Search", Plaintext: "string", SQLDomain: "public.eql_v4_text_search", Indexes: []string{"eq", "ope", "match"}, Query: "TextSearchQuery", QuerySQLDomain: "eql_v4.query_text_search", Producible: false, Reason: "every query of this type carries a match term, which text shorter than one 3-gram cannot make, so `=` or `<` on short text would be refused; how a query that is not a match leaves it out is not decided (#1131)"}, + {Name: "TextSearch", GoName: "TextSearch", Family: "text", Suffix: "Search", Plaintext: "string", SQLDomain: "public.eql_v4_text_search", Indexes: []string{"eq", "ope", "match"}, Query: "TextSearchQuery", QuerySQLDomain: "eql_v4.query_text_search", Producible: true, Reason: ""}, {Name: "Boolean", GoName: "Boolean", Family: "boolean", Suffix: "", Plaintext: "bool", SQLDomain: "public.eql_v4_boolean", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "Real", GoName: "Real", Family: "real", Suffix: "", Plaintext: "float32", SQLDomain: "public.eql_v4_real", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "RealEq", GoName: "RealEq", Family: "real", Suffix: "Eq", Plaintext: "float32", SQLDomain: "public.eql_v4_real_eq", Indexes: []string{"eq"}, Query: "RealEqQuery", QuerySQLDomain: "eql_v4.query_real_eq", Producible: true, Reason: ""}, @@ -1641,7 +1641,7 @@ func (v *TextSearchOreQuery) UnmarshalJSON(b []byte) error { } // TextSearch is the EQL type stored in public.eql_v4_text_search: the text family with the eq and ope and match indexes. -// The engine cannot produce it yet: every query of this type carries a match term, which text shorter than one 3-gram cannot make, so `=` or `<` on short text would be refused; how a query that is not a match leaves it out is not decided (#1131). +// The engine produces it from a string. type TextSearch []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. diff --git a/languages/golang/encrypt/eql/eql_test.go b/languages/golang/encrypt/eql/eql_test.go index 3758e261f..6a0563ffd 100644 --- a/languages/golang/encrypt/eql/eql_test.go +++ b/languages/golang/encrypt/eql/eql_test.go @@ -24,7 +24,7 @@ func TestTypesTableNamesTheProducibleTypes(t *testing.T) { "Smallint", "SmallintEq", "SmallintOrd", "SmallintOrdOpe", "Bigint", "BigintEq", "BigintOrd", "BigintOrdOpe", "Date", "DateEq", "Timestamp", "TimestampEq", "Numeric", "NumericEq", - "Text", "TextEq", "TextMatch", "TextOrd", "TextOrdOpe", + "Text", "TextEq", "TextMatch", "TextOrd", "TextOrdOpe", "TextSearch", "Boolean", "Real", "RealEq", "RealOrd", "RealOrdOpe", "Double", "DoubleEq", "DoubleOrd", "DoubleOrdOpe", diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index b19b3d6db..2095e3c1d 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -214,6 +214,7 @@ mod tests { "TextMatch", "TextOrd", "TextOrdOpe", + "TextSearch", "Boolean", "Real", "RealEq", diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index 4da201ab8..6bf48fd8c 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1386,6 +1386,7 @@ fn targets_lists_what_the_build_holds() { "TextMatch", "TextOrd", "TextOrdOpe", + "TextSearch", "Boolean", "Real", "RealEq", @@ -1812,6 +1813,19 @@ fn the_catalog_lists_each_types_operators() { ("arity", "one"), ("index", "match"), ]), + // Each operator through its own index, whose term alone the + // query carries. + "TextSearch" => { + let mut want = vec![exact("eq", "eq")]; + want.extend(orderings()); + want.push(vec![ + ("operator", "matches"), + ("fidelity", "probabilistic"), + ("arity", "one"), + ("index", "match"), + ]); + assert_eq!(listed, want, "{name}"); + } _ => { // A type the engine does not produce answers nothing, and // the other produced types are storage-only: no twin. diff --git a/languages/golang/encrypt/query_test.go b/languages/golang/encrypt/query_test.go index 7b948450a..9ba2f0fe5 100644 --- a/languages/golang/encrypt/query_test.go +++ b/languages/golang/encrypt/query_test.go @@ -430,6 +430,16 @@ func TestTheCatalogListsEachTypesOperators(t *testing.T) { if len(target.Operators) != 1 || target.Operators[0] != want { t.Fatalf("TextMatch answers %+v, want %+v", target.Operators, want) } + case target.Name == "TextSearch": + // Each through its own index: the query carries that term alone. + want := []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}} + for _, name := range []string{"lt", "lte", "gt", "gte"} { + want = append(want, record.Operator{Name: name, Fidelity: record.Exact, Index: record.Ope}) + } + want = append(want, record.Operator{Name: "matches", Fidelity: record.Probabilistic, Index: record.Match}) + if !slices.Equal(target.Operators, want) { + t.Fatalf("TextSearch answers %+v, want %+v", target.Operators, want) + } case len(target.Operators) != 0: // Every other type is storage-only or not produced. t.Fatalf("%s (producible %v) answers %+v, want none", target.Name, target.Producible, target.Operators) diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index 07fe98214..ab1637533 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -101,6 +101,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 orderable plaintext. Their query twins carry one bound, so the engine answers `<`, `<=`, `>` and `>=` (and `=` on text) through them, not `BETWEEN`. +- **`TextSearch` is producible** (`stack-encrypt` feature), from `String`. + The stored value carries `TextEq`'s `hm`, `TextOrd`'s `op` and + `TextMatch`'s `bf`. Text too short for a 3-gram (an empty string, a + two-letter code) is stored with the empty filter, `"bf": []`, where + `TextMatch` refuses it: `=` and `<` find the row through `hm` and `op`, + and no `@@` query does (a `@@` between two stored values is not a query, + and two stored empty filters match each other there, as in v3). Its `bf` + field is the new + `v4::terms::SearchBloomFilter` (as is `TextSearchOre`'s), whose JSON and + JSON Schema are `BloomFilter`'s and which takes the empty filter; + `TextMatch` and every query keep `BloomFilter`, which refuses it. Its + query is built per index, through + `TextSearchEqualityQuery`, `TextSearchOrderQuery` or + `TextSearchMatchQuery` (each `EncryptFrom` and + `Into`), so it carries only the term its operator reads: + `=` and `<` on two-letter text work, and a match query on it is refused. + The by-name dispatch picks the projection for the index it is handed. - **`encryption::targets::query` takes the index that answers the query's operator** (`targets` feature), an `&IndexSpec` after the name, as Stack Encrypt's `TargetResolver::query` hands it over. An index the type does @@ -145,6 +162,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 separately from `all()` (which stays the stored + SteVec conversion-target inventory). +### Breaking + +- **`v4::text::TextSearchQuery` and `TextSearchOreQuery` hold each term as + an `Option`**, left out of the JSON when `None`, to match EQL v4's + `query_text_search` and `query_text_search_ore`, which now take any one + of their terms (ADR-0002, amended 2026-10-19). `pub hm: Hmac256` is now + `pub hm: Option`, and likewise `op`/`ob` and `bf`: code that + builds one of these structs wraps each term in `Some`, code that reads + one handles `None`, and a JSON operand missing a term now deserializes + where it was refused. `TextSearch` and `TextSearchOre`'s `bf` is + `SearchBloomFilter`, not `BloomFilter` (`.0` reaches the `BloomFilter`). + No published release of this crate has the `v4` module (3.0.6 does not), + so this breaks only code built against unreleased sources; the v3 twins + are unchanged, as cipherstash-client's query carries every term. + ### Changed - **`TextEq` and `TextEqQuery` compare text in Unicode 16 NFC** diff --git a/packages/eql/crates/eql-bindings/README.md b/packages/eql/crates/eql-bindings/README.md index 19d3936d0..b8e5dbf3f 100644 --- a/packages/eql/crates/eql-bindings/README.md +++ b/packages/eql/crates/eql-bindings/README.md @@ -162,11 +162,22 @@ where that matters. Write the stored column on the left, `col @@ query`: the other order finds nothing until #1171 is fixed. Text too short to make a 3-gram (empty strings, two-letter codes) cannot be stored in a `TextMatch` column yet, and a `TextMatchQuery` for it is refused too: one filter type -builds both, and an empty filter as a query would match every row. The ORE -types (`*OrdOre`, `TextSearchOre`) wait for block ORE (#1132), `TextSearch` -for how a query that is not a match leaves out its match term (#1131), -`Date`, `Timestamp` and `Numeric` for a choice of their Rust plaintext types -(#1130), and `Json` for a decision on sealing a document as one leaf. +builds both, and an empty filter as a query would match every row. +`TextSearch` stores all three terms (`TextEq`'s `hm`, `TextOrd`'s `op` and +`TextMatch`'s `bf`). Text too short for a 3-gram is stored with the empty +filter (`"bf": []`, the `SearchBloomFilter` field type): `=` and `<` find it +through `hm` and `op`, and no `@@` query does, because a query's filter is +never empty. (A `@@` between two stored values, which no producer builds, is +not a query: two stored empty filters match each other there, as in v3.) Its query carries only the term its operator reads: build it +through `TextSearchEqualityQuery` (`=`, `<>`), `TextSearchOrderQuery` (`<`, +`<=`, `>`, `>=`) or `TextSearchMatchQuery` (`@@`) and convert into +`TextSearchQuery`, and the by-name dispatch picks the one for the index that +answers the operator. So `=` and `<` work on short text, a match query on it +is refused, and EQL v4 raises on an operator whose term the query lacks (or +holds as `null`, or as an empty filter). +The ORE types (`*OrdOre`, `TextSearchOre`) wait for block ORE, `Date`, +`Timestamp` and `Numeric` for a choice of their Rust plaintext types, and +`Json` for a decision on sealing a document as one leaf. Each ordered v4 family also has two range query types, one per order term: `IntegerRangeOreQuery` and `IntegerRangeOpeQuery`, and so on diff --git a/packages/eql/crates/eql-bindings/src/encryption.rs b/packages/eql/crates/eql-bindings/src/encryption.rs index 782224c14..a52bcd357 100644 --- a/packages/eql/crates/eql-bindings/src/encryption.rs +++ b/packages/eql/crates/eql-bindings/src/encryption.rs @@ -113,6 +113,7 @@ use stack_encrypt::{ use crate::{ v3::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw}, + v4::terms::SearchBloomFilter, Identifier, }; @@ -331,11 +332,12 @@ impl MatchConfig for EqlMatch { /// /// Text that cuts no token (empty, separator-only, or shorter than a 3-gram) /// is refused with [`TermError::EmptyTermText`], stored or queried. Stack -/// Encrypt stores such a value as the empty filter, but EQL treats an empty -/// filter as contained by every stored filter, and this one type builds both -/// the stored filter and the query operand, so it cannot tell a query from a -/// stored value: refusing both is the one way a query cannot match every -/// row. +/// Encrypt stores such a value as the empty filter, but this type is the +/// filter of every match query, and a query's filter must never be empty; +/// it is also `TextMatch`'s stored filter, and a `TextMatch` value with an +/// empty filter is a row no operator can find. A search domain's stored +/// filter, which `=` and `<` find through its other terms, is +/// [`SearchBloomFilter`] and takes the empty filter ([`SearchMatchVisitor`]). #[doc(hidden)] pub struct MatchVisitor; impl Visitor for MatchVisitor { @@ -404,6 +406,50 @@ impl DecryptField for BloomFilter { } } +/// Builds a stored search value's filter: [`MatchVisitor`]'s filter, except +/// that text which cuts no token stores the empty filter instead of being +/// refused (see [`SearchBloomFilter`]). +#[doc(hidden)] +pub struct SearchMatchVisitor; +impl Visitor for SearchMatchVisitor { + type Value = SearchBloomFilter; + // As in `MatchVisitor`: the one spelling of the terms' type that compiles + // against both the crates.io stack-encrypt and the tree. + #[allow(deprecated)] + fn matching(self, terms: MatchTerm) -> Result { + if terms.positions().is_empty() { + return Ok(SearchBloomFilter(BloomFilter(Vec::new()))); + } + MatchVisitor.matching(terms).map(SearchBloomFilter) + } +} +impl Transcode for SearchBloomFilter { + type Visitor = SearchMatchVisitor; + fn visitor() -> Self::Visitor { + SearchMatchVisitor + } +} +impl> EncryptFrom for SearchBloomFilter { + type Context = CallerContext; + fn encryption<'s, K: 'static>() -> Encryption<'s, S, Self, K, Self::Context> + where + S: 's, + { + // Configuration named by the type, as for `BloomFilter`. + #[allow(deprecated)] + let terms: Encryption<'s, S, MatchTerm, K, CallerContext> = target::matching(); + terms.transcode() + } +} +impl Decryptable for SearchBloomFilter { + const DECRYPTABLE: bool = false; +} +impl DecryptField for SearchBloomFilter { + fn decryption_field(self, _: Ctx) -> Option> { + None + } +} + /// Builds EQL's hex OPE term from the native order-preserving term. /// /// The bytes are Stack Encrypt's CLLW OPE ciphertext of the value's diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 2f4737605..3447065d0 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -347,7 +347,10 @@ pub fn decrypt<'a, K: 'static>( /// /// `index` is the type's index that answers the query's operator, one of /// [`Target::indexes`]; Stack Encrypt passes the one its -/// `OpSpec::answered_by` picks. +/// `OpSpec::answered_by` picks. Most twins carry every term of their type +/// whatever the index. `TextSearchQuery` carries the index's term alone +/// (EQL v4's search query terms are optional), so `=` and `<` on text too +/// short for a match term are answered, and `matches` on it is refused. /// /// # Errors /// @@ -369,17 +372,24 @@ pub fn query<'a, K: 'static>( } carries(target, index)?; let column = Identifier::from_label(context)?; - query_named(name, keyset, column, plaintext) + query_named(name, index, keyset, column, plaintext) } /// Refuse a query through an index the type does not carry. fn carries(target: &'static Target, index: &IndexSpec) -> Result<(), TargetError> { match target.indexes.iter().find(|key| **key == index.key()) { Some(_) => Ok(()), - None => Err(TargetError::NoIndex { - name: target.name, - index: index.key(), - }), + None => Err(no_index(target.name, index)), + } +} + +/// The refusal for a query on `name` through an index it does not carry. +/// Also the generated query dispatch's fall-through for a twin built per +/// index. +pub(crate) fn no_index(name: &'static str, index: &IndexSpec) -> TargetError { + TargetError::NoIndex { + name, + index: index.key(), } } @@ -558,6 +568,32 @@ where .try_map(|value| serde_json::to_vec(&value).map_err(|error| Error::Other(Box::new(error))))) } +/// Run the plan of `P`, a projection of the query twin `Q` that carries one +/// term, and resolve to `Q`'s JSON bytes: the twin of a type whose query +/// terms are optional, holding only the term `P` derives. The generated +/// query dispatch picks `P` by the index that answers the query's operator. +pub(crate) fn run_projected<'a, P, Q, S, K>( + name: &'static str, + keyset: &'a KeysetCipher<'_, K>, + column: NonEmpty, + plaintext: ValueFrame, +) -> Result, K>, TargetError> +where + P: EncryptFrom> + Into, + Q: Serialize, + S: Plaintext, + K: 'static, +{ + // Wiped when this returns, as in `run_target`. + let plaintext = zeroize::Zeroizing::new(S::from_frame(name, &plaintext)?); + Ok(keyset + .encrypt_as::(&plaintext, column) + .try_map(|value| { + let query: Q = value.into(); + serde_json::to_vec(&query).map_err(|error| Error::Other(Box::new(error))) + })) +} + /// Parse a stored EQL value as the target and describe opening it through /// the target's own `DecryptInto`, under the column the caller expects. pub(crate) fn open_target<'a, T, S, K>( diff --git a/packages/eql/crates/eql-bindings/src/v4/mod.rs b/packages/eql/crates/eql-bindings/src/v4/mod.rs index ea06e2889..24955deb5 100644 --- a/packages/eql/crates/eql-bindings/src/v4/mod.rs +++ b/packages/eql/crates/eql-bindings/src/v4/mod.rs @@ -23,8 +23,10 @@ //! published JSON Schema yet; those follow in `@cipherstash/eql` 4.0. The //! term newtypes and the [`DomainType`] trait are shared with v3: a v4 //! `Ciphertext` or `Hmac256` is v3's type, so a change to a term's validation -//! in `v3::terms` changes v4 too. What separates the two on the wire is the -//! envelope's [`SchemaVersion`], which refuses the other generation's `v`. +//! in `v3::terms` changes v4 too. The one v4 term v3 has no use for is +//! [`terms::SearchBloomFilter`], the stored filter of a search domain. What +//! separates the two on the wire is the envelope's [`SchemaVersion`], which +//! refuses the other generation's `v`. /// Generated: the `op::Answers` impls through which Stack Encrypt's typed /// query chain queries a producible type, each type's @@ -58,9 +60,58 @@ pub mod text; pub mod timestamp; pub use crate::v3::domain_type::{DomainType, SQL_SCHEMA}; -pub use crate::v3::terms; pub use inventory::{all, all_query, all_range_query}; +/// The wire-field newtypes of the v4 payloads: v3's, shared, and +/// [`SearchBloomFilter`](terms::SearchBloomFilter). +pub mod terms { + pub use crate::v3::terms::*; + + use schemars::JsonSchema; + use serde::{Deserialize, Serialize}; + use ts_rs::TS; + + /// The `bf` term of a stored search value (`TextSearch`, + /// `TextSearchOre`): a [`BloomFilter`] that may be empty. + /// + /// Text too short to cut a 3-gram (empty, separator-only, or shorter than + /// three characters) has no match tokens. A search column stores it with + /// the empty filter, Stack Encrypt's answer for a stored value, so `=` + /// and `<` still find it through its `hm` and `op`, and no `@@` query + /// does: EQL's `matches` finds a stored filter only when it contains the + /// query's, and an EQL v4 search query's filter is never empty (the + /// match projection refuses such text, and the SQL refuses `"bf": []`). + /// + /// [`BloomFilter`], the filter of `TextMatch` and of every query, still + /// refuses such text: `TextMatch` has no other term to find it by. The + /// JSON is [`BloomFilter`]'s, and so is the JSON Schema; only what Stack + /// Encrypt derives into it differs. + #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TS)] + #[serde(transparent)] + pub struct SearchBloomFilter(pub BloomFilter); + + /// [`BloomFilter`]'s schema, under its name: the wire format is one. + impl JsonSchema for SearchBloomFilter { + fn schema_name() -> std::borrow::Cow<'static, str> { + BloomFilter::schema_name() + } + + fn schema_id() -> std::borrow::Cow<'static, str> { + BloomFilter::schema_id() + } + + fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + BloomFilter::json_schema(generator) + } + } + + impl From for BloomFilter { + fn from(filter: SearchBloomFilter) -> Self { + filter.0 + } + } +} + /// The PostgreSQL typname prefix every `public` v4 domain carries. pub const PUBLIC_TYPNAME_PREFIX: &str = "eql_v4_"; diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index 67d8e09e0..698748bfe 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -1,13 +1,15 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The EQL types a Stack Encrypt data plan may name as a field target — every stored EQL v4 domain type in eql-domains::CATALOG order, as data a guest serializes (`TARGETS`), with the by-name dispatch that runs a producible type's own Rust plan. Generated from the catalog; the descriptor type, the errors and the public entry points stay hand-written in `crate::encryption::targets`, which documents the wire format. use crate::encryption::targets::{ - open_target, refuse, refuse_query, run_target, Opener, Target, TargetError, + no_index, open_target, refuse, refuse_query, run_projected, run_target, Opener, Target, + TargetError, }; use crate::v4::plaintext::Date; use crate::v4::plaintext::Decimal; use crate::v4::plaintext::Timestamp; use crate::Identifier; use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::target::IndexSpec; use stack_encrypt::{KeysetCipher, NonEmpty, Pending}; /// Every EQL type a plan may name as a target, in /// `eql-domains::CATALOG` order: one per stored domain (every flat @@ -495,10 +497,8 @@ pub const TARGETS: &[Target] = &[ indexes: &["eq", "ope", "match"], query: Some("TextSearchQuery"), query_sql_domain: Some("eql_v4.query_text_search"), - producible: false, - reason: Some( - "every query of this type carries a match term, which text shorter than one 3-gram cannot make, so `=` or `<` on short text would be refused; how a query that is not a match leaves it out is not decided (#1131)", - ), + producible: true, + reason: None, }, Target { name: "Boolean", @@ -766,6 +766,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), + "TextSearch" => run_target::( + "TextSearch", + keyset, + column, + plaintext, + ), "Boolean" => { run_target::("Boolean", keyset, column, plaintext) } @@ -884,6 +890,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "TextOrdOpe" => { open_target::("TextOrdOpe", opener, column, stored) } + "TextSearch" => { + open_target::("TextSearch", opener, column, stored) + } "Boolean" => { open_target::("Boolean", opener, column, stored) } @@ -912,8 +921,11 @@ pub(crate) fn decrypt_named<'a, K: 'static>( /// Run the named type's query twin for one plaintext, or refuse the /// name: as `encrypt_named` does, or as answering no query /// (`TargetError::NoQuery`) for a producible type with no twin. +/// `index` answers the query's operator; a twin whose terms are +/// optional carries that index's term alone. pub(crate) fn query_named<'a, K: 'static>( name: &str, + index: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, column: NonEmpty, plaintext: ValueFrame, @@ -1003,6 +1015,27 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), + "TextSearch" => match index.key() { + "eq" => run_projected::< + super::text::TextSearchEqualityQuery, + super::text::TextSearchQuery, + String, + K, + >("TextSearch", keyset, column, plaintext), + "ope" => run_projected::< + super::text::TextSearchOrderQuery, + super::text::TextSearchQuery, + String, + K, + >("TextSearch", keyset, column, plaintext), + "match" => run_projected::< + super::text::TextSearchMatchQuery, + super::text::TextSearchQuery, + String, + K, + >("TextSearch", keyset, column, plaintext), + _ => Err(no_index("TextSearch", index)), + }, "RealEq" => { run_target::("RealEq", keyset, column, plaintext) } diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index fb7e5546a..2de5a1458 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -1,6 +1,6 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `text` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw, OreBlock256}; +use crate::v4::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw, OreBlock256, SearchBloomFilter}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; @@ -316,7 +316,7 @@ pub struct TextSearchOre { pub c: Ciphertext, pub hm: Hmac256, pub ob: OreBlock256, - pub bf: BloomFilter, + pub bf: SearchBloomFilter, } impl DomainType for TextSearchOre { fn sql_domain_static() -> &'static str { @@ -352,15 +352,23 @@ impl DomainType for TextSearchOre { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +/// See the [complete encryption example](crate::encryption#example). +#[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] +#[stash(plaintext = String)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextSearch { + #[stash(default)] pub v: SchemaVersion, + #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, pub op: OpeCllw, - pub bf: BloomFilter, + pub bf: SearchBloomFilter, } impl DomainType for TextSearch { fn sql_domain_static() -> &'static str { @@ -630,7 +638,7 @@ impl DomainType for TextOrdOpeQuery { } /// `eql_v4.query_text_search_ore` — search domain query operand. /// -/// Operators: `=` `<>` `<` `<=` `>` `>=` `@@`. Required keys: `v` `i` `hm` `ob` `bf`. +/// Operators: `=` `<>` `<` `<=` `>` `>=` `@@`. Required keys: `v` `i`, and at least one of `hm` `ob` `bf`: a query carries the term its operator reads. #[cfg_attr(doc, doc = "")] #[cfg_attr( doc, @@ -641,9 +649,12 @@ impl DomainType for TextOrdOpeQuery { pub struct TextSearchOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub hm: Hmac256, - pub ob: OreBlock256, - pub bf: BloomFilter, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ob: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bf: Option, } impl DomainType for TextSearchOreQuery { fn sql_domain_static() -> &'static str { @@ -673,20 +684,25 @@ impl DomainType for TextSearchOreQuery { } /// `eql_v4.query_text_search` — search domain query operand. /// -/// Operators: `=` `<>` `<` `<=` `>` `>=` `@@`. Required keys: `v` `i` `hm` `op` `bf`. +/// Operators: `=` `<>` `<` `<=` `>` `>=` `@@`. Required keys: `v` `i`, and at least one of `hm` `op` `bf`: a query carries the term its operator reads. #[cfg_attr(doc, doc = "")] #[cfg_attr( doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +/// +/// Plaintext input: [`String`](String), through one projection per index — [`TextSearchEqualityQuery`], [`TextSearchOrderQuery`], [`TextSearchMatchQuery`] — each `EncryptFrom` and `Into`: a query carries the term of the index that answers its operator. Query operands do not decrypt. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextSearchQuery { pub v: SchemaVersion, pub i: Identifier, - pub hm: Hmac256, - pub op: OpeCllw, - pub bf: BloomFilter, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub op: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bf: Option, } impl DomainType for TextSearchQuery { fn sql_domain_static() -> &'static str { @@ -714,6 +730,81 @@ impl DomainType for TextSearchQuery { format!("{}{}.json", super::SCHEMA_ID_BASE, self.domain()) } } +/// The `eql_v4.query_text_search` query operand for `=` `<>`: the `hm` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchEqualityQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub hm: Hmac256, +} +impl From for TextSearchQuery { + fn from(query: TextSearchEqualityQuery) -> Self { + TextSearchQuery { + v: query.v, + i: query.i, + hm: Some(query.hm), + op: None, + bf: None, + } + } +} +/// The `eql_v4.query_text_search` query operand for `<` `<=` `>` `>=`: the `op` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchOrderQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub op: OpeCllw, +} +impl From for TextSearchQuery { + fn from(query: TextSearchOrderQuery) -> Self { + TextSearchQuery { + v: query.v, + i: query.i, + op: Some(query.op), + hm: None, + bf: None, + } + } +} +/// The `eql_v4.query_text_search` query operand for `@@`: the `bf` term alone. +/// +/// Implements `EncryptFrom`, deriving only this term, and converts into [`TextSearchQuery`], which is what is bound in SQL. +/// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. +#[derive(stack_encrypt::EncryptFrom)] +#[stash(plaintext = String)] +#[derive(Clone, Debug, PartialEq)] +pub struct TextSearchMatchQuery { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub bf: BloomFilter, +} +impl From for TextSearchQuery { + fn from(query: TextSearchMatchQuery) -> Self { + TextSearchQuery { + v: query.v, + i: query.i, + bf: Some(query.bf), + hm: None, + op: None, + } + } +} /// `eql_v4.query_text_range_ore` — block ORE range query operand: both bounds of one `BETWEEN`. /// /// Function: `eql_v4.between(column, range)`, both bounds included. Columns: `public.eql_v4_text_ord_ore` `public.eql_v4_text_search_ore`. Required keys: `v` `i` `lo` `hi`. diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index c2d7dd1da..35a1fedc3 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -14,6 +14,7 @@ use quote::{format_ident, quote}; use eql_domains::{Domain, DomainFamily, Term, CATALOG, ENVELOPE_KEYS, RANGE_BOUND_KEYS}; use crate::consts::RUST_GENERATED_MARKER; +use crate::context::v4_optional_query_terms; use crate::render::EqlVersion; use crate::writer::{ ensure_generated_paths_writable, normalized_set, remove_generated_orphans, @@ -198,7 +199,7 @@ fn render_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) -> Tok fields.extend(quote! { pub c: Ciphertext, }); for term in Term::payload_terms(domain.terms) { let fid = format_ident!("{}", term.json_key()); - let tid = format_ident!("{}", term.binding_newtype()); + let tid = format_ident!("{}", stored_term_newtype(eql, domain, term)); fields.extend(quote! { pub #fid: #tid, }); } @@ -276,13 +277,30 @@ fn render_query_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) .map(|o| format!("`{o}`")) .collect::>() .join(" "); - let keys_str = ["v", "i"] - .into_iter() - .chain(Term::term_json_keys(domain.terms)) - .map(|k| format!("`{k}`")) - .collect::>() - .join(" "); - let detail = format!(" Operators: {ops_str}. Required keys: {keys_str}."); + let quoted = |keys: &mut dyn Iterator| { + keys.map(|k| format!("`{k}`")).collect::>().join(" ") + }; + // An EQL v4 query whose terms are optional carries the one its operator + // reads (ADR-0002, amended 2026-10-19): each term is an `Option`, absent + // from the JSON when `None`, and the SQL domain requires one of them. + let optional = eql == EqlVersion::V4 && v4_optional_query_terms(domain); + let detail = if optional { + format!( + " Operators: {ops_str}. Required keys: {}, and at least one of {}: a query carries \ + the term its operator reads.", + quoted(&mut ["v", "i"].into_iter()), + quoted(&mut Term::term_json_keys(domain.terms).into_iter()), + ) + } else { + format!( + " Operators: {ops_str}. Required keys: {}.", + quoted( + &mut ["v", "i"] + .into_iter() + .chain(Term::term_json_keys(domain.terms)) + ) + ) + }; // Envelope minus `c`: `v`/`i` only. Kept in lockstep with the storage // struct's envelope triple (see `envelope_fields_match_catalog_keys`). @@ -292,9 +310,17 @@ fn render_query_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) for term in Term::payload_terms(domain.terms) { let fid = format_ident!("{}", term.json_key()); let tid = format_ident!("{}", term.binding_newtype()); - fields.extend(quote! { pub #fid: #tid, }); + fields.extend(if optional { + quote! { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub #fid: Option<#tid>, + } + } else { + quote! { pub #fid: #tid, } + }); } let term_keys = Term::term_json_keys(domain.terms); + let projections = render_query_projections(eql, family, domain); quote! { #[doc = #summary] @@ -330,6 +356,8 @@ fn render_query_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) } #schema_id } + + #projections } } @@ -411,11 +439,144 @@ fn render_range_query_struct(family: &DomainFamily, term: Term) -> TokenStream { } } +/// The newtype of a stored struct's `term` field. The term's own +/// ([`Term::binding_newtype`]), except the match term of an EQL v4 search +/// domain (one whose query terms are optional, [`v4_optional_query_terms`]): +/// `SearchBloomFilter`, the same JSON as `BloomFilter`, into which Stack +/// Encrypt derives the empty filter for text too short for a 3-gram instead +/// of refusing it. Such a value is still found by `=` and `<` through its +/// other terms; a query, and `TextMatch`, which has no other term, keep +/// `BloomFilter` and keep refusing that text. +pub fn stored_term_newtype(eql: EqlVersion, domain: &Domain, term: Term) -> &'static str { + if eql == EqlVersion::V4 && term == Term::Bloom && v4_optional_query_terms(domain) { + "SearchBloomFilter" + } else { + term.binding_newtype() + } +} + +/// The term newtypes a family module names: each stored struct's +/// ([`stored_term_newtype`]) and each query struct's +/// ([`Term::binding_newtype`]), after `Ciphertext`, in first-use order. +fn family_terms<'d>( + eql: EqlVersion, + domains: impl Iterator, +) -> Vec<&'static str> { + let mut used: Vec<&'static str> = vec!["Ciphertext"]; + for d in domains { + for term in Term::payload_terms(d.terms) { + for t in [stored_term_newtype(eql, d, term), term.binding_newtype()] { + if !used.contains(&t) { + used.push(t); + } + } + } + } + used +} + +/// The name of the projection of `domain`'s query twin that carries `term` +/// alone: `TextSearchEqualityQuery`, `TextSearchOrderQuery`, +/// `TextSearchMatchQuery`. Named by what the term answers, not by its key, +/// so the order projection of the ORE twin would read `…OrderQuery` too. +pub fn query_projection_ident(family: &DomainFamily, domain: &Domain, term: Term) -> String { + let role = match term { + Term::Hm => "Equality", + Term::Ore | Term::Ope => "Order", + Term::Bloom => "Match", + }; + format!("{}{role}Query", domain.struct_ident(family.name)) +} + +/// For a producible EQL v4 query twin whose terms are optional, one +/// projection per term: a struct carrying the envelope and that term alone, +/// deriving `EncryptFrom` so it derives only that term, and converting into +/// the twin. Stack Encrypt builds the twin through the projection of the +/// index that answers the query's operator, so a query never derives a term +/// its operator does not read (a match term, which text shorter than one +/// 3-gram cannot make, is derived only for `@@`). Not domains: no +/// `DomainType`, not in the inventory. Empty for every other domain. +fn render_query_projections( + eql: EqlVersion, + family: &DomainFamily, + domain: &Domain, +) -> TokenStream { + if eql != EqlVersion::V4 + || !v4_optional_query_terms(domain) + || encryption_gap(eql, family, domain).is_some() + { + return TokenStream::new(); + } + let twin = format_ident!("{}Query", domain.struct_ident(family.name)); + let query_domain = format!("{}.{}", eql.prefix(), domain.query_name(family.name)); + let (_, rust) = plaintext_of(family.name).expect("a derived family names its plaintext"); + let plaintext_ty: syn::Type = syn::parse_str(rust).expect("plaintext is a type path"); + let shown = rust.rsplit("::").next().unwrap_or(rust); + let terms = Term::payload_terms(domain.terms); + terms + .iter() + .map(|&term| { + let ident = format_ident!("{}", query_projection_ident(family, domain, term)); + let fid = format_ident!("{}", term.json_key()); + let tid = format_ident!("{}", term.binding_newtype()); + let ops = Term::operators_for_terms(domain.terms) + .into_iter() + .filter(|op| Term::extractor_for_operator(domain.terms, op) == Some(term.extractor())) + .map(|op| format!("`{op}`")) + .collect::>() + .join(" "); + let summary = format!( + " The `{query_domain}` query operand for {ops}: the `{}` term alone.", + term.json_key() + ); + let detail = format!( + " Implements `EncryptFrom<{shown}>`, deriving only this term, and converts into \ + [`{twin}`], which is what is bound in SQL." + ); + let others: TokenStream = terms + .iter() + .filter(|&&other| other != term) + .map(|other| { + let oid = format_ident!("{}", other.json_key()); + quote!(#oid: None,) + }) + .collect(); + quote! { + #[doc = #summary] + #[doc = ""] + #[doc = #detail] + #[doc = " Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`."] + #[derive(stack_encrypt::EncryptFrom)] + #[stash(plaintext = #plaintext_ty)] + #[derive(Clone, Debug, PartialEq)] + pub struct #ident { + #[stash(default)] + pub v: SchemaVersion, + #[stash(context_field)] + pub i: Identifier, + pub #fid: #tid, + } + + impl From<#ident> for #twin { + fn from(query: #ident) -> Self { + #twin { + v: query.v, + i: query.i, + #fid: Some(query.#fid), + #others + } + } + } + } + }) + .collect() +} + /// The `(family, domain)` pairs whose generated v4 structs carry the opt-in /// `stack-encrypt` derives: the storage and equality domains of every family /// whose plaintext eql-bindings names (`plaintext_of`), the OPE-ordered /// domains (`ord`, `ord_ope`) of those whose order terms the engine derives, -/// and text match. Every other stored domain has a reason in +/// and text match and search. Every other stored domain has a reason in /// [`encryption_gap`], and a test holds the two together so a new domain /// cannot arrive without either a derive or a reason. pub const ENCRYPTION_DOMAINS: &[(&str, &str)] = &[ @@ -442,6 +603,7 @@ pub const ENCRYPTION_DOMAINS: &[(&str, &str)] = &[ ("text", "match"), ("text", "ord"), ("text", "ord_ope"), + ("text", "search"), ("boolean", ""), ("real", ""), ("real", "eq"), @@ -518,15 +680,11 @@ pub fn encryption_gap( ordered is not decided (#1236)" } else if domain.terms.contains(&Term::Ore) { "EQL stores block ORE terms and the engine derives CLLW ORE (#1132)" - } else if domain.terms.contains(&Term::Bloom) && domain.terms.contains(&Term::Ope) { - "every query of this type carries a match term, which text shorter than one 3-gram \ - cannot make, so `=` or `<` on short text would be refused; how a query that is not a \ - match leaves it out is not decided (#1131)" } else { // Reached by any unlisted domain of a family whose plaintext is - // specified and whose terms are neither ORE nor match-and-order (an - // equality or OPE-ordered domain of a family that gains a plaintext, - // say): nothing in the catalog today. + // specified and whose terms are not ORE (an equality or OPE-ordered + // domain of a family that gains a plaintext, say): nothing in the + // catalog today. "this domain derives the way its family's other domains do; it is not listed in \ ENCRYPTION_DOMAINS yet" }) @@ -574,6 +732,28 @@ fn encryption_attrs( } }; let plaintext = plaintext_doc(&format!("[`{shown}`]({rust})")); + // The twin of an optional-term query is built by its projections + // (`render_query_projections`), one per index, so it carries no derive. + if query && v4_optional_query_terms(domain) { + let projections = Term::payload_terms(domain.terms) + .into_iter() + .map(|term| format!("[`{}`]", query_projection_ident(family, domain, term))) + .collect::>() + .join(", "); + let built = format!( + " Plaintext input: [`{shown}`]({rust}), through one projection per index — \ + {projections} — each `EncryptFrom<{shown}>` and `Into`: a query carries the \ + term of the index that answers its operator. Query operands do not decrypt." + ); + return ( + quote! { + #[doc = ""] + #[doc = #built] + }, + quote!(), + quote!(), + ); + } ( quote! { #[doc = ""] @@ -636,15 +816,7 @@ fn family_imports(version: EqlVersion, used: &[&'static str]) -> TokenStream { /// (exactly the term newtypes the family uses) followed by every domain's /// storage struct + impl, then a query twin for each term-bearing domain. pub fn render_family_bindings(version: EqlVersion, family: &DomainFamily) -> String { - let mut used: Vec<&'static str> = vec!["Ciphertext"]; - for d in family.domains { - for term in Term::payload_terms(d.terms) { - let t = term.binding_newtype(); - if !used.contains(&t) { - used.push(t); - } - } - } + let used = family_terms(version, family.domains.iter()); let imports = family_imports(version, &used); // Storage structs for every domain, then a query twin for each term-bearing @@ -698,15 +870,7 @@ pub fn render_family_bindings(version: EqlVersion, family: &DomainFamily) -> Str /// `json.rs` re-exports the generated struct (`pub use ...::Json;`) so /// `super::json::Json` resolves for the inventory/payload renderers. pub fn render_scalar_only_bindings(version: EqlVersion, family: &DomainFamily) -> String { - let mut used: Vec<&'static str> = vec!["Ciphertext"]; - for d in family.domains.iter().filter(|d| d.is_scalar()) { - for term in Term::payload_terms(d.terms) { - let t = term.binding_newtype(); - if !used.contains(&t) { - used.push(t); - } - } - } + let used = family_terms(version, family.domains.iter().filter(|d| d.is_scalar())); let imports = family_imports(version, &used); // Scalar storage structs only, then a query twin for each term-bearing scalar @@ -1460,6 +1624,50 @@ mod tests { ); } + /// A v4 search domain's stored filter is `SearchBloomFilter`, which takes + /// the empty filter of text too short for a 3-gram; `TextMatch` and every + /// query keep `BloomFilter`, which refuses that text. v3 is untouched. + #[test] + fn a_v4_search_domain_stores_a_filter_that_may_be_empty() { + let v4 = render_family_bindings(EqlVersion::V4, family("text")); + let field = |out: &str, name: &str| { + let start = out + .find(&format!("pub struct {name} {{")) + .unwrap_or_else(|| panic!("no struct {name}")); + let body = &out[start..]; + let body = &body[..body.find('}').unwrap()]; + body.lines() + .find(|line| line.contains("pub bf:")) + .map(|line| line.trim().to_owned()) + }; + assert_eq!( + field(&v4, "TextSearch").as_deref(), + Some("pub bf: SearchBloomFilter,") + ); + assert_eq!( + field(&v4, "TextSearchOre").as_deref(), + Some("pub bf: SearchBloomFilter,") + ); + assert_eq!( + field(&v4, "TextMatch").as_deref(), + Some("pub bf: BloomFilter,") + ); + assert_eq!( + field(&v4, "TextMatchQuery").as_deref(), + Some("pub bf: BloomFilter,") + ); + assert_eq!( + field(&v4, "TextSearchMatchQuery").as_deref(), + Some("pub bf: BloomFilter,") + ); + assert_eq!( + field(&v4, "TextSearchQuery").as_deref(), + Some("pub bf: Option,") + ); + let v3 = render_family_bindings(EqlVersion::V3, family("text")); + assert!(!v3.contains("SearchBloomFilter")); + } + #[test] fn bool_storage_only_family_has_one_struct_no_terms() { let out = render_family_bindings(EqlVersion::V3, family("boolean")); @@ -1878,15 +2086,8 @@ mod tests { "no order term for a date, timestamp or decimal" } else if domain.terms.contains(&Term::Ore) { "block ORE terms" - } else if domain.terms.contains(&Term::Bloom) - && domain.terms.contains(&Term::Ope) - { - "carries a match term" } else { - panic!( - "{full}: every specified domain without ORE or a match-and-order \ - query is derived: {reason}" - ) + panic!("{full}: every specified domain without ORE is derived: {reason}") }; assert!(reason.contains(expected), "{full}: {reason}"); } @@ -1907,9 +2108,12 @@ mod tests { let attrs = encryption_attrs(EqlVersion::V4, family, domain, query) .0 .to_string(); + // A query whose terms are optional is built by its per-index + // projections, which derive in its place. + let projected = query && v4_optional_query_terms(domain); assert_eq!( attrs.contains("stack_encrypt :: EncryptFrom"), - listed, + listed && !projected, "{} (query = {query})", domain.full_name(family.name) ); diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index 44ebcb9e7..7b56b91f8 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -31,17 +31,23 @@ //! //! The reasons a type is not producible: eql-bindings names no Rust //! plaintext type for the family; the engine derives no order term for a -//! date, timestamp or decimal; EQL's ORE term is block ORE where the engine -//! derives CLLW ORE (#1132); a -//! `TextSearch` query carries a match term that short text cannot make -//! (#1131); and the JSON index is a new engine operation (#1060). +//! date, timestamp or decimal; EQL's ORE term is block ORE where the +//! engine derives CLLW ORE; and the JSON index is a new engine operation. +//! Each reason string names the issue that tracks it. +//! +//! The query dispatch takes the index that answers the query's operator. A +//! twin whose terms are optional in EQL v4 (`TextSearchQuery`) is built +//! through the projection for that index, so it carries that term alone. use proc_macro2::TokenStream; use quote::{format_ident, quote}; use eql_domains::{Domain, DomainFamily, Shape, Term}; -use crate::bindings::{encryption_gap, format_rs, plaintext_of, stored_payload_domains}; +use crate::bindings::{ + encryption_gap, format_rs, plaintext_of, query_projection_ident, stored_payload_domains, +}; +use crate::context::v4_optional_query_terms; use crate::render::EqlVersion; /// The version every target is produced into. @@ -167,6 +173,11 @@ pub(crate) struct Row { /// The query twin's struct identifier and SQL domain, if the type /// answers a query. pub(crate) query: Option<(String, String)>, + /// For a query twin whose terms are optional in EQL v4, the projection + /// that builds it for each index, by the index's key: the query dispatch + /// runs the one for the index that answers the query's operator. + /// Empty for every other twin, whose plan derives all its terms. + pub(crate) projections: Vec<(&'static str, String)>, pub(crate) reason: Option<&'static str>, /// The operators the type answers in the typed query chain: none /// unless it is producible and has a query twin ([`answers_for_indexes`]). @@ -253,6 +264,19 @@ fn row(family: &'static DomainFamily, domain: &'static Domain) -> Row { ), indexes, query, + projections: if !is_stevec && v4_optional_query_terms(domain) { + Term::payload_terms(domain.terms) + .into_iter() + .map(|term| { + ( + index_key(term), + query_projection_ident(family, domain, term), + ) + }) + .collect() + } else { + Vec::new() + }, reason, answers, } @@ -411,14 +435,45 @@ fn render_targets_from(rows: &[Row]) -> String { .collect(); // A producible type with no query twin (a storage-only domain) gets no // query arm: the fall-through refuses it as answering no query. + // + // A twin whose terms are optional is built through the projection of + // the index that answers the query's operator, and carries that term + // alone; any other index is one the type does not carry, which + // `encryption::targets::query` refuses before dispatching here. let query_arms: TokenStream = producible .iter() .filter_map(|r| { let (query, _) = r.query.as_ref()?; let (name, ty, source) = arm(r, query); - Some(quote! { #name => run_target::<#ty, #source, K>(#name, keyset, column, plaintext), }) + if r.projections.is_empty() { + return Some( + quote! { #name => run_target::<#ty, #source, K>(#name, keyset, column, plaintext), }, + ); + } + let by_index: TokenStream = r + .projections + .iter() + .map(|(key, projection)| { + let (_, projection, _) = arm(r, projection); + quote! { + #key => run_projected::<#projection, #ty, #source, K>(#name, keyset, column, plaintext), + } + }) + .collect(); + Some(quote! { + #name => match index.key() { + #by_index + _ => Err(no_index(#name, index)), + }, + }) }) .collect(); + let projected = producible.iter().any(|r| !r.projections.is_empty()); + let index_param = if projected { + quote!(index) + } else { + quote!(_index) + }; // One `use` per plaintext a producible type names by path (`String` is // in the prelude), so the arms stay short enough to read. let mut imports: Vec<&str> = producible @@ -435,10 +490,10 @@ fn render_targets_from(rows: &[Row]) -> String { quote!(use #path;) }) .collect(); - let helpers = if producible.is_empty() { - quote!() - } else { - quote!(open_target, run_target,) + let helpers = match (producible.is_empty(), projected) { + (true, _) => quote!(), + (false, false) => quote!(open_target, run_target,), + (false, true) => quote!(no_index, open_target, run_projected, run_target,), }; let mod_doc = " The EQL types a Stack Encrypt data plan may name as a field target — \ @@ -453,6 +508,7 @@ fn render_targets_from(rows: &[Row]) -> String { #![doc = #mod_doc] use stack_encrypt::dynamic::ValueFrame; + use stack_encrypt::target::IndexSpec; #imports use crate::encryption::targets::{#helpers refuse, refuse_query, Opener, Target, TargetError}; @@ -499,8 +555,11 @@ fn render_targets_from(rows: &[Row]) -> String { /// Run the named type's query twin for one plaintext, or refuse the /// name: as `encrypt_named` does, or as answering no query /// (`TargetError::NoQuery`) for a producible type with no twin. + /// `index` answers the query's operator; a twin whose terms are + /// optional carries that index's term alone. pub(crate) fn query_named<'a, K: 'static>( name: &str, + #index_param: &IndexSpec, keyset: &'a KeysetCipher<'_, K>, column: NonEmpty, plaintext: ValueFrame, @@ -652,13 +711,9 @@ mod tests { target_gap(family, family.domain_by_name(name).expect("domain")) }; let text = eql_domains::TEXT; - for producible in ["", "eq", "match", "ord", "ord_ope"] { + for producible in ["", "eq", "match", "ord", "ord_ope", "search"] { assert!(gap(&text, producible).is_none(), "text {producible:?}"); } - assert!( - gap(&text, "search").unwrap().contains("match term"), - "TextSearch waits on how a query leaves out its match term" - ); for block_ore in ["ord_ore", "search_ore"] { assert!( gap(&text, block_ore).unwrap().contains("block ORE"), @@ -784,29 +839,58 @@ mod tests { }; assert_eq!(count("run_target", &name), 1, "{name}: one encrypt arm"); assert_eq!(count("open_target", &name), 1, "{name}: one decrypt arm"); + let projected = v4_optional_query_terms(domain); assert_eq!( count("run_target", &format!("{name}Query")), - usize::from(!domain.terms.is_empty()), + usize::from(!domain.terms.is_empty() && !projected), "{name}: a query arm exactly when it has a query twin" ); + if projected { + // One arm per index, each through the projection for it. + for term in Term::payload_terms(domain.terms) { + let projection = query_projection_ident(family, domain, term); + let needle = format!( + "\"{}\"=>run_projected::(\"{name}\",", + index_key(term) + ); + // rustfmt leaves a trailing comma in a broken turbofish. + let unbroken = tight.replace(",>", ">"); + assert_eq!(unbroken.matches(&needle).count(), 1, "{needle}"); + } + } } // No unproducible type has an arm: the only `=>` arms are the - // producible ones and the three fall-throughs. - let with_twin = ENCRYPTION_DOMAINS + // producible ones, one per index of a projected twin, and the + // fall-throughs. + let (with_twin, by_index) = ENCRYPTION_DOMAINS .iter() - .filter(|(f, d)| { + .map(|(f, d)| { let family = eql_domains::CATALOG.iter().find(|x| x.name == *f).unwrap(); - !family.domain_by_name(d).unwrap().terms.is_empty() + family.domain_by_name(d).unwrap() }) - .count(); + .filter(|domain| !domain.terms.is_empty()) + .fold((0, 0), |(twins, by_index), domain| { + let indexes = if v4_optional_query_terms(domain) { + Term::payload_terms(domain.terms).len() + } else { + 0 + }; + (twins + 1, by_index + indexes) + }); + assert!(by_index > 0, "TextSearch's query is projected"); let arms = out.matches("\" => ").count(); assert_eq!( arms, - ENCRYPTION_DOMAINS.len() * 2 + with_twin, + ENCRYPTION_DOMAINS.len() * 2 + with_twin + by_index, "arms: {out}" ); assert_eq!(out.matches("_ => Err(refuse(name))").count(), 2); assert_eq!(out.matches("_ => Err(refuse_query(name))").count(), 1); + assert_eq!( + out.matches("_ => Err(no_index(\"TextSearch\", index))") + .count(), + 1 + ); } /// A type answers operators in the typed query chain exactly when the diff --git a/packages/eql/mise.toml b/packages/eql/mise.toml index 128ab2fcb..a13b0fbc5 100644 --- a/packages/eql/mise.toml +++ b/packages/eql/mise.toml @@ -340,7 +340,7 @@ psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -f release/cipherstash-encrypt- # `--ignored` selects by attribute, and cargo exits 0 on an empty selection, so # a test that loses its `#[ignore]` or is renamed would leave this task passing # on nothing. List what the filter selects before running it. -for suite in text_eq v4_types v4_order v4_date_time_numeric v4_range; do +for suite in text_eq v4_types v4_order v4_date_time_numeric v4_range v4_text_search; do listing=$(cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored --list | sed -n 's/: test$//p') [ -n "$listing" ] || { echo "error: '--ignored' selects no test in $suite; the PostgreSQL test has gone missing." >&2; exit 1; } cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index 43091ada3..203bda902 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -585,7 +585,7 @@ mod the_table { /// The types the engine produces in EQL v4 today: the storage and /// equality domains of every family whose plaintext eql-bindings names, /// the OPE-ordered domains of those the engine derives an order term - /// for, and text match. + /// for, and text match and search. #[test] fn the_producible_types_are_the_ones_with_no_ore_and_a_rust_plaintext() { let producible: Vec<&str> = targets::targets() @@ -619,6 +619,7 @@ mod the_table { "TextMatch", "TextOrd", "TextOrdOpe", + "TextSearch", "Boolean", "Real", "RealEq", @@ -639,7 +640,6 @@ mod the_table { "no order term for a date, timestamp or decimal yet" } (_, "OrdOre" | "SearchOre") => "derives CLLW ORE (#1132)", - ("text", "Search") => "carries a match term", (family, suffix) => panic!("{}: unexpected {family} {suffix}", row.name), }; assert!(reason.contains(expected), "{}: {reason}", row.name); diff --git a/packages/eql/tests/encryption/tests/typed_queries.rs b/packages/eql/tests/encryption/tests/typed_queries.rs index 26f4f7b05..5174df470 100644 --- a/packages/eql/tests/encryption/tests/typed_queries.rs +++ b/packages/eql/tests/encryption/tests/typed_queries.rs @@ -318,7 +318,8 @@ fn the_typed_chain_answers_what_the_data_path_answers() { } } } - // The gap today: every producible ordered type. A type leaves this list + // The gap today: every producible ordered type, and text search, whose + // twin carries an order term among its three. A type leaves this list // when the typed chain assembles its twin, and its answers are then // held to the data path's above. assert_eq!( @@ -332,6 +333,7 @@ fn the_typed_chain_answers_what_the_data_path_answers() { "BigintOrdOpe", "TextOrd", "TextOrdOpe", + "TextSearch", "RealOrd", "RealOrdOpe", "DoubleOrd", diff --git a/packages/eql/tests/encryption/tests/v4_order.rs b/packages/eql/tests/encryption/tests/v4_order.rs index 89852221f..dc3290840 100644 --- a/packages/eql/tests/encryption/tests/v4_order.rs +++ b/packages/eql/tests/encryption/tests/v4_order.rs @@ -81,14 +81,17 @@ fn op(value: &serde_json::Value) -> &str { value["op"].as_str().expect("an ordered value carries op") } -/// Every producible ordered type is in [`ORDERED`] and the other way -/// round: a type that becomes producible is covered here or fails. +/// Every producible `Ord` and `OrdOpe` type is in [`ORDERED`] and the other +/// way round: a type that becomes producible is covered here or fails. +/// `TextSearch` carries an OPE term too; its query carries one term per +/// operator, and `v4_text_search.rs` covers it. #[test] fn the_ordered_producible_types_are_these() { let mut ordered: Vec<&str> = targets::targets() .iter() .filter(|t| t.producible && t.indexes.contains(&"ope")) .map(|t| t.name) + .filter(|name| *name != "TextSearch") .collect(); ordered.sort_unstable(); let mut expected = ORDERED.to_vec(); diff --git a/packages/eql/tests/encryption/tests/v4_text_search.rs b/packages/eql/tests/encryption/tests/v4_text_search.rs new file mode 100644 index 000000000..bc90bb75e --- /dev/null +++ b/packages/eql/tests/encryption/tests/v4_text_search.rs @@ -0,0 +1,582 @@ +//! EQL v4's `TextSearch`: one column answering `=`, `<` and `@@`, whose +//! query carries only the term its operator reads (ADR-0002, amended +//! 2026-10-19). +//! +//! A stored value carries all three terms, each the one the single-index +//! type stores (`TextEq`'s `hm`, `TextOrd`'s `op`, `TextMatch`'s `bf`). +//! Text too short to cut a 3-gram stores the empty filter, where `TextMatch` +//! refuses it, so `=` and `<` find it and no `@@` query does. A query is built +//! through the projection of the index that answers its operator, so `=` and +//! `<` on such text work and a match query on it is refused. Against an EQL +//! v4 database: a query with any one term is a `query_text_search`, one with +//! none is not, and each operator refuses an operand without its term (or +//! with it `null`, or an empty filter) rather than matching no row. + +mod common; + +use eql_bindings::encryption::targets; +use eql_bindings::v4::text::{ + TextSearch, TextSearchEqualityQuery, TextSearchMatchQuery, TextSearchOrderQuery, + TextSearchQuery, +}; +use eql_bindings::Identifier; +use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::sem::TermError; +use stack_encrypt::target::IndexSpec; +use stack_encrypt::{Encrypt, Label, NonEmpty}; + +fn frame(value: T) -> ValueFrame { + ValueFrame::of(value).unwrap() +} + +fn column() -> NonEmpty { + Identifier::for_column("people", "name").unwrap() +} + +fn label() -> Label { + Label::new(["people", "name"]).unwrap() +} + +/// The index a query on `TextSearch` is asked through, by its key. +fn index(key: &str) -> IndexSpec { + IndexSpec::parse(key).unwrap() +} + +/// The stored JSON the named type writes for `text`. +async fn stored( + keyset: &stack_encrypt::KeysetCipher<'_, common::Observed>, + name: &str, + text: &str, +) -> serde_json::Value { + let bytes = targets::encrypt(name, keyset, &label(), frame(text)) + .unwrap() + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} + +/// The `TextSearch` query for `text` through the index `key`. +async fn query( + keyset: &stack_encrypt::KeysetCipher<'_, common::Observed>, + key: &str, + text: &str, +) -> Result { + let bytes = targets::query("TextSearch", &index(key), keyset, &label(), frame(text)) + .unwrap() + .await?; + Ok(serde_json::from_slice(&bytes).unwrap()) +} + +fn keys(value: &serde_json::Value) -> Vec<&str> { + let mut keys: Vec<&str> = value + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + keys +} + +#[tokio::test] +async fn a_stored_value_carries_the_three_single_index_terms() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let search = stored(&keyset, "TextSearch", "Alice").await; + assert_eq!(keys(&search), ["bf", "c", "hm", "i", "op", "v"]); + assert_eq!(search["v"], 4); + assert_eq!(search["hm"], stored(&keyset, "TextEq", "Alice").await["hm"]); + assert_eq!( + search["op"], + stored(&keyset, "TextOrd", "Alice").await["op"] + ); + assert_eq!( + search["bf"], + stored(&keyset, "TextMatch", "Alice").await["bf"] + ); + let opened = targets::decrypt( + "TextSearch", + &cipher, + &label(), + search.to_string().as_bytes(), + ) + .unwrap() + .await + .unwrap(); + assert_eq!(opened.risky_ref(), frame("Alice").risky_ref()); +} + +/// Text too short to cut a 3-gram has no match tokens. `TextSearch` stores +/// it with the empty filter and the same `hm` and `op` as `TextEq` and +/// `TextOrd`, so `=` and `<` find it and no match query does. `TextMatch`, +/// which has no other term to find it by, still refuses it. +#[tokio::test] +async fn short_text_stores_the_empty_filter_where_text_match_refuses_it() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + for text in ["Al", "", " "] { + let search = stored(&keyset, "TextSearch", text).await; + assert_eq!(keys(&search), ["bf", "c", "hm", "i", "op", "v"], "{text:?}"); + assert_eq!(search["bf"], serde_json::json!([]), "{text:?}"); + assert_eq!( + search["hm"], + stored(&keyset, "TextEq", text).await["hm"], + "{text:?}" + ); + assert_eq!( + search["op"], + stored(&keyset, "TextOrd", text).await["op"], + "{text:?}" + ); + let opened = targets::decrypt( + "TextSearch", + &cipher, + &label(), + search.to_string().as_bytes(), + ) + .unwrap() + .await + .unwrap(); + assert_eq!(opened.risky_ref(), frame(text).risky_ref(), "{text:?}"); + + let refused = targets::encrypt("TextMatch", &keyset, &label(), frame(text)) + .unwrap() + .await + .unwrap_err(); + assert!( + matches!( + refused, + stack_encrypt::Error::Term(TermError::EmptyTermText) + ), + "TextMatch {text:?}: {refused:?}" + ); + } +} + +/// Each index's query carries its term alone, and it is the stored one. +#[tokio::test] +async fn a_query_carries_only_the_term_its_operator_reads() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let search = stored(&keyset, "TextSearch", "Alice").await; + for (key, term) in [("eq", "hm"), ("ope", "op"), ("match", "bf")] { + let q = query(&keyset, key, "Alice").await.unwrap(); + assert_eq!(keys(&q), { + let mut expected = vec!["i", "v", term]; + expected.sort_unstable(); + expected + }); + assert_eq!(q[term], search[term], "{key}: the stored {term}"); + assert_eq!(q["v"], 4); + } + // An index the type does not carry is refused before anything runs. + let Err(refused) = targets::query("TextSearch", &IndexSpec::Ore, &keyset, &label(), frame("x")) + else { + panic!("an ore query on TextSearch was dispatched"); + }; + assert!( + matches!( + refused, + targets::TargetError::NoIndex { + name: "TextSearch", + index: "ore" + } + ), + "{refused:?}" + ); +} + +/// `=` and `<` on text shorter than one 3-gram: the query carries `hm` or +/// `op`, never an empty filter. A match query on it is refused, as a +/// `TextMatchQuery` is, rather than sent as a filter that matches every row. +#[tokio::test] +async fn short_text_is_queried_by_equality_and_order_and_refused_for_match() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let eq = query(&keyset, "eq", "Al").await.unwrap(); + assert_eq!(keys(&eq), ["hm", "i", "v"]); + assert!(eq["hm"].as_str().is_some_and(|hm| !hm.is_empty())); + let ord = query(&keyset, "ope", "Al").await.unwrap(); + assert_eq!(keys(&ord), ["i", "op", "v"]); + assert!(ord["op"].as_str().is_some_and(|op| !op.is_empty())); + // The order term of "Al" sorts before "Alice"'s: a prefix first. + let alice = stored(&keyset, "TextSearch", "Alice").await; + assert!(ord["op"].as_str().unwrap() < alice["op"].as_str().unwrap()); + + let refused = query(&keyset, "match", "Al").await.unwrap_err(); + assert!( + matches!( + refused, + stack_encrypt::Error::Term(TermError::EmptyTermText) + ), + "{refused:?}" + ); +} + +/// The typed path: each projection derives its one term and converts into +/// the `TextSearchQuery` the by-name dispatch writes, byte for byte. +#[tokio::test] +async fn the_typed_projections_are_what_the_dispatch_writes() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let text = "Al".to_owned(); + let eq: TextSearchEqualityQuery = keyset.encrypt_as(&text, column()).await.unwrap(); + let ord: TextSearchOrderQuery = keyset.encrypt_as(&text, column()).await.unwrap(); + for (typed, key) in [ + (TextSearchQuery::from(eq), "eq"), + (TextSearchQuery::from(ord), "ope"), + ] { + assert_eq!( + serde_json::to_value(&typed).unwrap(), + query(&keyset, key, "Al").await.unwrap(), + "{key}" + ); + } + let matched: TextSearchMatchQuery = keyset + .encrypt_as(&"Alice".to_owned(), column()) + .await + .unwrap(); + let matched = TextSearchQuery::from(matched); + assert!(matched.hm.is_none() && matched.op.is_none() && matched.bf.is_some()); + assert_eq!( + serde_json::to_value(&matched).unwrap(), + query(&keyset, "match", "Alice").await.unwrap() + ); + // A query with no term deserializes, as the JSON Schema allows; it is + // the database that refuses it. + let bare: TextSearchQuery = serde_json::from_value(serde_json::json!({ + "v": 4, "i": {"t": "people", "c": "name"} + })) + .unwrap(); + assert!(bare.hm.is_none() && bare.op.is_none() && bare.bf.is_none()); + + // The stored value round-trips through the typed path too. + let stored: TextSearch = keyset + .encrypt_as(&"Alice".to_owned(), column()) + .await + .unwrap(); + let opened: String = cipher.decrypt_as(stored, column().into()).await.unwrap(); + assert_eq!(opened, "Alice"); + // And short text through the typed path stores the empty filter, as the + // dispatch does. + let short: TextSearch = keyset.encrypt_as(&text, column()).await.unwrap(); + assert!(short.bf.0 .0.is_empty()); + assert_eq!( + serde_json::to_value(&short.bf).unwrap(), + serde_json::json!([]) + ); + let opened: String = cipher.decrypt_as(short, column().into()).await.unwrap(); + assert_eq!(opened, "Al"); +} + +// ---- PostgreSQL ------------------------------------------------------------- + +fn connect() -> postgres::Client { + postgres::Client::connect( + &std::env::var("EQL_TEST_DATABASE_URL").expect("EQL_TEST_DATABASE_URL"), + postgres::NoTls, + ) + .unwrap() +} + +/// SQLSTATE `check_violation`. +const CHECK_VIOLATION: &str = "23514"; +/// SQLSTATE `invalid_parameter_value`. +const INVALID_PARAMETER_VALUE: &str = "22023"; + +/// The rows whose `name` the SQL's predicate selects, the operand bound as +/// `$1`. +fn ids(db: &mut postgres::Client, sql: &str, operand: &serde_json::Value) -> Vec { + let mut ids: Vec = db + .query(sql, &[&operand.to_string()]) + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|r| r.get(0)) + .collect(); + ids.sort_unstable(); + ids +} + +/// The error the SQL raises with the operand bound as `$1`. +fn refusal(db: &mut postgres::Client, sql: &str, operand: &serde_json::Value) -> postgres::Error { + match db.query(sql, &[&operand.to_string()]) { + Ok(rows) => panic!("{sql}: answered {} rows, expected a refusal", rows.len()), + Err(error) => error, + } +} + +fn sqlstate(error: &postgres::Error) -> &str { + error.code().map_or("", |code| code.code()) +} + +/// Needs an EQL v4 installation. Rows written through the by-name dispatch, +/// queried with operands that carry one term each: `=`, `<`, `>` and `@@` +/// find the rows the plaintexts say, two-letter and empty text, stored with +/// the empty filter, included on `=` and `<` and found by no `@@`; a +/// `query_text_search` holds any one term and refuses none; and each +/// operator refuses an operand without its term, naming the term, never the +/// operand. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_answers_each_operator_through_its_own_term() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + db.batch_execute( + "CREATE TEMP TABLE v4_search (id int PRIMARY KEY, name public.eql_v4_text_search);", + ) + .unwrap(); + // 4 and 5 are too short for a 3-gram: stored with `"bf": []`, which the + // domain's CHECK accepts. + let names = [(1, "alice"), (2, "Bob"), (3, "carol"), (4, "Al"), (5, "")]; + for (id, name) in names { + let value = runtime.block_on(stored(&keyset, "TextSearch", name)); + db.execute( + "INSERT INTO v4_search VALUES ($1, $2::text::jsonb::public.eql_v4_text_search)", + &[&id, &value.to_string()], + ) + .unwrap(); + } + let q = |key: &str, text: &str| runtime.block_on(query(&keyset, key, text)).unwrap(); + let select = |predicate: &str| format!("SELECT id FROM v4_search WHERE {predicate}"); + let operand = "$1::text::jsonb::eql_v4.query_text_search"; + + // Each operator through its own term. + assert_eq!( + ids( + &mut db, + &select(&format!("name = {operand}")), + &q("eq", "Bob") + ), + [2] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name <> {operand}")), + &q("eq", "Bob") + ), + [1, 3, 4, 5] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name < {operand}")), + &q("ope", "bob") + ), + [1, 4, 5] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name >= {operand}")), + &q("ope", "bob") + ), + [2, 3] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name @@ {operand}")), + &q("match", "aro") + ), + [3] + ); + // Two-letter text, which has no match term, on `=` and `<`/`>`: stored + // and queried. + assert_eq!( + ids( + &mut db, + &select(&format!("name = {operand}")), + &q("eq", "Al") + ), + [4] + ); + assert_eq!( + ids(&mut db, &select(&format!("name = {operand}")), &q("eq", "")), + [5] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name > {operand}")), + &q("ope", "Bo") + ), + [2, 3] + ); + assert_eq!( + ids( + &mut db, + &select(&format!("name < {operand}")), + &q("ope", "Bo") + ), + [1, 4, 5] + ); + // The function forms, as PostgREST calls them. + assert_eq!( + ids( + &mut db, + &select(&format!("eql_v4.lt(name, {operand})")), + &q("ope", "bob") + ), + [1, 4, 5] + ); + // No match query finds a stored empty filter: `matches` needs the stored + // filter to contain the query's, and a query's is never empty. + for needle in ["ali", "alic", "Bob", "aro", "xyz"] { + let found = ids( + &mut db, + &select(&format!("name @@ {operand}")), + &q("match", needle), + ); + assert!( + !found.contains(&4) && !found.contains(&5), + "{needle:?} matched an empty filter: {found:?}" + ); + } + assert_eq!( + ids( + &mut db, + &select(&format!("name @@ {operand}")), + &q("match", "ali") + ), + [1] + ); + + // The domain holds a query with any one term, and refuses one with none. + let full = q("eq", "x"); + for key in ["eq", "ope", "match"] { + db.query(&format!("SELECT {operand}"), &[&q(key, "xyz").to_string()]) + .unwrap_or_else(|e| panic!("{key}: {e}")); + } + let mut bare = full.clone(); + bare.as_object_mut().unwrap().remove("hm"); + let error = refusal(&mut db, &format!("SELECT {operand}"), &bare); + assert_eq!(sqlstate(&error), CHECK_VIOLATION, "{error}"); + // The v3 operand is untouched: it still requires every term. + let mut v3 = q("eq", "x"); + v3["v"] = 3.into(); + // Only when the v3 bundle is installed beside v4; this database may + // hold v4 alone. + let has_v3: bool = db + .query_one( + "SELECT to_regtype('eql_v3.query_text_search') IS NOT NULL", + &[], + ) + .unwrap() + .get(0); + if has_v3 { + let error = refusal( + &mut db, + "SELECT $1::text::jsonb::eql_v3.query_text_search", + &v3, + ); + assert_eq!(sqlstate(&error), CHECK_VIOLATION, "{error}"); + } + + // Each operator refuses an operand without the term it reads, naming + // the term and never the operand. + for (predicate, key, missing) in [ + ("name = {}", "ope", "hm"), + ("name <> {}", "match", "hm"), + ("name < {}", "eq", "op"), + ("name >= {}", "match", "op"), + ("name @@ {}", "eq", "bf"), + ("name @@ {}", "ope", "bf"), + ("eql_v4.eq_term({}) IS NOT NULL", "ope", "hm"), + ("eql_v4.ord_term({}) IS NOT NULL", "eq", "op"), + ("eql_v4.match_term({}) IS NOT NULL", "ope", "bf"), + ] { + let sql = select(&predicate.replace("{}", operand)); + let without = q(key, "carol"); + let error = refusal(&mut db, &sql, &without); + assert_eq!(sqlstate(&error), INVALID_PARAMETER_VALUE, "{sql}: {error}"); + let message = error + .as_db_error() + .map(|e| e.message().to_owned()) + .unwrap_or_default(); + assert!( + message.contains(&format!( + "eql_v4.query_text_search value has no \"{missing}\" term" + )), + "{sql}: {message}" + ); + let carried = without.as_object().unwrap(); + for term in ["hm", "op", "bf"].iter().filter_map(|t| carried.get(*t)) { + let shown = term.to_string(); + let shown = shown.trim_matches('"'); + assert!( + !message.contains(shown), + "{sql}: the message quotes the operand" + ); + } + } + + // A term that is JSON `null`, or a filter with no positions, is as + // missing as an absent one. Neither comes from Stack Encrypt, which + // leaves an absent term out and never builds an empty query filter; a + // hand-built operand is refused rather than compared as NULL (no row), + // or as the empty filter only the short-text rows store. + let mut null_hm = q("eq", "carol"); + null_hm["hm"] = serde_json::Value::Null; + let mut empty_bf = q("match", "carol"); + empty_bf["bf"] = serde_json::json!([]); + for (predicate, operand_value, missing) in [ + ("name = {}", &null_hm, "hm"), + ("eql_v4.eq_term({}) IS NOT NULL", &null_hm, "hm"), + ("name @@ {}", &empty_bf, "bf"), + ("eql_v4.match_term({}) IS NOT NULL", &empty_bf, "bf"), + ] { + let sql = select(&predicate.replace("{}", operand)); + let error = refusal(&mut db, &sql, operand_value); + assert_eq!(sqlstate(&error), INVALID_PARAMETER_VALUE, "{sql}: {error}"); + let message = error + .as_db_error() + .map(|e| e.message().to_owned()) + .unwrap_or_default(); + assert!( + message.contains(&format!( + "eql_v4.query_text_search value has no \"{missing}\" term" + )), + "{sql}: {message}" + ); + } + + // The ORE twin follows the same rule, and a present `ob` is still a + // non-empty array. Its order term is EQL's block ORE, which the engine + // does not derive, so only the equality and match operands are real. + // An installer that could not create the ORE operator class (no + // superuser) poisons the ORE domains, and then there is nothing to test. + let poisoned: bool = db + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'eql_ore_unavailable' \ + AND contypid = 'eql_v4.query_text_search_ore'::regtype)", + &[], + ) + .unwrap() + .get(0); + if poisoned { + return; + } + let ore = "$1::text::jsonb::eql_v4.query_text_search_ore"; + db.query(&format!("SELECT {ore}"), &[&q("eq", "xyz").to_string()]) + .unwrap(); + db.query(&format!("SELECT {ore}"), &[&q("match", "xyz").to_string()]) + .unwrap(); + let mut empty_ob = q("eq", "xyz"); + empty_ob["ob"] = serde_json::json!([]); + let error = refusal(&mut db, &format!("SELECT {ore}"), &empty_ob); + assert_eq!(sqlstate(&error), CHECK_VIOLATION, "{error}"); + let error = refusal(&mut db, &format!("SELECT {ore}"), &bare); + assert_eq!(sqlstate(&error), CHECK_VIOLATION, "{error}"); + let error = refusal( + &mut db, + &format!("SELECT eql_v4.ord_term_ore({ore}) IS NOT NULL"), + &q("eq", "xyz"), + ); + assert_eq!(sqlstate(&error), INVALID_PARAMETER_VALUE, "{error}"); +} diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index 90c41e932..85c005610 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -444,33 +444,45 @@ async fn every_producible_type_dispatches_by_name() { ); } Some(_) => { - let query = targets::query( - name, - &common::index_of(name), - &keyset, - &label(), - sample(kind), - ) - .unwrap() - .await - .unwrap(); - let query: serde_json::Value = serde_json::from_slice(&query).unwrap(); - assert!( - query.get("c").is_none(), - "{name}: a query carries no ciphertext" - ); - if target.indexes.contains(&"eq") { - assert_eq!( - query["hm"], json["hm"], - "{name}: the query probes the stored term" + // Asked through each index the type carries, the query + // carries that index's term, the stored one. TextSearch's + // carries it alone (EQL v4's search query terms are + // optional); every other twin carries all of its terms + // whatever the index. + let term_of = |key: &str| match key { + "eq" => "hm", + "ope" => "op", + "match" => "bf", + other => panic!("{name}: no term for index {other}"), + }; + for key in target.indexes { + let index = stack_encrypt::target::IndexSpec::parse(key).unwrap(); + let query = targets::query(name, &index, &keyset, &label(), sample(kind)) + .unwrap() + .await + .unwrap(); + let query: serde_json::Value = serde_json::from_slice(&query).unwrap(); + assert!( + query.get("c").is_none(), + "{name}: a query carries no ciphertext" ); - } - if target.indexes.contains(&"ope") { - assert!(json["op"].is_string(), "{name} stores an op"); + let term = term_of(key); + assert!(json[term].is_string() || json[term].is_array(), "{name}"); assert_eq!( - query["op"], json["op"], - "{name}: the query compares the stored order term" + query[term], json[term], + "{name} via {key}: the query carries the stored term" ); + let carried: Vec<&str> = target + .indexes + .iter() + .map(|key| term_of(key)) + .filter(|term| query.get(term).is_some()) + .collect(); + if name == "TextSearch" { + assert_eq!(carried, [term], "{name} via {key}: its term alone"); + } else { + assert_eq!(carried.len(), target.indexes.len(), "{name} via {key}"); + } } } } From 213d998b3a35d264769ac97604bbc5e2a36caa08 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 19 Oct 2026 11:11:15 +1100 Subject: [PATCH 4/4] test(golang): generate and seal a TextSearch field, each query carrying one term A directory Entry model holds one TextSearch field. Generated over the deterministic EQL guest, its stored value carries bf, c, hm, i, op and v and opens back; Eq, Lt and Gte return a query with only hm or op, the stored term, for "Alice" and for the two-letter "Al"; Matches returns one with only bf, which Verify checks; and Matches on "Al", like storing "Qz" or "", is refused with ErrTerm without quoting the value. stashgen gets a golden case for the same field over its static engine, which now lists TextSearch with its three indexes, so the generated methods (Eq, Lt, Lte, Gt, Gte, Matches) are pinned and compile against the stub SDK. --- .../golang/encrypt/eql_v4_text_search_test.go | 159 ++++++++++++++++++ .../encrypt/internal/testusers/directory.go | 13 ++ .../encrypt/internal/testusers/entry_stash.go | 141 ++++++++++++++++ .../golang/stashgen/enginetest/enginetest.go | 1 + languages/golang/stashgen/golden_test.go | 3 + .../testdata/cases/directory/directory.go | 12 ++ .../cases/directory/entry_stash.go.golden | 141 ++++++++++++++++ .../stashgen/testdata/cases/directory/go.mod | 7 + .../testdata/stubsdk/encrypt/eql/eql.go | 6 + 9 files changed, 483 insertions(+) create mode 100644 languages/golang/encrypt/eql_v4_text_search_test.go create mode 100644 languages/golang/encrypt/internal/testusers/directory.go create mode 100644 languages/golang/encrypt/internal/testusers/entry_stash.go create mode 100644 languages/golang/stashgen/testdata/cases/directory/directory.go create mode 100644 languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden create mode 100644 languages/golang/stashgen/testdata/cases/directory/go.mod diff --git a/languages/golang/encrypt/eql_v4_text_search_test.go b/languages/golang/encrypt/eql_v4_text_search_test.go new file mode 100644 index 000000000..8f23f64d8 --- /dev/null +++ b/languages/golang/encrypt/eql_v4_text_search_test.go @@ -0,0 +1,159 @@ +package encrypt_test + +import ( + "context" + "encoding/json" + "errors" + "sort" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// EQL v4's TextSearch, sealed and queried through generated code over the +// deterministic guest build with the EQL types. A stored value carries all +// three terms, short text with the empty filter; a query carries only the +// one its operator reads (ADR-0002, amended 2026-10-19), so Eq and Lt take a +// two-letter name and Matches refuses one. +// packages/eql/tests/encryption/tests/v4_text_search.rs holds the same rules +// against PostgreSQL. + +// keysAndTerms is an EQL value's sorted top-level keys, and the value of +// each key. +func keysAndTerms(t *testing.T, value []byte) (string, map[string]json.RawMessage) { + t.Helper() + var raw map[string]json.RawMessage + if err := json.Unmarshal(value, &raw); err != nil { + t.Fatalf("the EQL value is not JSON: %v", err) + } + if string(raw["v"]) != "4" { + t.Errorf("v = %s, want 4", raw["v"]) + } + keys := make([]string, 0, len(raw)) + for k := range raw { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ","), raw +} + +func TestV4TextSearchStoresEveryTermAndQueriesCarryOne(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + encrypted, err := testusers.EncryptEntry(ctx, cipher, []testusers.Entry{{ID: 1, Name: "Alice"}}) + if err != nil { + t.Fatal(err) + } + keys, stored := keysAndTerms(t, encrypted[0].Name) + if keys != "bf,c,hm,i,op,v" { + t.Errorf("stored keys %s, want bf,c,hm,i,op,v", keys) + } + decrypted, err := testusers.DecryptEntry(ctx, cipher, encrypted) + if err != nil { + t.Fatal(err) + } + if decrypted[0].Name != "Alice" { + t.Errorf("opened %q", decrypted[0].Name) + } + + f := testusers.EntryFields.Name + // Each method's query carries its operator's term alone, the stored one. + for _, tc := range []struct { + method string + query func(string) ([]byte, error) + term string + }{ + {"Eq", func(v string) ([]byte, error) { return f.Eq(ctx, cipher, v) }, "hm"}, + {"Lt", func(v string) ([]byte, error) { return f.Lt(ctx, cipher, v) }, "op"}, + {"Gte", func(v string) ([]byte, error) { return f.Gte(ctx, cipher, v) }, "op"}, + } { + q, err := tc.query("Alice") + if err != nil { + t.Fatalf("%s: %v", tc.method, err) + } + keys, terms := keysAndTerms(t, q) + if want := strings.Join(sortedWith("i", "v", tc.term), ","); keys != want { + t.Errorf("%s query keys %s, want %s", tc.method, keys, want) + } + if string(terms[tc.term]) != string(stored[tc.term]) { + t.Errorf("%s query's %s is not the stored one", tc.method, tc.term) + } + // Text too short for a 3-gram still has an equality and an order + // term. + short, err := tc.query("Al") + if err != nil { + t.Fatalf("%s on two-letter text: %v", tc.method, err) + } + if keys, _ := keysAndTerms(t, short); keys != strings.Join(sortedWith("i", "v", tc.term), ",") { + t.Errorf("%s query keys on two-letter text %s", tc.method, keys) + } + } + + m, err := f.Matches(ctx, cipher, "Alice") + if err != nil { + t.Fatal(err) + } + keys, terms := keysAndTerms(t, m.Query()) + if keys != "bf,i,v" { + t.Errorf("Matches query keys %s, want bf,i,v", keys) + } + if string(terms["bf"]) != string(stored["bf"]) { + t.Error("the match query's bf is not the stored filter") + } + if ok, err := m.Verify(ctx, []string{"alice", "Bob"}); err != nil || !ok[0] || ok[1] { + t.Errorf("Verify = %v, %v", ok, err) + } + + // A match query on text too short for a 3-gram is refused, never sent + // as an empty filter. + if q, err := f.Matches(ctx, cipher, "Al"); !errors.Is(err, encrypt.ErrTerm) { + t.Errorf("Matches on two-letter text = %v, %v, want ErrTerm", q, err) + } + // Storing such text, an empty string included, keeps every term, the + // match term as the empty filter (where TextMatch refuses the text): Eq + // and Lt find the row through hm and op, and no Matches query does. + short, err := testusers.EncryptEntry(ctx, cipher, []testusers.Entry{{ID: 2, Name: "Al"}, {ID: 3, Name: ""}}) + if err != nil { + t.Fatalf("storing short text in a TextSearch: %v", err) + } + for i, name := range []string{"Al", ""} { + keys, terms := keysAndTerms(t, short[i].Name) + if keys != "bf,c,hm,i,op,v" { + t.Errorf("stored keys for %q %s, want bf,c,hm,i,op,v", name, keys) + } + if string(terms["bf"]) != "[]" { + t.Errorf("stored filter for %q is %s, want []", name, terms["bf"]) + } + eq, err := f.Eq(ctx, cipher, name) + if err != nil { + t.Fatalf("Eq(%q): %v", name, err) + } + if _, q := keysAndTerms(t, eq); string(q["hm"]) != string(terms["hm"]) { + t.Errorf("Eq(%q)'s hm is not the stored one", name) + } + lt, err := f.Lt(ctx, cipher, name) + if err != nil { + t.Fatalf("Lt(%q): %v", name, err) + } + if _, q := keysAndTerms(t, lt); string(q["op"]) != string(terms["op"]) { + t.Errorf("Lt(%q)'s op is not the stored one", name) + } + } + opened, err := testusers.DecryptEntry(ctx, cipher, short) + if err != nil { + t.Fatal(err) + } + if opened[0].Name != "Al" || opened[1].Name != "" { + t.Errorf("opened %q and %q", opened[0].Name, opened[1].Name) + } +} + +// sortedWith is the given keys, sorted. +func sortedWith(keys ...string) []string { + sort.Strings(keys) + return keys +} diff --git a/languages/golang/encrypt/internal/testusers/directory.go b/languages/golang/encrypt/internal/testusers/directory.go new file mode 100644 index 000000000..7850d7197 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/directory.go @@ -0,0 +1,13 @@ +package testusers + +//go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Entry -name Entry + +// Entry stores a name as EQL v4's TextSearch: one column that answers `=`, +// the orderings and `@@`. Its query carries only the term its operator +// reads, so `=` and `<` work on a two-letter name while `@@` on one is +// refused (ADR-0002, amended 2026-10-19). +type Entry struct { + _ struct{} `stash:"context=directory"` + ID int64 `stash:"id,passthrough"` + Name string `stash:"name,encrypt_into=TextSearch"` +} diff --git a/languages/golang/encrypt/internal/testusers/entry_stash.go b/languages/golang/encrypt/internal/testusers/entry_stash.go new file mode 100644 index 000000000..de9ce0098 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/entry_stash.go @@ -0,0 +1,141 @@ +// Code generated by stashgen. DO NOT EDIT. + +package testusers + +import ( + "context" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion1 + +// Entry prints its sealed fields in the clear: it has no String or LogValue +// method. Write them, or run stashgen with -redact. + +type EncryptedEntry struct { + ID int64 + Name eql.TextSearch +} + +func (e EncryptedEntry) String() string { + return gensupport.Redacted("EncryptedEntry", map[string]any{"ID": e.ID}, "Name") +} + +func (e EncryptedEntry) LogValue() slog.Value { + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "Name") +} + +// Stops compiling when Entry gains, loses, reorders or retypes a field. +var _ = entryShape(Entry{}) + +type entryShape struct { + _ struct{} + ID int64 + Name string +} + +var entryDeclaration = gensupport.Declare("directory"). + Passthrough("id"). + EncryptInto("name", gensupport.String, "TextSearch") + +var entryCodec = gensupport.New(gensupport.Generated[Entry, EncryptedEntry]{ + TypeName: "Entry", + Declaration: entryDeclaration, + PrintsPlaintext: true, + Source: func(v Entry) gensupport.Values { + return gensupport.Values{ + "id": v.ID, + "name": v.Name, + } + }, + Seal: func(rec gensupport.Record) (EncryptedEntry, error) { + var e EncryptedEntry + var err error + if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { + return EncryptedEntry{}, err + } + e.Name = eql.TextSearch(rec["name"].EQL) + return e, nil + }, + Open: func(e EncryptedEntry) gensupport.Record { + return gensupport.Record{ + "id": {Value: e.ID}, + "name": {EQL: e.Name}, + } + }, + Value: func(e EncryptedEntry, vals gensupport.Values) (Entry, error) { + var v Entry + var err error + if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { + return Entry{}, err + } + if v.Name, err = gensupport.Get[string](vals, "name"); err != nil { + return Entry{}, err + } + return v, nil + }, +}) + +// EncryptEntry seals each Entry, with one ZeroKMS request for each 500 sealed +// values, plus one the first time a keyset is used. The result has one element +// for each input, in the same order. +func EncryptEntry(ctx context.Context, cipher *encrypt.Cipher, values []Entry) ([]EncryptedEntry, error) { + return entryCodec.Encrypt(ctx, cipher, values) +} + +// DecryptEntry opens each EncryptedEntry, with one ZeroKMS request for each 500 +// sealed values, plus one the first time a keyset is used. +func DecryptEntry(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedEntry) ([]Entry, error) { + return entryCodec.Decrypt(ctx, d, encrypted) +} + +var EntryFields = struct { + Name EntryNameField +}{ + Name: EntryNameField{gensupport.NewField[string](entryDeclaration, "name")}, +} + +type EntryNameField struct { + field gensupport.Field[string] +} + +func (f EntryNameField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearch, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.TextSearch(out.EQL), err +} + +func (f EntryNameField) Eq(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "eq", v) + return eql.TextSearchQuery(term), err +} + +func (f EntryNameField) Lt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "lt", v) + return eql.TextSearchQuery(term), err +} + +func (f EntryNameField) Lte(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "lte", v) + return eql.TextSearchQuery(term), err +} + +func (f EntryNameField) Gt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "gt", v) + return eql.TextSearchQuery(term), err +} + +func (f EntryNameField) Gte(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "gte", v) + return eql.TextSearchQuery(term), err +} + +// Matches derives the matches query for v. Its answer can hold rows that do not +// match: decrypt the rows it fetches and keep those Verify reports true. +func (f EntryNameField) Matches(ctx context.Context, c *encrypt.Cipher, v string) (encrypt.EQLMatchQuery[eql.TextSearchQuery, string], error) { + return gensupport.EQLMatches[eql.TextSearchQuery](ctx, f.field, c, v) +} diff --git a/languages/golang/stashgen/enginetest/enginetest.go b/languages/golang/stashgen/enginetest/enginetest.go index 7aeba26bc..166b0bc6e 100644 --- a/languages/golang/stashgen/enginetest/enginetest.go +++ b/languages/golang/stashgen/enginetest/enginetest.go @@ -33,6 +33,7 @@ var eqlTypes = []struct { {stashgen.EQLType{Name: "Text", Plaintext: stashgen.KindString, Producible: true}, ""}, {stashgen.EQLType{Name: "TextEq", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality}, Query: "TextEqQuery", Producible: true}, ""}, {stashgen.EQLType{Name: "TextMatch", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexMatch}, Query: "TextMatchQuery", Producible: true}, ""}, + {stashgen.EQLType{Name: "TextSearch", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality, stashgen.IndexOpe, stashgen.IndexMatch}, Query: "TextSearchQuery", Producible: true}, ""}, {stashgen.EQLType{Name: "Boolean", Plaintext: stashgen.KindBool, Producible: true}, ""}, {stashgen.EQLType{Name: "Real", Plaintext: stashgen.KindFloat, Producible: true}, "float32"}, {stashgen.EQLType{Name: "Double", Plaintext: stashgen.KindFloat, Producible: true}, "float64"}, diff --git a/languages/golang/stashgen/golden_test.go b/languages/golang/stashgen/golden_test.go index fd66a8b48..f721f299a 100644 --- a/languages/golang/stashgen/golden_test.go +++ b/languages/golang/stashgen/golden_test.go @@ -39,6 +39,9 @@ var goldenCases = map[string]goldenCase{ // the context field to a column. "tenants": {req: stashgen.Request{Type: "Note", Models: []stashgen.ModelRequest{{Name: "Rows", Type: "NoteRow"}}}, golden: "note_stash.go.golden"}, "foreign": {req: stashgen.Request{Type: "individualStash", For: "pb.Individual"}, golden: "individualstash_stash.go.golden"}, + // An EQL type with three indexes, TextSearch: Eq, the orderings and + // Matches, each returning its own query. + "directory": {req: stashgen.Request{Type: "Entry"}, golden: "entry_stash.go.golden"}, } func TestGolden(t *testing.T) { diff --git a/languages/golang/stashgen/testdata/cases/directory/directory.go b/languages/golang/stashgen/testdata/cases/directory/directory.go new file mode 100644 index 000000000..23b1f8918 --- /dev/null +++ b/languages/golang/stashgen/testdata/cases/directory/directory.go @@ -0,0 +1,12 @@ +package directory + +//go:generate go tool stashgen -type Entry + +// Entry stores a name as EQL v4's TextSearch: one column that answers +// =, the orderings and @@, each query carrying only the term its operator +// reads. +type Entry struct { + _ struct{} `stash:"context=directory"` + ID int64 `stash:"id,passthrough" db:"id"` + Name string `stash:"name,encrypt_into=TextSearch" db:"name"` +} diff --git a/languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden b/languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden new file mode 100644 index 000000000..d0393bf7c --- /dev/null +++ b/languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden @@ -0,0 +1,141 @@ +// Code generated by stashgen. DO NOT EDIT. + +package directory + +import ( + "context" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion1 + +// Entry prints its sealed fields in the clear: it has no String or LogValue +// method. Write them, or run stashgen with -redact. + +type EncryptedEntry struct { + ID int64 `db:"id"` + Name eql.TextSearch `db:"name"` +} + +func (e EncryptedEntry) String() string { + return gensupport.Redacted("EncryptedEntry", map[string]any{"ID": e.ID}, "Name") +} + +func (e EncryptedEntry) LogValue() slog.Value { + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "Name") +} + +// Stops compiling when Entry gains, loses, reorders or retypes a field. +var _ = entryShape(Entry{}) + +type entryShape struct { + _ struct{} + ID int64 + Name string +} + +var declaration = gensupport.Declare("directory"). + Passthrough("id"). + EncryptInto("name", gensupport.String, "TextSearch") + +var codec = gensupport.New(gensupport.Generated[Entry, EncryptedEntry]{ + TypeName: "Entry", + Declaration: declaration, + PrintsPlaintext: true, + Source: func(v Entry) gensupport.Values { + return gensupport.Values{ + "id": v.ID, + "name": v.Name, + } + }, + Seal: func(rec gensupport.Record) (EncryptedEntry, error) { + var e EncryptedEntry + var err error + if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { + return EncryptedEntry{}, err + } + e.Name = eql.TextSearch(rec["name"].EQL) + return e, nil + }, + Open: func(e EncryptedEntry) gensupport.Record { + return gensupport.Record{ + "id": {Value: e.ID}, + "name": {EQL: e.Name}, + } + }, + Value: func(e EncryptedEntry, vals gensupport.Values) (Entry, error) { + var v Entry + var err error + if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { + return Entry{}, err + } + if v.Name, err = gensupport.Get[string](vals, "name"); err != nil { + return Entry{}, err + } + return v, nil + }, +}) + +// Encrypt seals each Entry, with one ZeroKMS request for each 500 sealed +// values, plus one the first time a keyset is used. The result has one element +// for each input, in the same order. +func Encrypt(ctx context.Context, cipher *encrypt.Cipher, directory []Entry) ([]EncryptedEntry, error) { + return codec.Encrypt(ctx, cipher, directory) +} + +// Decrypt opens each EncryptedEntry, with one ZeroKMS request for each 500 +// sealed values, plus one the first time a keyset is used. +func Decrypt(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedEntry) ([]Entry, error) { + return codec.Decrypt(ctx, d, encrypted) +} + +var Fields = struct { + Name NameField +}{ + Name: NameField{gensupport.NewField[string](declaration, "name")}, +} + +type NameField struct { + field gensupport.Field[string] +} + +func (f NameField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearch, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.TextSearch(out.EQL), err +} + +func (f NameField) Eq(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "eq", v) + return eql.TextSearchQuery(term), err +} + +func (f NameField) Lt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "lt", v) + return eql.TextSearchQuery(term), err +} + +func (f NameField) Lte(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "lte", v) + return eql.TextSearchQuery(term), err +} + +func (f NameField) Gt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "gt", v) + return eql.TextSearchQuery(term), err +} + +func (f NameField) Gte(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextSearchQuery, error) { + term, err := f.field.Term(ctx, c, "gte", v) + return eql.TextSearchQuery(term), err +} + +// Matches derives the matches query for v. Its answer can hold rows that do not +// match: decrypt the rows it fetches and keep those Verify reports true. +func (f NameField) Matches(ctx context.Context, c *encrypt.Cipher, v string) (encrypt.EQLMatchQuery[eql.TextSearchQuery, string], error) { + return gensupport.EQLMatches[eql.TextSearchQuery](ctx, f.field, c, v) +} diff --git a/languages/golang/stashgen/testdata/cases/directory/go.mod b/languages/golang/stashgen/testdata/cases/directory/go.mod new file mode 100644 index 000000000..c489c9950 --- /dev/null +++ b/languages/golang/stashgen/testdata/cases/directory/go.mod @@ -0,0 +1,7 @@ +module example.com/app + +go 1.26 + +require github.com/cipherstash/stack/languages/golang v0.0.0 + +replace github.com/cipherstash/stack/languages/golang => ../../stubsdk diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go index 17fac8ef2..3bc6a8f87 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go @@ -13,3 +13,9 @@ type TextMatch []byte // TextMatchQuery is the query value for TextMatch. type TextMatchQuery []byte + +// TextSearch is an EQL value with equality, order and match terms. +type TextSearch []byte + +// TextSearchQuery is the query value for TextSearch. +type TextSearchQuery []byte