Implement full-text index for entries, make name and description inde…

directory/README.md

@@ -252,6 +252,75 @@ Run `make generate` before invoking `go build` or `go test` directly.
 See [Host LMS and catalog profiles](host-profiles.md) for `lmsConfig.vendor`,
 `catalogConfig.profile`, precedence, parser overrides, diagnostics and migration.
 
+## Directory entry symbols
+
+When patching entry symbols, omit `id` to create a new symbol. A supplied `id`
+must identify an existing symbol owned by the entry being patched; unknown or
+foreign IDs return HTTP 400 and roll back the entire PATCH. Symbols cannot be
+moved between entries through the API.
+
+Application writes lock the owner entry before modifying its symbols. Direct
+SQL maintenance must follow the same order; the search trigger alone cannot
+prevent deadlocks caused by acquiring these locks in the opposite order.
+
+## Directory entry searches
+
+`cql.serverChoice` searches a combined document containing the entry's name,
+description, organization ID, email, phone number, and all its own symbols
+(both `authority:symbol` and bare symbol values). It uses PostgreSQL full-text
+search with the fixed `simple` configuration: case is normalized without English
+stemming or stop-word removal. NULL fields contribute no text. Parent and child
+entry data are not included.
+
+| CQL example | Behavior |
+| --- | --- |
+| `cql.serverChoice="central library"` | Both words, potentially in different fields |
+| `cql.serverChoice all "central library"` | Both words |
+| `cql.serverChoice any "central library"` | Either word |
+| `cql.serverChoice adj "central library"` | Adjacent words in order |
+| `cql.serverChoice="centr*"` | Word-prefix matching |
+
+This index is available on both `/entries` and `/entries/owned`; existing access
+restrictions, counts, pagination, and result ordering apply. Results are not
+ranked by relevance. Email addresses, phone numbers, and symbols follow
+PostgreSQL tokenization rather than arbitrary substring matching. Use dedicated
+field indexes such as `symbol` for exact identifier comparisons.
+
+A stored search vector and GIN index are maintained transactionally by database
+triggers when entry fields or symbols change, including imports. Migration 018
+backfills existing entries. SQLC expands wildcard selections into explicit column
+lists, so adding the search column preserves the result shape expected by
+pre-018 binaries. Existing binaries can continue serving against the upgraded
+schema after the migration completes.
+
+Migration 018 runs as one transaction: it adds the search column, creates the
+symbols ownership index before backfill, installs the search triggers, backfills
+all entries, and builds the GIN index. The initial schema change takes an
+exclusive lock on `entries` that is held until the transaction completes,
+blocking entry reads and writes throughout the backfill and index builds. The
+regular symbols index build also blocks symbol writes until commit. Migration
+duration depends on data volume and active transactions; requests may time out
+while waiting. Concurrent transactions that acquire locks in the opposite order
+can also deadlock with the migration.
+
+Plan a maintenance window for this upgrade. Stop all Directory instances and
+import jobs, pause other database writers, and allow active transactions to
+finish before starting one upgraded instance to apply migrations. Once migration
+018 succeeds and that instance is ready, restore the remaining instances and
+resume traffic and imports. A rolling update alone does not enforce this
+sequence. If migration fails, inspect the error and migration version/dirty state
+before retrying; a failed transaction rolls back its schema changes, but the
+migration runner may leave the version marked dirty and prevent startup.
+
+The `name` and `description` CQL indexes match case insensitively by default,
+using PostgreSQL `lower()` on both the stored value and the search term. For
+example, `name="central library"` matches `Central Library`, and
+`description="*LIBRARY*"` matches descriptions containing `library` in any case.
+This also applies to `==`, `exact`, and `<>` comparisons. Existing wildcard and
+escaping rules apply; no CQL relation modifier is required. Other indexes and
+result ordering retain their existing behavior. This does not add full-text
+search or accent normalization.
+
 ### Pickup locations
 
 Set `illConfig.isPickupLocation` to `true` to offer an institution or branch in the request form's pickup-location selector. This flag is independent of `lmsConfig.requesterPickupLocation`: manual workflows can offer locations without an LMS code. Migration 012 enables the flag for existing entries whose requester pickup code is non-null (including empty strings), preserving other ILL settings. Other entries remain unselected unless explicitly enabled. Imports can set the flag in `illConfig`.

directory/api.yaml

@@ -164,7 +164,12 @@ paths:
       parameters:
         - name: cql
           in: query
-          description: CQL query filter. The parentSymbol index matches entries by their immediate parent's symbols.
+          description: >-
+            CQL query filter. cql.serverChoice searches entry name, description,
+            organization ID, email, phone number, and owned symbols using the simple
+            full-text configuration. The default relation requires all words;
+            all, any, adj, and trailing word-prefix wildcards are supported.
+            The parentSymbol index matches entries by their immediate parent's symbols.
           required: false
           style: form
           schema:
@@ -251,7 +256,13 @@ paths:
       parameters:
         - name: cql
           in: query
-          description: CQL query filter applied within the current tenant's owned entries. The parentSymbol index matches entries by their immediate parent's symbols.
+          description: >-
+            CQL query filter applied within the current tenant's owned entries.
+            cql.serverChoice searches entry name, description, organization ID,
+            email, phone number, and owned symbols using the simple full-text
+            configuration. The default relation requires all words; all, any, adj,
+            and trailing word-prefix wildcards are supported.
+            The parentSymbol index matches entries by their immediate parent's symbols.
           required: false
           style: form
           schema:
@@ -3068,7 +3079,7 @@ components:
         id:
           type: string
           format: uuid
-          description: Unique id
+          description: Existing symbol ID owned by the entry being patched. Omit to create a new symbol. Unknown or foreign IDs return HTTP 400; symbols cannot be moved between entries.
         symbol:
           type: string
           minLength: 1

directory/api/entries.go

@@ -202,11 +202,14 @@ const defaultEntryLimit = 10
 func handleEntryCQL(cqlString string, noBaseArgs int) (pgcql.Query, error) {
 	def := pgcql.NewPgDefinition()
 
-	f := pgcql.NewFieldString().WithLikeOps()
+	def.AddField("cql.serverChoice", pgcql.NewFieldTsVector().
+		WithLanguage("simple").WithServerChoiceRel(cql.ALL).WithColumn("e.search"))
+
+	f := pgcql.NewFieldString().WithLikeOps().WithLower()
 	f.SetColumn("e.name")
 	def.AddField("name", f)
 
-	f = pgcql.NewFieldString().WithLikeOps()
+	f = pgcql.NewFieldString().WithLikeOps().WithLower()
 	f.SetColumn("e.description")
 	def.AddField("description", f)
 
@@ -785,7 +788,7 @@ func (a ApiImpl) AddEntry(ctx context.Context, request AddEntryRequestObject) (A
 
 	if request.Body.Symbols != nil {
 		for _, symbol := range *request.Body.Symbols {
-			_, err = qtx.UpsertSymbol(ctx, db.UpsertSymbolParams{
+			_, err = qtx.CreateSymbol(ctx, db.CreateSymbolParams{
 				Owner:     insertedEntry.ID,
 				Symbol:    strings.ToUpper(symbol.Symbol),
 				Authority: strings.ToUpper(symbol.Authority),
@@ -1082,14 +1085,21 @@ func (a ApiImpl) UpdateEntry(ctx context.Context, request UpdateEntryRequestObje
 			}
 		}
 
-		// Update/create symbols
+		// The entry is already locked. Restrict updates to its own symbols so
+		// the search trigger never needs to lock another entry after a symbol.
 		for _, symbol := range reqsyms {
-			_, err = qtx.UpsertSymbol(ctx, db.UpsertSymbolParams{
-				ID:        symbol.Id,
-				Owner:     orig.ID,
-				Symbol:    strings.ToUpper(symbol.Symbol),
-				Authority: strings.ToUpper(symbol.Authority),
-			})
+			if symbol.Id == nil {
+				_, err = qtx.CreateSymbol(ctx, db.CreateSymbolParams{
+					Owner: orig.ID, Symbol: strings.ToUpper(symbol.Symbol), Authority: strings.ToUpper(symbol.Authority),
+				})
+			} else {
+				_, err = qtx.UpdateOwnedSymbol(ctx, db.UpdateOwnedSymbolParams{
+					ID: *symbol.Id, Owner: orig.ID, Symbol: strings.ToUpper(symbol.Symbol), Authority: strings.ToUpper(symbol.Authority),
+				})
+				if errors.Is(err, pgx.ErrNoRows) {
+					return UpdateEntry400TextResponse("Symbol ID does not belong to this entry"), nil
+				}
+			}
 			if err != nil {
 				var pge *pgconn.PgError
 				if errors.As(err, &pge) {

directory/api/entries_test.go

@@ -104,6 +104,112 @@ func TestHandleEntryCQL(t *testing.T) {
 	}
 }
 
+func TestHandleEntryCQLCaseInsensitive(t *testing.T) {
+	for _, field := range []string{"name", "description"} {
+		t.Run(field, func(t *testing.T) {
+			for _, tc := range []struct {
+				clause string
+				op     string
+				term   string
+			}{
+				{`="Central Library"`, "=", "Central Library"},
+				{`=="Central Library"`, "=", "Central Library"},
+				{` exact "Central Library"`, "=", "Central Library"},
+				{`<>"Central Library"`, "<>", "Central Library"},
+				{`="*Central?Library*"`, "LIKE", "%Central_Library%"},
+				{`<>"*Central*"`, "NOT LIKE", "%Central%"},
+				{`="Central\*Library"`, "=", "Central*Library"},
+				{`="*A\"B\\C*"`, "LIKE", "%A\"B\\\\C%"},
+				{`="*Central_100%*"`, "LIKE", "%Central\\_100\\%%"},
+			} {
+				t.Run(tc.clause, func(t *testing.T) {
+					res, err := handleEntryCQL(field+tc.clause, 0)
+					if err != nil {
+						t.Fatalf("translate CQL: %v", err)
+					}
+					want := "lower(e." + field + ") " + tc.op + " lower($1)"
+					if got := res.GetWhereClause(); got != want {
+						t.Errorf("WHERE clause = %q, want %q", got, want)
+					}
+					if got := res.GetQueryArguments(); !reflect.DeepEqual(got, []any{tc.term}) {
+						t.Errorf("arguments = %#v, want %#v", got, []any{tc.term})
+					}
+				})
+			}
+		})
+	}
+
+	t.Run("combined fields with existing arguments", func(t *testing.T) {
+		res, err := handleEntryCQL(`name="*Central*" OR description="*Library*"`, 1)
+		if err != nil {
+			t.Fatalf("translate combined CQL: %v", err)
+		}
+		want := "lower(e.name) LIKE lower($2) OR lower(e.description) LIKE lower($3)"
+		if got := res.GetWhereClause(); got != want {
+			t.Errorf("WHERE clause = %q, want %q", got, want)
+		}
+		if got := res.GetQueryArguments(); !reflect.DeepEqual(got, []any{"%Central%", "%Library%"}) {
+			t.Errorf("unexpected arguments: %#v", got)
+		}
+	})
+
+	t.Run("type remains case sensitive", func(t *testing.T) {
+		res, err := handleEntryCQL(`type="*Institution*"`, 0)
+		if err != nil {
+			t.Fatalf("translate type CQL: %v", err)
+		}
+		if got := res.GetWhereClause(); got != "e.type LIKE $1" {
+			t.Errorf("unexpected type predicate: %s", got)
+		}
+	})
+}
+
+func TestHandleEntryCQLServerChoice(t *testing.T) {
+	for _, tc := range []struct {
+		query string
+		term  string
+	}{
+		{`cql.serverChoice="Central Library"`, "'Central'&'Library'"},
+		{`"Central Library"`, "'Central'&'Library'"},
+		{`cql.serverChoice all "Central Library"`, "'Central'&'Library'"},
+		{`cql.serverChoice any "Central Library"`, "'Central'|'Library'"},
+		{`cql.serverChoice adj "Central Library"`, "'Central'<->'Library'"},
+		{`cql.serverChoice="Centr*"`, "'Centr':*"},
+		{`cql.serverChoice="O'Brien"`, "'O''Brien'"},
+		{`cql.serverChoice="A\"B\\C"`, "'A\"B\\C'"},
+	} {
+		t.Run(tc.query, func(t *testing.T) {
+			res, err := handleEntryCQL(tc.query, 0)
+			if err != nil {
+				t.Fatalf("translate serverChoice CQL: %v", err)
+			}
+			if got := res.GetWhereClause(); got != "e.search @@ to_tsquery('simple', $1)" {
+				t.Errorf("unexpected search predicate: %s", got)
+			}
+			if got := res.GetQueryArguments(); !reflect.DeepEqual(got, []any{tc.term}) {
+				t.Errorf("arguments = %#v, want %#v", got, []any{tc.term})
+			}
+		})
+	}
+	t.Run("composes with existing filters and offsets", func(t *testing.T) {
+		res, err := handleEntryCQL(`cql.serverChoice="Central" AND isPickupLocation=true`, 1)
+		if err != nil {
+			t.Fatalf("translate combined CQL: %v", err)
+		}
+		if got := res.GetWhereClause(); !strings.HasPrefix(got, "e.search @@ to_tsquery('simple', $2) AND ") || !strings.HasSuffix(got, " = $3") {
+			t.Errorf("unexpected combined predicate: %s", got)
+		}
+		if got := res.GetQueryArguments(); !reflect.DeepEqual(got, []any{"'Central'", true}) {
+			t.Errorf("unexpected combined arguments: %#v", got)
+		}
+	})
+	for _, query := range []string{`cql.serverChoice="*central*"`, `cql.serverChoice="centr?"`} {
+		if _, err := handleEntryCQL(query, 0); err == nil {
+			t.Errorf("expected unsupported wildcard error for %s", query)
+		}
+	}
+}
+
 func TestBuildOwnedEntryListQueryScopesCQL(t *testing.T) {
 	cqlQuery := `name="Owned" OR tenant="OTHER"`
 	limit := Limit(5)

directory/import/db/entry.go

@@ -340,7 +340,7 @@ func replaceEntryChildren(ctx context.Context, queries *db.Queries, entryID uuid
 		return err
 	}
 	for _, symbol := range data.Symbols {
-		if _, err := queries.UpsertSymbol(ctx, db.UpsertSymbolParams{Owner: entryID, Authority: symbol.Authority, Symbol: symbol.Symbol}); err != nil {
+		if _, err := queries.CreateSymbol(ctx, db.CreateSymbolParams{Owner: entryID, Authority: symbol.Authority, Symbol: symbol.Symbol}); err != nil {
 			return err
 		}
 	}

directory/import/db/repo_test.go

@@ -526,7 +526,10 @@ func TestImportEntryUsesParentUUIDWhenParentSymbolChangesBeforeRowLock(t *testin
 	}()
 	waitForLockUnavailable(t, ctx, lockUnavailable)
 
-	_, err = testPool.Exec(ctx, `UPDATE symbols SET owner=$1 WHERE authority='ISIL' AND symbol='PARENT'`, replacementParentID)
+	// Symbol changes now lock their owners to refresh the search document. Move
+	// the symbol in the transaction holding the original parent's lock, before
+	// allowing the importer to acquire it; the parent UUID must remain stable.
+	_, err = blocker.Exec(ctx, `UPDATE symbols SET owner=$1 WHERE authority='ISIL' AND symbol='PARENT'`, replacementParentID)
 	require.NoError(t, err)
 	require.NoError(t, blocker.Commit(ctx))
 	require.NoError(t, <-importDone)

directory/migrations/018_entry_search.down.sql

@@ -0,0 +1,10 @@
+DROP TRIGGER directory_entry_search_symbol_refresh ON symbols;
+DROP TRIGGER directory_entry_search_symbol_lock ON symbols;
+DROP TRIGGER directory_entry_search_entry ON entries;
+DROP FUNCTION directory_entry_search_on_symbol();
+DROP FUNCTION directory_entry_search_lock_symbol_owners();
+DROP FUNCTION directory_entry_search_on_entry();
+DROP FUNCTION directory_entry_search_document(uuid, text, text, text, text, text);
+DROP INDEX entries_search_idx;
+DROP INDEX symbols_owner_authority_symbol_idx;
+ALTER TABLE entries DROP COLUMN search;