From 30079d5f5f002aad16a426f9561b558c9b326d88 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Fri, 16 Oct 2026 20:04:08 +0000 Subject: [PATCH 1/3] Add pluggable WAL storage providers Introduce the WalProvider SPI and properties to configure the available set of WAL providers and the current provider. This commit converts the current DFS implementation to a DfsWalProvider and the upgrade code converts all existing wal entries in ZooKeeper and the root and metadata tables to use the 'dfs' provider id. Post-upgrade and on instance restart the `instance.wal.provider' property can be set to the class name of a WALProvider SPI implementation. The 'instance.wal.providers' property must contain the class name and configuration properties for the current and all in-use providers. An InMemoryWalProvider class in the test module provides an example implementation and is used in various ITs. Assisted-By: OpenAI GPT-6 Luna --- WAL-SPI.md | 188 ++++++++++++ assemble/conf/accumulo.properties | 6 + .../apache/accumulo/core/conf/Property.java | 12 + .../accumulo/core/spi/wal/WalProvider.java | 191 ++++++++++++ .../core/tabletserver/log/LogEntry.java | 31 +- .../accumulo/core/conf/PropertyTest.java | 16 + .../metadata/schema/TabletMetadataTest.java | 18 +- .../core/tabletserver/log/LogEntryTest.java | 48 ++- .../MiniAccumuloClusterImpl.java | 11 + server/base/pom.xml | 4 + .../apache/accumulo/server/ServerContext.java | 7 + .../apache/accumulo/server/fs/VolumeUtil.java | 10 +- .../accumulo/server/log/DfsWalProvider.java | 182 ++++++++++++ .../server/log/WalProviderRegistry.java | 178 +++++++++++ .../accumulo/server/log/WalStateManager.java | 64 +++- .../state/AbstractTabletStateStore.java | 14 +- .../state/LoggingTabletStateStore.java | 6 +- .../manager/state/TabletStateStore.java | 10 +- .../manager/state/ZooTabletStateStore.java | 6 +- .../checkCommand/SystemConfigCheckRunner.java | 2 +- .../accumulo/server/fs/VolumeUtilTest.java | 7 +- .../server/log/WalProviderRegistryTest.java | 62 ++++ .../server/log/WalStateManagerTest.java | 91 ++++++ .../manager/state/TabletManagementTest.java | 7 +- .../src/test/resources/accumulo.properties | 2 + .../gc/GarbageCollectWriteAheadLogs.java | 20 +- .../accumulo/gc/SimpleGarbageCollector.java | 1 + .../gc/GarbageCollectWriteAheadLogsTest.java | 12 +- .../gc/src/test/resources/accumulo.properties | 2 + .../accumulo/manager/BalanceManager.java | 2 +- .../org/apache/accumulo/manager/Manager.java | 42 +++ .../accumulo/manager/TabletGroupWatcher.java | 12 +- .../manager/recovery/RecoveryManager.java | 31 +- .../manager/upgrade/Upgrader11to12.java | 102 ++++++- .../tableOps/merge/MergeTabletsTest.java | 4 +- .../tableOps/split/UpdateTabletsTest.java | 4 +- .../manager/upgrade/Upgrader11to12Test.java | 77 +++++ .../src/test/resources/accumulo.properties | 2 + .../apache/accumulo/tserver/ScanServer.java | 1 + .../apache/accumulo/tserver/TabletServer.java | 9 +- .../accumulo/tserver/log/DfsLogger.java | 151 ++++------ .../accumulo/tserver/log/LogSorter.java | 39 +-- .../tserver/log/TabletServerLogger.java | 3 +- .../accumulo/tserver/logger/LogReader.java | 4 +- .../accumulo/tserver/WalRemovalOrderTest.java | 4 +- .../tserver/log/RecoveryLogsIteratorTest.java | 4 +- .../tserver/log/SortedLogRecoveryTest.java | 4 +- .../src/test/resources/accumulo.properties | 2 + shell/pom.xml | 5 + .../commands/ListTabletsCommandTest.java | 7 +- .../MissingWalHeaderCompletesRecoveryIT.java | 7 +- .../test/fate/ManagerRepoIT_SimpleSuite.java | 4 +- .../functional/AmpleConditionalWriterIT.java | 13 +- .../functional/InMemoryWalProviderIT.java | 144 +++++++++ .../TabletManagementIteratorIT.java | 3 +- .../test/functional/WalProviderSwitchIT.java | 172 +++++++++++ .../test/wal/InMemoryWalProvider.java | 277 ++++++++++++++++++ .../accumulo/test/wal/InMemoryWalStore.java | 223 ++++++++++++++ test/src/main/resources/accumulo.properties | 2 + .../test/wal/InMemoryWalProviderTest.java | 118 ++++++++ 60 files changed, 2456 insertions(+), 224 deletions(-) create mode 100644 WAL-SPI.md create mode 100644 core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java create mode 100644 server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java create mode 100644 server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java create mode 100644 server/base/src/test/java/org/apache/accumulo/server/log/WalProviderRegistryTest.java create mode 100644 server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java create mode 100644 test/src/main/java/org/apache/accumulo/test/functional/InMemoryWalProviderIT.java create mode 100644 test/src/main/java/org/apache/accumulo/test/functional/WalProviderSwitchIT.java create mode 100644 test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java create mode 100644 test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalStore.java create mode 100644 test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java diff --git a/WAL-SPI.md b/WAL-SPI.md new file mode 100644 index 00000000000..1f872a67f69 --- /dev/null +++ b/WAL-SPI.md @@ -0,0 +1,188 @@ + +# Write-ahead log storage SPI (current implementation) + +Accumulo's WAL storage SPI lets a TabletServer create raw write-ahead logs using a configured +`WalProvider`. The provider owns storage of the raw byte stream; Accumulo owns the WAL format, +encryption, tablet bookkeeping, sorting, and replay. The implementation is split between the +[SPI interface][spi], the [provider registry][registry], and the built-in +[filesystem provider][dfs-provider]. + +## Configuration and provider identity + +Two instance-wide properties configure providers in `accumulo.properties`: + +| Property | Purpose | +| ------------------------ | ------------------------------------------------------------- | +| `instance.wal.providers` | JSON map of provider classes to their option maps. | +| `instance.wal.provider` | Class for **new** WALs; must be a key in the map above. | + +Both properties are required. `instance.wal.providers` defaults to an empty map and +`instance.wal.provider` defaults to an empty string. For the built-in provider, configure +`org.apache.accumulo.server.log.DfsWalProvider` with an empty option map: + +```properties +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider +``` + +For a provider with options, put string-valued options in its map, for example +`{"com.example.MyWalProvider":{"endpoint":"https://wal.example"}}`. The +[distribution configuration][dist-config] shows these settings; their definitions are in +[Property.java][property]. + +Each implementation needs a public no-argument constructor and a stable `getId()` matching +`[A-Za-z0-9]+`. The ID `dfs` is reserved for `DfsWalProvider` and legacy filesystem WALs. The +[registry][registry] loads each configured class, passes its options to `init(InitParameters)` once per +server process, rejects duplicate provider IDs, and selects the current class by class name. The +default `init(InitParameters)` implementation calls `init(ServiceEnvironment)`, preserving providers +that implement the older initialization method. A +[memoized registry][context] is accessed by the Manager, TabletServers, ScanServers, and GC. The +[Manager][manager] also checks WAL IDs in tablet metadata and ZooKeeper markers against the +configured IDs at startup. + +Changing the current provider affects newly created logs only. Keep old providers in +`instance.wal.providers` while their WALs are still referenced, so recovery and cleanup can use +them. Provider IDs and configuration must remain consistent across server processes. + +## Upgrade of existing WAL references + +The [11-to-12 upgrader][upgrader] assigns the built-in provider ID `dfs` to pre-SPI filesystem +WALs without moving or rewriting the WAL files: + +- Legacy tablet log qualifiers `-/` become `dfs/`: the upgrader adds the new + qualifier and deletes the old one. This covers the root tablet metadata stored in ZooKeeper, + metadata-tablet references, and user-tablet references. Already converted qualifiers are left + alone. +- Legacy ZooKeeper WAL marker data `,` becomes `,dfs,`. + Markers already containing a provider ID are retained. The upgrader checks marker format and + verifies that an updated marker was persisted. +- The upgrader checks for remaining legacy `-/` log qualifiers in the metadata tables; it also + verifies the root tablet's WAL references. Malformed markers or leftover legacy qualifiers cause + the upgrade to fail rather than silently treating them as another provider's logs. + +Configure `DfsWalProvider` in `instance.wal.providers` during the upgrade and retain it while old +WAL references or markers exist, even if a different class is selected in `instance.wal.provider` +for new writes. Otherwise the Manager's provider-reference check or subsequent recovery and GC +cannot handle those existing `dfs` references. The upgrade changes **reference metadata**, not the +raw WAL format or the sorted-log recovery process. + +## Provider contract + +[`WalProvider`][spi] exposes: + +| Method | Use | +| ------------------------------- | ----------------------------------------------------- | +| `create(serverAddress, UUID)` | Exclusively create a WAL and return its `Writer`. | +| `open(reference)` | Read a WAL from byte zero for sorting/recovery. | +| `exists(reference)` | Check a WAL, including while it may still be written. | +| `delete(reference)` | Remove an unreferenced WAL; should be idempotent. | +| `getId()` / `init(environment)` | Supply the persisted ID and initialize the provider. | + +The nested `Writer` supplies `getReference()`, `getOutputStream()`, `sync(Durability)`, +`getLength()`, and `close()`. It can optionally report `getCurrentReplication()` and +`getPipeline()` for diagnostics (the defaults return `-1` and an empty array). Accumulo writes the +WAL header and records to the output stream: a provider must retain the exact bytes in order and +return them through `open()`. Providers must be thread-safe for operations on different logs; a +writer must also coordinate concurrent appends and sync requests. `FLUSH` must survive a +TabletServer process failure, `SYNC` must provide the synchronization guarantee expected of synced +WAL writes, and `LOG` does not invoke `Writer.sync()`. Closing a writer must be idempotent. + +The reference returned by `Writer.getReference()` is persisted and must stay usable by that +provider's `open()` and `delete()`. It must be a fully qualified Hadoop path in the standard WAL +layout, with a final `/` suffix (the address's `:` becomes `+`). The current +[recovery path conversion][rpath] also expects a `wal` directory immediately before the +server component; it maps `/wal//` to `/recovery/`. A custom +provider can use a custom Hadoop filesystem URI, but this path structure and access to the +corresponding recovery destination are current integration constraints. + +## Writing and tracking a WAL + +1. [`TabletServerLogger`][tslog] obtains `getCurrent()` from the registry when creating a + log. Despite its name, [`DfsLogger`][dfs-logger] uses the selected provider's `create()` and + `Writer`, so the WAL need not be stored by `DfsWalProvider`. +2. `DfsLogger` validates the returned reference as a [`LogEntry`][log-entry], writes the versioned + header and crypto parameters, wraps the provider output stream for encryption, writes an `OPEN` + entry, and syncs it before using the log. Subsequent tablet definitions, mutations, and + compaction events use the existing WAL serialization. A background sync task batches durability + requests and calls `Writer.sync(FLUSH)` or `Writer.sync(SYNC)`; it uses the optional + replication/pipeline methods for diagnostics. +3. `LogEntry` associates the provider ID with the path. Tablet metadata stores it as a log-column + qualifier of the form `/`. [`WalStateManager`][wal-state] records the WAL + state (`OPEN`, `CLOSED`, or `UNREFERENCED`), provider ID, and path in ZooKeeper WAL markers. This + lets recovery and GC identify which provider created a given log, independently of the provider + currently selected for new writes. + +[`DfsWalProvider`][dfs-provider] is the built-in implementation. It chooses an Accumulo volume and +creates a filesystem-backed WAL under `wal//` using `VolumeManager`; its writer +uses `hflush()` for `FLUSH` and `hsync()` for `SYNC`. It opens and checks files through +`VolumeManager`, and uses the filesystem trash or recursive deletion for cleanup. Where the +underlying stream is an HDFS `DFSOutputStream`, it can report replication and pipeline information. + +## Recovery and garbage collection + +The [Manager's `RecoveryManager`][rm] obtains the provider ID from the `LogEntry`, +derives the sibling `recovery/` directory using `RecoveryPath`, and queues a sort. Filesystem +WALs with ID `dfs` use the configured Hadoop log closer before sorting; other providers do not use +that closer. The queued work carries the provider ID, source reference, and recovery destination. + +[`LogSorter`][sorter] opens the raw source through `VolumeManager` for `dfs` or through the +registered provider's `open(reference)` for another ID. It decrypts and decodes Accumulo's WAL +records, sorts them into RFiles in the recovery directory via `VolumeManager`, and writes a finished +marker. The sorted recovery files are **not** read through `WalProvider.open()`: +[`ResolvedSortedLog`][resolved-log] validates that directory and marker, +[`RecoveryLogsIterator`][rli] opens the RFiles using the filesystem selected by +`VolumeManager`, and [`SortedLogRecovery`][slr] replays the tablet's applicable +mutations. + +Thus the SPI permits alternative storage for the **raw WAL**, but it does not replace the existing +filesystem-based sorted-log recovery pipeline. HDFS itself is not a requirement of +`SortedLogRecovery`: its [unit test][slr-test] uses a local `file://` filesystem. The recovery +destination must nevertheless be accessible through `VolumeManager`, and the reference must satisfy +the path layout above. + +After WALs are no longer referenced, [`GarbageCollectWriteAheadLogs`][wal-gc] deletes raw `dfs` +logs through `VolumeManager` and other raw logs through their registered provider's +`delete(reference)`. It removes sorted recovery directories through `VolumeManager` separately. +The [system configuration check][c] uses the provider's `exists(reference)` when +checking WAL markers. + +[spi]: core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java +[registry]: server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java +[dfs-provider]: server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java +[dist-config]: assemble/conf/accumulo.properties +[property]: core/src/main/java/org/apache/accumulo/core/conf/Property.java +[context]: server/base/src/main/java/org/apache/accumulo/server/ServerContext.java +[manager]: server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +[upgrader]: server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +[rpath]: server/base/src/main/java/org/apache/accumulo/server/manager/recovery/RecoveryPath.java +[tslog]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java +[dfs-logger]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java +[log-entry]: core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java +[wal-state]: server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java +[rm]: server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java +[sorter]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java +[resolved-log]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/ResolvedSortedLog.java +[rli]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogsIterator.java +[slr]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/SortedLogRecovery.java +[slr-test]: server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java +[wal-gc]: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java +[c]: server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java diff --git a/assemble/conf/accumulo.properties b/assemble/conf/accumulo.properties index 6e6de34cc63..5a9532f1d02 100644 --- a/assemble/conf/accumulo.properties +++ b/assemble/conf/accumulo.properties @@ -23,6 +23,12 @@ ## Sets location in HDFS where Accumulo will store data instance.volumes=hdfs://localhost:8020/accumulo +## Configure the WAL provider implementation consistently on every server +## For the built-in filesystem-backed implementation, configure: +## org.apache.accumulo.server.log.DfsWalProvider +# instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +# instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider + ## Sets location of Zookeepers instance.zookeeper.host=localhost:2181 diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index cec1a3f9649..aafef25bef4 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -260,6 +260,18 @@ temporary files (for example, when creating a pre-split table). \ + "encryption, replace this classname with an implementation of the" + "org.apache.accumulo.core.spi.crypto.CryptoFactory interface.", "2.1.0"), + INSTANCE_WAL_PROVIDERS("instance.wal.providers", "{}", PropertyType.JSON, + "Required JSON object mapping fully qualified class names implementing " + + "org.apache.accumulo.core.spi.wal.WalProvider to string-valued option maps. Use an " + + "empty map when a provider has no options. All configured providers are validated at startup, must be " + + "available to Managers, TabletServers, ScanServers, and GC, and must include providers " + + "referenced by existing WALs. This must be consistent across the instance.", + "4.0.0"), + INSTANCE_WAL_PROVIDER("instance.wal.provider", "", PropertyType.CLASSNAME, + "Required. The class implementing org.apache.accumulo.core.spi.wal.WalProvider used to create new " + + "WALs. It must also be listed in instance.wal.providers and must be consistent " + + "across the instance.", + "4.0.0"), // general properties GENERAL_PREFIX("general.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of accumulo overall, but" diff --git a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java new file mode 100644 index 00000000000..842c3dca1b2 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.spi.wal; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Map; +import java.util.UUID; + +import org.apache.accumulo.core.client.Durability; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; + +/** + * Storage provider for TabletServer write-ahead logs. Implementations are initialized once per + * server process and must be thread-safe because Accumulo may create, read, and delete different + * logs concurrently. The provider stores an opaque byte stream: Accumulo writes the log header and + * records to the returned output stream, and a reader must return those exact bytes in their + * original write order. Configured implementations must have a public no-argument constructor. + * + *

+ * A provider ID is persisted with every log reference. It must match {@code [A-Za-z0-9]+} and + * remain stable for as long as any log created by the provider can be referenced. Every provider + * that may be needed by existing tablet metadata or WAL markers must be configured as available. + * + *

+ * To keep Accumulo's existing recovery pipeline, log references must be fully qualified Hadoop + * paths in the standard WAL layout, ending in {@code /}. Providers may use a + * custom Hadoop filesystem URI, but must preserve this path layout. + * + * @since 4.0.0 + */ +public interface WalProvider { + + /** Regular expression for valid provider IDs. */ + String PROVIDER_ID_REGEX = "[A-Za-z0-9]+"; + + /** + * Returns the stable identifier persisted with logs created by this provider. It must be unique + * among the configured providers and match {@link #PROVIDER_ID_REGEX}. + * + * @return stable provider ID + */ + String getId(); + + /** + * Initializes the provider once after construction. Implementations may retain the environment + * for accessing system configuration and must not assume that properties change during this + * process lifetime. Throwing an exception prevents the server process from starting. + * + * @param environment system configuration and Accumulo service information + */ + default void init(ServiceEnvironment environment) {} + + /** + * Initializes this provider with its configured options. The default implementation calls + * {@link #init(ServiceEnvironment)} for compatibility with providers that only implement that + * method. + * + * @param params the service environment and options configured for this provider + * @since 4.0.0 + */ + default void init(InitParameters params) { + init(params.getServiceEnvironment()); + } + + /** Parameters passed to {@link #init(InitParameters)}. */ + interface InitParameters { + + /** @return system configuration and Accumulo service information */ + ServiceEnvironment getServiceEnvironment(); + + /** @return the provider-specific options configured in {@code instance.wal.providers} */ + Map getOptions(); + } + + /** + * Creates a new exclusive WAL for the given TabletServer and UUID. The returned reference must be + * unique, fully qualified, and end in the supplied server address (with {@code :} replaced by + * {@code +}) followed by the supplied UUID. The writer must preserve every byte written to its + * output stream in order. + * + * @param serverAddress advertised TabletServer address in {@code host:port} form + * @param id globally unique identifier for the WAL + * @return a new writer whose reference can later be passed to {@link #open(String)} and + * {@link #delete(String)} + */ + Writer create(String serverAddress, UUID id) throws IOException; + + /** + * Opens a completed or abandoned WAL for recovery. The returned stream must start at byte zero + * and reproduce the bytes written by the corresponding writer exactly and in the same order. + * Accumulo only opens logs after their TabletServer has closed or lost ownership of them. + * + * @param reference reference returned by {@link Writer#getReference()} + */ + InputStream open(String reference) throws IOException; + + /** + * Checks whether a WAL reference currently exists without opening it. This may be called while a + * WAL is still being written. + * + * @param reference reference returned by {@link Writer#getReference()} + * @return true if the WAL exists + */ + boolean exists(String reference) throws IOException; + + /** + * Deletes a WAL that Accumulo has determined is no longer referenced by any tablet. Deletion + * should be idempotent because a server may retry cleanup after an interruption. + * + * @param reference reference returned by {@link Writer#getReference()} + */ + void delete(String reference) throws IOException; + + /** + * A newly created WAL and its append-only output stream. Accumulo may call + * {@link #sync(Durability)} while other threads append through the output stream; implementations + * must ensure a sync covers all bytes written before it is requested. + */ + interface Writer extends AutoCloseable { + + /** + * Returns the stable, fully qualified reference for this WAL. It is persisted in tablet + * metadata and WAL markers and must remain usable by {@link WalProvider#open(String)} and + * {@link WalProvider#delete(String)}. + */ + String getReference(); + + /** + * Returns the stream Accumulo writes the WAL header and records to. Accumulo owns and closes + * this stream through {@link #close()}; implementations may buffer internally but must preserve + * every byte and its order, and make buffered bytes readable after the corresponding sync or + * close completes. + */ + OutputStream getOutputStream(); + + /** + * Makes all preceding bytes durable according to Accumulo's requested durability. {@code FLUSH} + * must survive a TabletServer process failure and {@code SYNC} must satisfy the filesystem + * synchronization guarantee used by Accumulo for synced WAL writes. This method is not called + * for {@code LOG} durability. + */ + void sync(Durability durability) throws IOException; + + /** + * Returns the number of bytes written so far, including the WAL header, for TabletServer WAL + * metrics and diagnostics. This method may run concurrently with writes. + */ + long getLength() throws IOException; + + /** + * Returns the current replication level when the provider can report one, or {@code -1} when + * that information is unavailable. + */ + default int getCurrentReplication() throws IOException { + return -1; + } + + /** + * Returns a diagnostic description of the current storage pipeline, or an empty array when the + * provider has no pipeline information. + */ + default String[] getPipeline() { + return new String[0]; + } + + /** + * Closes the WAL and releases all associated resources. Closing must be idempotent because + * Accumulo may close a writer after an initialization error. + */ + @Override + void close() throws IOException; + } +} diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java index 57820065b07..b9706f6afb5 100644 --- a/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java +++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java @@ -26,6 +26,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; +import org.apache.accumulo.core.spi.wal.WalProvider; import org.apache.hadoop.io.Text; import org.checkerframework.checker.nullness.qual.NonNull; @@ -36,12 +37,15 @@ public final class LogEntry { private final String path; + private final String providerId; private final HostAndPort tserver; private final UUID uniqueId; private final Text columnQualifier; - private LogEntry(String path, HostAndPort tserver, UUID uniqueId, Text columnQualifier) { + private LogEntry(String path, String providerId, HostAndPort tserver, UUID uniqueId, + Text columnQualifier) { this.path = path; + this.providerId = providerId; this.tserver = tserver; this.uniqueId = uniqueId; this.columnQualifier = columnQualifier; @@ -54,15 +58,19 @@ private LogEntry(String path, HostAndPort tserver, UUID uniqueId, Text columnQua * For example, file:///some/dir/path/localhost+1234/927ba659-d109-4bce-b0a5-bcbbcb9942a2 is a * valid path. * + * @param providerId provider that owns this WAL * @param path path to validate * @return an object representation of this log entry * @throws IllegalArgumentException if the path is invalid */ - public static LogEntry fromPath(String path) { - return validatedLogEntry(path, null); + public static LogEntry fromPath(String providerId, String path) { + return validatedLogEntry(path, providerId, null); } - private static LogEntry validatedLogEntry(String path, Text columnQualifier) { + private static LogEntry validatedLogEntry(String path, String providerId, Text columnQualifier) { + Preconditions.checkArgument( + providerId != null && providerId.matches(WalProvider.PROVIDER_ID_REGEX), + "Invalid WAL provider ID: %s", providerId); String[] parts = path.split("/"); if (parts.length < 2) { @@ -96,7 +104,7 @@ private static LogEntry validatedLogEntry(String path, Text columnQualifier) { throw new IllegalArgumentException(badUuidMsg); } - return new LogEntry(path, tserver, uuid, columnQualifier); + return new LogEntry(path, providerId, tserver, uuid, columnQualifier); } /** @@ -115,7 +123,7 @@ public static LogEntry fromMetaWalEntry(Entry entry) { Text qualifier = entry.getKey().getColumnQualifier(); String[] parts = qualifier.toString().split("/", 2); Preconditions.checkArgument(parts.length == 2, "Malformed write-ahead log %s", qualifier); - return validatedLogEntry(parts[1], qualifier); + return validatedLogEntry(parts[1], parts[0], qualifier); } @NonNull @@ -129,6 +137,10 @@ public String getPath() { return path; } + public String getProviderId() { + return providerId; + } + @NonNull public UUID getUniqueID() { return uniqueId; @@ -145,14 +157,15 @@ public boolean equals(Object other) { return true; } if (other instanceof LogEntry) { - return path.equals(((LogEntry) other).path); + return path.equals(((LogEntry) other).path) + && providerId.equals(((LogEntry) other).providerId); } return false; } @Override public int hashCode() { - return Objects.hash(path); + return Objects.hash(providerId, path); } /** @@ -163,7 +176,7 @@ public Text getColumnQualifier() { } private Text newCQ() { - return new Text("-/" + getPath()); + return new Text(providerId + "/" + getPath()); } /** diff --git a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java index be100685e85..df6add2f924 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java @@ -106,6 +106,22 @@ public void testProperties() { } } + @Test + public void testInstanceWalProviderProperties() { + assertEquals("instance.wal.providers", Property.INSTANCE_WAL_PROVIDERS.getKey()); + assertEquals("instance.wal.provider", Property.INSTANCE_WAL_PROVIDER.getKey()); + assertEquals("{}", Property.INSTANCE_WAL_PROVIDERS.getDefaultValue()); + assertEquals("", Property.INSTANCE_WAL_PROVIDER.getDefaultValue()); + assertEquals(PropertyType.JSON, Property.INSTANCE_WAL_PROVIDERS.getType()); + assertEquals(PropertyType.CLASSNAME, Property.INSTANCE_WAL_PROVIDER.getType()); + assertTrue(Property.isValidProperty(Property.INSTANCE_WAL_PROVIDERS.getKey(), + Property.INSTANCE_WAL_PROVIDERS.getDefaultValue())); + assertFalse(Property.isValidZooPropertyKey(Property.INSTANCE_WAL_PROVIDERS.getKey())); + assertFalse(Property.isFixedZooPropertyKey(Property.INSTANCE_WAL_PROVIDERS)); + assertFalse(Property.isValidZooPropertyKey(Property.INSTANCE_WAL_PROVIDER.getKey())); + assertFalse(Property.isFixedZooPropertyKey(Property.INSTANCE_WAL_PROVIDER)); + } + @Test public void testPorts() { HashSet usedPorts = new HashSet<>(); diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java index b56315d8d70..5aabea33b30 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java @@ -171,9 +171,9 @@ public void testAllColumns() { mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); - LogEntry le1 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); le1.addToMutation(mutation); - LogEntry le2 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); le2.addToMutation(mutation); StoredTabletFile sf1 = StoredTabletFile.of(new Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf")); @@ -515,8 +515,8 @@ public void testTmBuilderImmutable() { assertThrows(UnsupportedOperationException.class, () -> tm.getFilesMap().put(stf, new DataFileValue(0, 0, 0))); assertTrue(tm.getLogs().isEmpty()); - assertThrows(UnsupportedOperationException.class, - () -> tm.getLogs().add(LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()))); + assertThrows(UnsupportedOperationException.class, () -> tm.getLogs() + .add(LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()))); assertTrue(tm.getScans().isEmpty()); assertThrows(UnsupportedOperationException.class, () -> tm.getScans().add(stf)); assertTrue(tm.getLoaded().isEmpty()); @@ -534,7 +534,7 @@ public void testTmBuilderImmutable() { b.table(TableId.of("4")); b.extCompaction(ecid, ecMeta); b.file(stf, new DataFileValue(0, 0, 0)); - b.log(LogEntry.fromPath("localhost+8020/" + UUID.randomUUID())); + b.log(LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID())); b.scan(stf); b.loadedFile(stf, FateId.from(FateInstanceType.USER, UUID.randomUUID())); b.compacted(FateId.from(FateInstanceType.USER, UUID.randomUUID())); @@ -552,8 +552,8 @@ public void testTmBuilderImmutable() { assertThrows(UnsupportedOperationException.class, () -> tm2.getFilesMap().put(stf, new DataFileValue(0, 0, 0))); assertEquals(1, tm2.getLogs().size()); - assertThrows(UnsupportedOperationException.class, - () -> tm2.getLogs().add(LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()))); + assertThrows(UnsupportedOperationException.class, () -> tm2.getLogs() + .add(LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()))); assertEquals(1, tm2.getScans().size()); assertThrows(UnsupportedOperationException.class, () -> tm2.getScans().add(stf)); assertEquals(1, tm2.getLoaded().size()); @@ -829,8 +829,8 @@ public void testBuilder() { new CompactionMetadata(Set.of(sf1, sf2), rf1, "cid1", CompactionKind.USER, (short) 3, ResourceGroupId.of("Q1"), true, FateId.from(type, UUID.randomUUID())); - LogEntry le1 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); - LogEntry le2 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); FateId selFilesFateId = FateId.from(type, UUID.randomUUID()); SelectedFiles selFiles = new SelectedFiles(Set.of(sf1, sf4), false, selFilesFateId, diff --git a/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java b/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java index 579571d81de..9da934cf385 100644 --- a/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java +++ b/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java @@ -29,6 +29,7 @@ import java.util.stream.Stream; import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; @@ -39,6 +40,7 @@ public class LogEntryTest { private final String validHost = "localhost+9997"; private final UUID validUUID = UUID.randomUUID(); + private static final String PROVIDER_ID = "example"; private final String validPath = "viewfs:/a/accumulo/wal/" + validHost + "/" + validUUID; @Test @@ -48,8 +50,8 @@ public void testColumnFamily() { @Test public void testFromPath() { - var logEntry = LogEntry.fromPath(validPath); - verifyLogEntry(logEntry, new Text("-/" + validPath)); + var logEntry = LogEntry.fromPath(PROVIDER_ID, validPath); + verifyLogEntry(logEntry, new Text(PROVIDER_ID + "/" + validPath)); } @Test @@ -60,6 +62,33 @@ public void testFromMetadata() { verifyLogEntry(logEntry, new Text(columnQualifier)); } + @Test + public void testProviderIdentityFromMetadata() { + var entry = LogEntry.fromPath("example", validPath); + var qualifier = entry.getColumnQualifier(); + var restored = LogEntry.fromMetaWalEntry(new SimpleImmutableEntry<>( + new Key("1<", LogColumnFamily.STR_NAME, qualifier.toString()), new Value())); + + assertEquals("example", restored.getProviderId()); + assertEquals(validPath, restored.getPath()); + assertEquals(entry, restored); + assertEquals(PROVIDER_ID, LogEntry.fromPath(PROVIDER_ID, validPath).getProviderId()); + } + + @Test + public void testLegacyMetadataQualifierRejected() { + var legacyEntry = new SimpleImmutableEntry<>( + new Key("1<", LogColumnFamily.STR_NAME, "-/" + validPath), new Value()); + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromMetaWalEntry(legacyEntry)); + } + + @Test + public void testProviderIdMustBeAlphanumeric() { + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath("provider-id", validPath)); + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath("provider_id", validPath)); + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath("provider.id", validPath)); + } + // helper for testing build from constructor or from metadata private void verifyLogEntry(LogEntry logEntry, Text expectedColumnQualifier) { assertEquals(validPath, logEntry.toString()); @@ -71,8 +100,8 @@ private void verifyLogEntry(LogEntry logEntry, Text expectedColumnQualifier) { @Test public void testEquals() { - LogEntry one = LogEntry.fromPath(validPath); - LogEntry two = LogEntry.fromPath(validPath); + LogEntry one = LogEntry.fromPath(PROVIDER_ID, validPath); + LogEntry two = LogEntry.fromPath(PROVIDER_ID, validPath); assertNotSame(one, two); assertEquals(one.toString(), two.toString()); @@ -93,13 +122,14 @@ public void testValidPaths() { var validPath3 = "dir2/" + validPath2; Stream.of(validPath1, validPath2, validPath3) - .forEach(s -> assertDoesNotThrow(() -> LogEntry.fromPath(s))); + .forEach(s -> assertDoesNotThrow(() -> LogEntry.fromPath(PROVIDER_ID, s))); } @Test public void testBadPathLength() { Stream.of("foo", "", validHost).forEach(badPath -> { - var e = assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath(badPath)); + var e = assertThrows(IllegalArgumentException.class, + () -> LogEntry.fromPath(PROVIDER_ID, badPath)); assertTrue(e.getMessage().contains("The path should end with tserver/UUID.")); }); } @@ -108,7 +138,8 @@ public void testBadPathLength() { public void testInvalidHostPort() { Stream.of("default:9997", "default+badPort").forEach(badHostAndPort -> { var badPath = badHostAndPort + "/" + validUUID; - var e = assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath(badPath)); + var e = assertThrows(IllegalArgumentException.class, + () -> LogEntry.fromPath(PROVIDER_ID, badPath)); assertTrue(e.getMessage().contains("Expected: host+port. Found '" + badHostAndPort + "'")); }); } @@ -117,7 +148,8 @@ public void testInvalidHostPort() { public void testInvalidUUID() { var badUUID = "badUUID"; var pathWithBadUUID = validHost + "/" + badUUID; - var e = assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath(pathWithBadUUID)); + var e = assertThrows(IllegalArgumentException.class, + () -> LogEntry.fromPath(PROVIDER_ID, pathWithBadUUID)); assertTrue(e.getMessage().contains("Expected valid UUID. Found '" + badUUID + "'")); } diff --git a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java index 9a4c4fd9bcd..2205125f283 100644 --- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java @@ -101,6 +101,7 @@ import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.init.Initialize; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.util.AccumuloStatus; import org.apache.accumulo.server.util.PortUtils; import org.apache.accumulo.server.util.ZooZap; @@ -181,6 +182,16 @@ public MiniAccumuloClusterImpl(MiniAccumuloConfigImpl config) throws IOException config.setProperty(Property.MANAGER_TABLET_GROUP_WATCHER_INTERVAL, "5s"); } + if (!config.useExistingInstance()) { + if (!config.getSiteConfig().containsKey(Property.INSTANCE_WAL_PROVIDERS.getKey())) { + config.setProperty(Property.INSTANCE_WAL_PROVIDERS, + "{\"" + DfsWalProvider.class.getName() + "\":{}}"); + } + if (!config.getSiteConfig().containsKey(Property.INSTANCE_WAL_PROVIDER.getKey())) { + config.setProperty(Property.INSTANCE_WAL_PROVIDER, DfsWalProvider.class.getName()); + } + } + this.config = config.initialize(); this.clientProperties = Suppliers.memoize( () -> Accumulo.newClientProperties().from(config.getClientPropsFile().toPath()).build()); diff --git a/server/base/pom.xml b/server/base/pom.xml index 56a418487a3..098b5533a07 100644 --- a/server/base/pom.xml +++ b/server/base/pom.xml @@ -38,6 +38,10 @@ com.beust jcommander + + com.fasterxml.jackson.core + jackson-databind + com.github.ben-manes.caffeine caffeine diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java index 16b20f0d985..9949e94660e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java @@ -75,6 +75,7 @@ import org.apache.accumulo.server.conf.store.PropStore; import org.apache.accumulo.server.conf.store.impl.ZooPropStore; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.WalProviderRegistry; import org.apache.accumulo.server.mem.LowMemoryDetector; import org.apache.accumulo.server.metadata.ServerAmpleImpl; import org.apache.accumulo.server.metrics.MetricsInfoImpl; @@ -112,6 +113,7 @@ public class ServerContext extends ClientContext { private final Supplier sharedScheduledThreadPool; private final Supplier securityOperation; private final Supplier cryptoFactorySupplier; + private final Supplier walProviderRegistry; private final Supplier lowMemoryDetector; private final AtomicReference serverLock = new AtomicReference<>(); private final Supplier metricsInfoSupplier; @@ -148,6 +150,7 @@ private ServerContext(ServerInfo info, ResourceGroupId rgid) { secretManager = memoize(() -> new AuthenticationTokenSecretManager(getInstanceID(), getConfiguration().getTimeInMillis(Property.GENERAL_DELEGATION_TOKEN_LIFETIME))); cryptoFactorySupplier = memoize(() -> CryptoFactoryLoader.newInstance(getConfiguration())); + walProviderRegistry = memoize(() -> new WalProviderRegistry(this)); sharedScheduledThreadPool = memoize(() -> ThreadPools.getServerThreadPools() .createGeneralScheduledExecutorService(getConfiguration())); securityOperation = @@ -321,6 +324,10 @@ public CryptoServiceFactory getCryptoFactory() { return cryptoFactorySupplier.get(); } + public WalProviderRegistry getWalProviderRegistry() { + return walProviderRegistry.get(); + } + @Override public Ample getAmple() { return new ServerAmpleImpl(this); diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java index e41237b087b..6b961866ed2 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java @@ -34,6 +34,7 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.server.fs.VolumeManager.FileType; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; @@ -80,8 +81,12 @@ public static Path switchVolume(Path path, FileType ft, Map replaceme } public static LogEntry switchVolume(LogEntry le, Map replacements) { + if (!le.getProviderId().equals(DfsWalProvider.PROVIDER_ID)) { + return null; + } Path switchedPath = switchVolume(new Path(le.getPath()), FileType.WAL, replacements); - return switchedPath == null ? null : LogEntry.fromPath(switchedPath.toString()); + return switchedPath == null ? null + : LogEntry.fromPath(le.getProviderId(), switchedPath.toString()); } public static boolean needsVolumeReplacement(final Map replacements, @@ -130,6 +135,9 @@ public static void volumeReplacementEvaluation(final Map replacements log.trace("Using volume replacements: {}", replacements); for (LogEntry logEntry : tm.getLogs()) { + if (!logEntry.getProviderId().equals(DfsWalProvider.PROVIDER_ID)) { + continue; + } log.trace("Evaluating walog {} for replacement.", logEntry); LogEntry switchedLogEntry = switchVolume(logEntry, replacements); if (switchedLogEntry != null) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java new file mode 100644 index 00000000000..096c978b215 --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.log; + +import java.io.IOException; +import java.io.InputStream; +import java.util.UUID; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.Durability; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; +import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment; +import org.apache.accumulo.core.spi.wal.WalProvider; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.ServiceEnvironmentImpl; +import org.apache.accumulo.server.fs.VolumeChooserEnvironmentImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSOutputStream; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; + +/** Filesystem-backed WAL provider. */ +public class DfsWalProvider implements WalProvider { + + /** Stable provider ID used for WALs stored on HDFS. */ + public static final String PROVIDER_ID = "dfs"; + + private ServerContext context; + + @Override + public String getId() { + return PROVIDER_ID; + } + + @Override + public void init(ServiceEnvironment environment) { + if (!(environment instanceof ServiceEnvironmentImpl serviceEnvironment)) { + throw new IllegalArgumentException("DfsWalProvider requires the server service environment"); + } + context = serviceEnvironment.getContext(); + } + + @Override + public Writer create(String serverAddress, UUID id) throws IOException { + String addressForFilename = serverAddress.replace(':', '+'); + var chooserEnv = + new VolumeChooserEnvironmentImpl(VolumeChooserEnvironment.Scope.LOGGER, context); + String logPath = + context.getVolumeManager().choose(chooserEnv, context.getBaseUris()) + Path.SEPARATOR + + Constants.WAL_DIR + Path.SEPARATOR + addressForFilename + Path.SEPARATOR + id; + var path = new Path(logPath); + VolumeManager fs = context.getVolumeManager(); + short replication = (short) context.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION); + if (replication == 0) { + replication = fs.getDefaultReplication(path); + } + long blockSize = context.getConfiguration().getAsBytes(Property.TSERV_WAL_BLOCKSIZE); + if (blockSize == 0) { + blockSize = (long) (context.getConfiguration().getAsBytes(Property.TSERV_WAL_MAX_SIZE) * 1.1); + } + FSDataOutputStream output; + if (context.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC)) { + output = fs.createSyncable(path, 0, replication, blockSize); + } else { + output = fs.create(path, true, 0, replication, blockSize); + } + try { + output.setDropBehind(Boolean.TRUE); + } catch (UnsupportedOperationException e) { + // Not all filesystems support drop-behind. + } catch (IOException e) { + // Drop-behind is a performance hint, not a requirement. + } + if (!fs.canSyncAndFlush(path)) { + org.slf4j.LoggerFactory.getLogger(DfsWalProvider.class) + .warn("sync not supported for log file {}. Data loss may occur.", logPath); + } + return new DfsWriter(logPath, output); + } + + @Override + public InputStream open(String reference) throws IOException { + return context.getVolumeManager().open(new Path(reference)); + } + + @Override + public boolean exists(String reference) throws IOException { + return context.getVolumeManager().exists(new Path(reference)); + } + + @Override + public void delete(String reference) throws IOException { + VolumeManager fs = context.getVolumeManager(); + Path path = new Path(reference); + if (!fs.moveToTrash(path)) { + fs.deleteRecursively(path); + } + } + + private static class DfsWriter implements Writer { + private final String reference; + private final FSDataOutputStream output; + private boolean closed; + + DfsWriter(String reference, FSDataOutputStream output) { + this.reference = reference; + this.output = output; + } + + @Override + public String getReference() { + return reference; + } + + @Override + public java.io.OutputStream getOutputStream() { + return output; + } + + @Override + public void sync(Durability durability) throws IOException { + if (durability == Durability.SYNC) { + output.hsync(); + } else if (durability == Durability.FLUSH) { + output.hflush(); + } else { + throw new IllegalArgumentException("Unexpected WAL sync durability " + durability); + } + } + + @Override + public long getLength() throws IOException { + return output.getPos(); + } + + @Override + public int getCurrentReplication() throws IOException { + var wrapped = output.getWrappedStream(); + return wrapped instanceof DFSOutputStream dfs ? dfs.getCurrentBlockReplication() : -1; + } + + @Override + public String[] getPipeline() { + var wrapped = output.getWrappedStream(); + if (wrapped instanceof DFSOutputStream dfs) { + DatanodeInfo[] pipeline = dfs.getPipeline(); + String[] names = new String[pipeline.length]; + for (int i = 0; i < pipeline.length; i++) { + names[i] = pipeline[i].getName(); + } + return names; + } + return new String[0]; + } + + @Override + public synchronized void close() throws IOException { + if (!closed) { + output.close(); + closed = true; + } + } + } +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java new file mode 100644 index 00000000000..e08f4970100 --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.log; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.accumulo.core.classloader.ClassLoaderUtil; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.PropertyType; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; +import org.apache.accumulo.core.spi.wal.WalProvider; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.ServiceEnvironmentImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** Loads and validates the configured WAL providers for a server process. */ +public class WalProviderRegistry { + + private static final Logger log = LoggerFactory.getLogger(WalProviderRegistry.class); + private static final ObjectMapper JSON_MAPPER = + new ObjectMapper().enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); + + private final Map providers = new HashMap<>(); + private final Map providersByClass = new HashMap<>(); + private final WalProvider current; + + static void validateProviderId(String id, String className) { + if (id == null || !id.matches(WalProvider.PROVIDER_ID_REGEX)) { + throw new IllegalArgumentException("Invalid WAL provider ID '" + id + "' from " + className + + "; IDs must match " + WalProvider.PROVIDER_ID_REGEX); + } + } + + public WalProviderRegistry(ServerContext context) { + var conf = context.getConfiguration(); + ClassLoaderUtil.initContextFactory(conf); + var environment = new ServiceEnvironmentImpl(context); + String currentClass = conf.get(Property.INSTANCE_WAL_PROVIDER).trim(); + if (currentClass.isEmpty()) { + throw new IllegalArgumentException( + "No current WAL provider configured in " + Property.INSTANCE_WAL_PROVIDER); + } + var providerOptions = parseProviderConfig(conf.get(Property.INSTANCE_WAL_PROVIDERS)); + for (var entry : providerOptions.entrySet()) { + String className = entry.getKey(); + try { + WalProvider provider = + ConfigurationTypeHelper.getClassInstance(null, className, WalProvider.class); + provider.init(new ProviderInitParameters(environment, entry.getValue())); + String id = provider.getId(); + validateProviderId(id, className); + if (id.equals(DfsWalProvider.PROVIDER_ID) + && !className.equals(DfsWalProvider.class.getName())) { + throw new IllegalArgumentException("WAL provider ID '" + DfsWalProvider.PROVIDER_ID + + "' is reserved for legacy filesystem WALs and " + DfsWalProvider.class.getName()); + } + WalProvider previous = providers.putIfAbsent(id, provider); + if (previous != null) { + throw new IllegalArgumentException("Duplicate WAL provider ID '" + id + "' from " + + previous.getClass().getName() + " and " + className); + } + providersByClass.put(className, provider); + log.info("Loaded WAL provider {} with ID {}", className, id); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Unable to load configured WAL provider " + className, e); + } + } + + current = providersByClass.get(currentClass); + if (current == null) { + throw new IllegalArgumentException( + "Current WAL provider " + currentClass + " from " + Property.INSTANCE_WAL_PROVIDER + + " is not listed in " + Property.INSTANCE_WAL_PROVIDERS); + } + } + + static Map> parseProviderConfig(String json) { + final JsonNode root; + try { + root = JSON_MAPPER.readTree(json); + } catch (IOException e) { + throw new IllegalArgumentException("Invalid JSON in " + Property.INSTANCE_WAL_PROVIDERS, e); + } + if (root == null || !root.isObject()) { + throw new IllegalArgumentException(Property.INSTANCE_WAL_PROVIDERS + + " must be a JSON object mapping provider classes to option maps"); + } + + Map> providers = new LinkedHashMap<>(); + root.fields().forEachRemaining(entry -> { + String className = entry.getKey(); + if (className.isBlank() || !PropertyType.CLASSNAME.isValidFormat(className)) { + throw new IllegalArgumentException("Invalid WAL provider class name '" + className + "'"); + } + JsonNode optionsNode = entry.getValue(); + if (!optionsNode.isObject()) { + throw new IllegalArgumentException("Options for WAL provider " + className + + " must be a JSON object (use {} when there are no options)"); + } + Map options = new LinkedHashMap<>(); + optionsNode.fields().forEachRemaining(option -> { + if (!option.getValue().isTextual()) { + throw new IllegalArgumentException("Option '" + option.getKey() + "' for WAL provider " + + className + " must have a string value"); + } + options.put(option.getKey(), option.getValue().textValue()); + }); + providers.put(className, Collections.unmodifiableMap(options)); + }); + return Collections.unmodifiableMap(providers); + } + + private record ProviderInitParameters(ServiceEnvironment environment, Map options) + implements WalProvider.InitParameters { + @Override + public ServiceEnvironment getServiceEnvironment() { + return environment; + } + + @Override + public Map getOptions() { + return options; + } + } + + public WalProvider getCurrent() { + return current; + } + + public WalProvider get(String id) throws ProviderNotFoundException { + WalProvider provider = providers.get(id); + if (provider == null) { + throw new ProviderNotFoundException(id); + } + return provider; + } + + public Set getIds() { + return Set.copyOf(providers.keySet()); + } + + public static class ProviderNotFoundException extends IOException { + private static final long serialVersionUID = 1L; + + public ProviderNotFoundException(String id) { + super("WAL provider with ID '" + id + "' is not configured in " + + Property.INSTANCE_WAL_PROVIDERS); + } + } +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java index a0c18971a39..eaca6c4ebd8 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java @@ -33,6 +33,7 @@ import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy; import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeMissingPolicy; import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.server.ServerContext; import org.apache.hadoop.fs.Path; import org.apache.zookeeper.KeeperException; @@ -51,7 +52,7 @@ * The state of the WALs is kept in Zookeeper under /accumulo/<instanceid>/wals. For each * server, there is a znode formatted like the TServerInstance.toString(): "host:port[sessionid]". * Under the server znode, is a node for each log, using the UUID for the log. In each of the WAL - * znodes, is the current state of the log, and the full path to the log. + * znodes, is the current state of the log, the provider ID, and the full path to the log. * *

* The state [OPEN, CLOSED, UNREFERENCED] is what the tablet server believes to be the state of the @@ -86,7 +87,7 @@ public enum WalState { UNREFERENCED } - public record WalStatePath(WalState state, Path path) { + public record WalStatePath(WalState state, String providerId, Path path) { } private final ZooReaderWriter zoo; @@ -131,12 +132,21 @@ public void initWalMarker(TServerInstance tsi) throws WalMarkerException { // Tablet server opens a new WAL public void addNewWalMarker(TServerInstance tsi, Path path) throws WalMarkerException { - updateState(tsi, path, WalState.OPEN); + updateState(tsi, DfsWalProvider.PROVIDER_ID, path, WalState.OPEN); + } + + public void addNewWalMarker(TServerInstance tsi, LogEntry entry) throws WalMarkerException { + updateState(tsi, entry.getProviderId(), new Path(entry.getPath()), WalState.OPEN); } private void updateState(TServerInstance tsi, Path path, WalState state) throws WalMarkerException { - byte[] data = (state + "," + path).getBytes(UTF_8); + updateState(tsi, DfsWalProvider.PROVIDER_ID, path, state); + } + + private void updateState(TServerInstance tsi, String providerId, Path path, WalState state) + throws WalMarkerException { + byte[] data = (state + "," + providerId + "," + path).getBytes(UTF_8); try { NodeExistsPolicy policy = NodeExistsPolicy.OVERWRITE; if (state == WalState.OPEN) { @@ -157,14 +167,25 @@ public void walUnreferenced(TServerInstance tsi, Path path) throws WalMarkerExce updateState(tsi, path, WalState.UNREFERENCED); } + public void walUnreferenced(TServerInstance tsi, LogEntry entry) throws WalMarkerException { + updateState(tsi, entry.getProviderId(), new Path(entry.getPath()), WalState.UNREFERENCED); + } + + public void closeWal(TServerInstance tsi, LogEntry entry) throws WalMarkerException { + closeWal(tsi, new Path(entry.getPath()), entry.getProviderId()); + } + public static WalStatePath parse(byte[] data) { - String[] parts = new String(data, UTF_8).split(","); - return new WalStatePath(WalState.valueOf(parts[0]), new Path(parts[1])); + String[] parts = new String(data, UTF_8).split(",", 3); + if (parts.length != 3) { + throw new IllegalArgumentException("Malformed WAL state: " + new String(data, UTF_8)); + } + return new WalStatePath(WalState.valueOf(parts[0]), parts[1], new Path(parts[2])); } // Manager needs to know the logs for the given instance - public List getWalsInUse(TServerInstance tsi) throws WalMarkerException { - List result = new ArrayList<>(); + public List getWalsInUse(TServerInstance tsi) throws WalMarkerException { + List result = new ArrayList<>(); try { String zpath = root() + "/" + tsi; zoo.sync(zpath); @@ -183,7 +204,7 @@ public List getWalsInUse(TServerInstance tsi) throws WalMarkerException { if (zdata != null) { WalStatePath parts = parse(zdata); if (parts.state() != WalState.UNREFERENCED) { - result.add(parts.path()); + result.add(LogEntry.fromPath(parts.providerId(), parts.path().toString())); } } } @@ -244,7 +265,13 @@ public Set getAllState() throws WalMarkerException { for (Entry> entry : markers.entrySet()) { for (UUID id : entry.getValue()) { // WAL markers can be deleted while reading them. - result.add(state(entry.getKey(), id)); + try { + result.add(state(entry.getKey(), id)); + } catch (WalMarkerException e) { + if (!(e.getCause() instanceof KeeperException.NoNodeException)) { + throw e; + } + } } } return result; @@ -281,6 +308,21 @@ public void forget(TServerInstance instance) throws WalMarkerException { // manager can mark a log as unreferenced after it has made log recovery markers on the tablets // that need to be recovered public void closeWal(TServerInstance instance, Path path) throws WalMarkerException { - updateState(instance, path, WalState.CLOSED); + closeWal(instance, path, null); + } + + private void closeWal(TServerInstance instance, Path path, String providerId) + throws WalMarkerException { + WalStatePath existing; + try { + existing = state(instance, UUID.fromString(path.getName())); + } catch (WalMarkerException e) { + if (e.getCause() instanceof KeeperException.NoNodeException) { + return; + } + throw e; + } + updateState(instance, providerId == null ? existing.providerId() : providerId, path, + WalState.CLOSED); } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java index 775331960a3..a550c43cc9f 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java @@ -36,7 +36,6 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.server.ServerContext; -import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -114,13 +113,13 @@ public Set setFutureLocations(Collection assignments) @Override public void unassign(Collection tablets, - Map> logsForDeadServers) throws DistributedStoreException { + Map> logsForDeadServers) throws DistributedStoreException { unassign(tablets, logsForDeadServers, null); } @Override public void suspend(Collection tablets, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException { unassign(tablets, logsForDeadServers, suspensionTimestamp); } @@ -129,7 +128,7 @@ protected abstract void processSuspension(Ample.ConditionalTabletMutator tabletM TabletMetadata tm, SteadyTime suspensionTimestamp); private void unassign(Collection tablets, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException { try (var tabletsMutator = ample.conditionallyMutateTablets()) { for (TabletMetadata tm : tablets) { @@ -145,11 +144,10 @@ private void unassign(Collection tablets, updateLastLocation(tabletMutator, tm.getLocation().getServerInstance(), tm.getLast()); tabletMutator.deleteLocation(tm.getLocation()); if (logsForDeadServers != null) { - List logs = logsForDeadServers.get(tm.getLocation().getServerInstance()); + List logs = logsForDeadServers.get(tm.getLocation().getServerInstance()); if (logs != null) { - for (Path log : logs) { - LogEntry entry = LogEntry.fromPath(log.toString()); - tabletMutator.putWal(entry); + for (LogEntry log : logs) { + tabletMutator.putWal(log); } } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java index b5da85b3ad9..bd1cd462604 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java @@ -30,8 +30,8 @@ import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; -import org.apache.hadoop.fs.Path; import com.google.common.net.HostAndPort; @@ -82,7 +82,7 @@ public void setLocations(Collection assignments) throws DistributedS @Override public void unassign(Collection tablets, - Map> logsForDeadServers) throws DistributedStoreException { + Map> logsForDeadServers) throws DistributedStoreException { wrapped.unassign(tablets, logsForDeadServers); if (logsForDeadServers == null) { @@ -96,7 +96,7 @@ public void unassign(Collection tablets, @Override public void suspend(Collection tablets, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException { wrapped.suspend(tablets, logsForDeadServers, suspensionTimestamp); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java index d369f7bae76..06eefd79892 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java @@ -31,9 +31,9 @@ import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.server.ServerContext; -import org.apache.hadoop.fs.Path; /** * Interface for storing information about tablet assignments. There are three implementations: @@ -84,14 +84,14 @@ Set setFutureLocations(Collection assignments) * @param logsForDeadServers a cache of logs in use by servers when they died */ void unassign(Collection tablets, - Map> logsForDeadServers) throws DistributedStoreException; + Map> logsForDeadServers) throws DistributedStoreException; /** * Mark tablets as having no known or future location, but desiring to be returned to their * previous tserver. */ void suspend(Collection tablets, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException; /** @@ -100,13 +100,13 @@ void suspend(Collection tablets, void unsuspend(Collection tablets) throws DistributedStoreException; public static void unassign(ServerContext context, TabletMetadata tm, - Map> logsForDeadServers) throws DistributedStoreException { + Map> logsForDeadServers) throws DistributedStoreException { getStoreForTablet(tm.getExtent(), context).unassign(Collections.singletonList(tm), logsForDeadServers); } public static void suspend(ServerContext context, TabletMetadata tm, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException { getStoreForTablet(tm.getExtent(), context).suspend(Collections.singletonList(tm), logsForDeadServers, suspensionTimestamp); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java index 600b96a8feb..92f01ca8c6b 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java @@ -43,11 +43,11 @@ import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.iterators.SystemIteratorEnvironment; import org.apache.accumulo.server.iterators.SystemIteratorEnvironmentImpl; -import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -177,7 +177,7 @@ private static void validateTablets(Collection tablets) { @Override public void unassign(Collection tablets, - Map> logsForDeadServers) throws DistributedStoreException { + Map> logsForDeadServers) throws DistributedStoreException { validateTablets(tablets); super.unassign(tablets, logsForDeadServers); @@ -187,7 +187,7 @@ public void unassign(Collection tablets, @Override public void suspend(Collection tablets, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException { validateTablets(tablets); super.suspend(tablets, logsForDeadServers, suspensionTimestamp); diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java index 0c0998812cf..1c7c3c8288e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java @@ -171,7 +171,7 @@ private static boolean checkZKWALsMetadata(ServerContext context) throws Excepti Map> missingWals = new HashMap<>(); for (var instanceAndWals : walsBefore.entrySet()) { for (var wal : instanceAndWals.getValue()) { - if (!context.getVolumeManager().exists(wal.path())) { + if (!context.getWalProviderRegistry().get(wal.providerId()).exists(wal.path().toString())) { missingWals.computeIfAbsent(instanceAndWals.getKey(), k -> new HashSet<>()).add(wal); } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java index 033cf172125..46d65958f5e 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.server.fs.VolumeManager.FileType; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.Test; @@ -170,17 +171,17 @@ public void testWalVolumeReplacment() { String walUUID = UUID.randomUUID().toString(); String fileName = "hdfs://nn1/accumulo/wal/localhost+9997/" + walUUID; - LogEntry le = LogEntry.fromPath(fileName); + LogEntry le = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, fileName); LogEntry fixedVolume = VolumeUtil.switchVolume(le, replacements); assertEquals("viewfs:/a/accumulo/wal/localhost+9997/" + walUUID, fixedVolume.getPath()); fileName = "hdfs://nn1:9000/accumulo/wal/localhost+9997/" + walUUID; - le = LogEntry.fromPath(fileName); + le = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, fileName); fixedVolume = VolumeUtil.switchVolume(le, replacements); assertEquals("viewfs:/a/accumulo/wal/localhost+9997/" + walUUID, fixedVolume.getPath()); fileName = "hdfs://nn2/accumulo/wal/localhost+9997/" + walUUID; - le = LogEntry.fromPath(fileName); + le = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, fileName); fixedVolume = VolumeUtil.switchVolume(le, replacements); assertEquals("viewfs:/b/accumulo/wal/localhost+9997/" + walUUID, fixedVolume.getPath()); } diff --git a/server/base/src/test/java/org/apache/accumulo/server/log/WalProviderRegistryTest.java b/server/base/src/test/java/org/apache/accumulo/server/log/WalProviderRegistryTest.java new file mode 100644 index 00000000000..b50feaa7b06 --- /dev/null +++ b/server/base/src/test/java/org/apache/accumulo/server/log/WalProviderRegistryTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.log; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +class WalProviderRegistryTest { + + @Test + void providerIdsMustBeAlphanumeric() { + assertDoesNotThrow(() -> WalProviderRegistry.validateProviderId("Provider123", "provider")); + assertThrows(IllegalArgumentException.class, + () -> WalProviderRegistry.validateProviderId("provider-id", "provider")); + assertThrows(IllegalArgumentException.class, + () -> WalProviderRegistry.validateProviderId("provider_id", "provider")); + assertThrows(IllegalArgumentException.class, + () -> WalProviderRegistry.validateProviderId("provider.id", "provider")); + assertThrows(IllegalArgumentException.class, + () -> WalProviderRegistry.validateProviderId("-", "provider")); + } + + @Test + void parsesProviderClassesAndOptions() { + var config = WalProviderRegistry.parseProviderConfig( + "{\"org.example.Provider\":{},\"org.example.OtherProvider\":{\"endpoint\":\"test\",\"retries\":\"3\"}}"); + + assertEquals(Map.of("org.example.Provider", Map.of(), "org.example.OtherProvider", + Map.of("endpoint", "test", "retries", "3")), config); + } + + @Test + void rejectsInvalidProviderConfiguration() { + List.of("[]", "\"provider\"", "{\"\":{}}", "{\"org.example.Provider\":\"\"}", + "{\"org.example.Provider\":{\"count\":3}}", "{\"org.example.Provider\":{}} trailing", + "{\"org.example.Provider\":{},\"org.example.Provider\":{}}") + .forEach(json -> assertThrows(IllegalArgumentException.class, + () -> WalProviderRegistry.parseProviderConfig(json), json)); + } +} diff --git a/server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java b/server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java new file mode 100644 index 00000000000..90919873aae --- /dev/null +++ b/server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.log; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; + +import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.core.zookeeper.ZooSession; +import org.apache.accumulo.server.ServerContext; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.Test; + +class WalStateManagerTest { + + @Test + void parseDefaultProviderWalState() { + var parsed = WalStateManager + .parse(("CLOSED," + DfsWalProvider.PROVIDER_ID + ",hdfs://nn/accumulo/wal/host+1234/id") + .getBytes(UTF_8)); + assertEquals(WalStateManager.WalState.CLOSED, parsed.state()); + assertEquals(DfsWalProvider.PROVIDER_ID, parsed.providerId()); + assertEquals(new Path("hdfs://nn/accumulo/wal/host+1234/id"), parsed.path()); + } + + @Test + void rejectLegacyWalState() { + assertThrows(IllegalArgumentException.class, + () -> WalStateManager.parse("CLOSED,hdfs://nn/accumulo/wal/host+1234/id".getBytes(UTF_8))); + } + + @Test + void parseProviderWalState() { + var parsed = WalStateManager + .parse("UNREFERENCED,example,hdfs://nn/accumulo/wal/host+1234/id".getBytes(UTF_8)); + assertEquals(WalStateManager.WalState.UNREFERENCED, parsed.state()); + assertEquals("example", parsed.providerId()); + assertEquals(new Path("hdfs://nn/accumulo/wal/host+1234/id"), parsed.path()); + } + + @Test + void getWalsInUsePreservesProviderId() throws Exception { + ServerContext context = createMock(ServerContext.class); + ZooSession session = createMock(ZooSession.class); + ZooReaderWriter zoo = createMock(ZooReaderWriter.class); + TServerInstance server = new TServerInstance("localhost:1234[SESSION]"); + String serverPath = WalStateManager.ZWALS + "/" + server; + String reference = + "customfs://nn/accumulo/wal/localhost+1234/11111111-1111-1111-1111-111111111111"; + + expect(context.getZooSession()).andReturn(session); + expect(session.asReaderWriter()).andReturn(zoo); + expect(zoo.exists(WalStateManager.ZWALS)).andReturn(true); + zoo.sync(serverPath); + expectLastCall(); + expect(zoo.getChildren(serverPath)).andReturn(List.of("11111111-1111-1111-1111-111111111111")); + expect(zoo.getData(serverPath + "/11111111-1111-1111-1111-111111111111")) + .andReturn(("CLOSED,custom," + reference).getBytes(UTF_8)); + + replay(context, session, zoo); + WalStateManager manager = new WalStateManager(context); + assertEquals(List.of(LogEntry.fromPath("custom", reference)), manager.getWalsInUse(server)); + verify(context, session, zoo); + } +} diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java index 0bab62dfbc2..21d5fa0b1c1 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java @@ -55,6 +55,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; @@ -108,9 +109,11 @@ private SortedMap createMetadataEntryKV(KeyExtent extent) { mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); - LogEntry le1 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); le1.addToMutation(mutation); - LogEntry le2 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); le2.addToMutation(mutation); StoredTabletFile sf1 = diff --git a/server/base/src/test/resources/accumulo.properties b/server/base/src/test/resources/accumulo.properties index b63ed5c0617..2a7b4af5260 100644 --- a/server/base/src/test/resources/accumulo.properties +++ b/server/base/src/test/resources/accumulo.properties @@ -21,3 +21,5 @@ instance.dfs.dir=${project.build.directory}/instanceTest instance.secret=TEST_SYSTEM_SECRET +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java index b09c9ea9975..6146b1d2147 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java @@ -56,6 +56,7 @@ import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.WalStateManager; import org.apache.accumulo.server.log.WalStateManager.WalMarkerException; import org.apache.accumulo.server.log.WalStateManager.WalState; @@ -270,8 +271,23 @@ private long removeFiles(Collection collection, final GCStatus sta try { for (WalStatePath stateFile : collection) { Path path = stateFile.path(); - futures.put(path, removeFile(deleteThreadPool, path, counter, - "Removing " + stateFile.state() + " WAL " + path)); + if (stateFile.providerId().equals(DfsWalProvider.PROVIDER_ID)) { + futures.put(path, removeFile(deleteThreadPool, path, counter, + "Removing " + stateFile.state() + " WAL " + path)); + } else { + futures.put(path, deleteThreadPool.submit(() -> { + try { + log.debug("Removing {} WAL {} using provider {}", stateFile.state(), path, + stateFile.providerId()); + context.getWalProviderRegistry().get(stateFile.providerId()).delete(path.toString()); + counter.incrementAndGet(); + } catch (FileNotFoundException e) { + // ignored + } catch (IOException e) { + log.error("Unable to delete {} WAL {}", stateFile.providerId(), path, e); + } + })); + } } while (!futures.isEmpty()) { diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 13044b69aa9..516897970fc 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -96,6 +96,7 @@ public class SimpleGarbageCollector extends AbstractServer implements Iface { protected SimpleGarbageCollector(ServerOpts opts, String[] args) { super(ServerId.Type.GARBAGE_COLLECTOR, opts, ServerContext::new, args); + getContext().getWalProviderRegistry(); final AccumuloConfiguration conf = getConfiguration(); diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java index 99064550840..e769e104562 100644 --- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java +++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java @@ -41,6 +41,7 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.WalStateManager; import org.apache.accumulo.server.log.WalStateManager.WalState; import org.apache.accumulo.server.log.WalStateManager.WalStatePath; @@ -99,7 +100,7 @@ public void testRemoveUnusedLog() throws Exception { EasyMock.expect(marker.getAllMarkers()).andReturn(markers).once(); EasyMock.expect(marker.state(server1, id)) - .andReturn(new WalStatePath(WalState.UNREFERENCED, path)); + .andReturn(new WalStatePath(WalState.UNREFERENCED, DfsWalProvider.PROVIDER_ID, path)); EasyMock.expect(fs.deleteRecursively(path)).andReturn(true).once(); marker.removeWalMarker(server1, id); EasyMock.expectLastCall().once(); @@ -139,7 +140,8 @@ public void testKeepClosedLog() throws Exception { EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1)); EasyMock.expect(marker.getAllMarkers()).andReturn(markers).once(); - EasyMock.expect(marker.state(server1, id)).andReturn(new WalStatePath(WalState.CLOSED, path)); + EasyMock.expect(marker.state(server1, id)) + .andReturn(new WalStatePath(WalState.CLOSED, DfsWalProvider.PROVIDER_ID, path)); EasyMock.replay(conf, context, marker, tserverSet, fs); var gc = new GarbageCollectWriteAheadLogs(context, fs, tserverSet, marker) { @@ -175,7 +177,8 @@ public void deleteUnreferencedLogOnDeadServer() throws Exception { EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1)); EasyMock.expect(marker.getAllMarkers()).andReturn(markers2).once(); - EasyMock.expect(marker.state(server2, id)).andReturn(new WalStatePath(WalState.OPEN, path)); + EasyMock.expect(marker.state(server2, id)) + .andReturn(new WalStatePath(WalState.OPEN, DfsWalProvider.PROVIDER_ID, path)); EasyMock.expect(fs.deleteRecursively(path)).andReturn(true).once(); marker.removeWalMarker(server2, id); @@ -215,7 +218,8 @@ public void ignoreReferenceLogOnDeadServer() throws Exception { EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1)); EasyMock.expect(marker.getAllMarkers()).andReturn(markers2).once(); - EasyMock.expect(marker.state(server2, id)).andReturn(new WalStatePath(WalState.OPEN, path)); + EasyMock.expect(marker.state(server2, id)) + .andReturn(new WalStatePath(WalState.OPEN, DfsWalProvider.PROVIDER_ID, path)); EasyMock.replay(conf, context, fs, marker, tserverSet); GarbageCollectWriteAheadLogs gc = diff --git a/server/gc/src/test/resources/accumulo.properties b/server/gc/src/test/resources/accumulo.properties index 7a9d5c90663..1669b68ee68 100644 --- a/server/gc/src/test/resources/accumulo.properties +++ b/server/gc/src/test/resources/accumulo.properties @@ -18,3 +18,5 @@ # # File needs to exist for unit tests +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/BalanceManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/BalanceManager.java index 9baae8d74dd..5135d72965c 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/BalanceManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/BalanceManager.java @@ -312,7 +312,7 @@ private void balanceCompleted(Ample.DataLevel level, long migrations) { /** * Waits for the given data levels to complete a run of balancing with zero migrations. */ - private void waitForBalance(Set levels) { + void waitForBalance(Set levels) { synchronized (balancedNotifier) { var snapshot = new EnumMap(Ample.DataLevel.class); for (var dl : levels) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index ed20d3e0ffc..ce0ac1e9dbb 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -104,8 +105,10 @@ import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metrics.MetricsInfo; import org.apache.accumulo.core.metrics.MetricsProducer; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.Retry; import org.apache.accumulo.core.util.Timer; @@ -133,6 +136,9 @@ import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.compaction.CompactionConfigStorage; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.WalProviderRegistry; +import org.apache.accumulo.server.log.WalStateManager; +import org.apache.accumulo.server.log.WalStateManager.WalStatePath; import org.apache.accumulo.server.manager.LiveTServerSet; import org.apache.accumulo.server.manager.LiveTServerSet.LiveTServersSnapshot; import org.apache.accumulo.server.manager.LiveTServerSet.TServerConnection; @@ -486,6 +492,7 @@ protected Manager(ServerOpts opts, String[] args) throws IOException { super(ServerId.Type.MANAGER, opts, serverContextFactory, args); ServerContext context = super.getContext(); + context.getWalProviderRegistry(); upgradeCoordinator = new UpgradeCoordinator(context); balanceManager = new BalanceManager(); @@ -917,6 +924,36 @@ private void setupPrimaryMetrics() { } } + private void validateWalProviderReferences() { + getBalanceManager().waitForBalance(EnumSet.of(DataLevel.ROOT, DataLevel.METADATA)); + WalProviderRegistry registry = getContext().getWalProviderRegistry(); + Set referencedIds = new HashSet<>(); + for (DataLevel level : DataLevel.values()) { + try (var tablets = getContext().getAmple().readTablets().forLevel(level) + .fetch(TabletMetadata.ColumnType.LOGS).build()) { + for (TabletMetadata tablet : tablets) { + for (LogEntry entry : tablet.getLogs()) { + referencedIds.add(entry.getProviderId()); + } + } + } + } + try { + for (WalStatePath state : new WalStateManager(getContext()).getAllState()) { + referencedIds.add(state.providerId()); + } + } catch (WalStateManager.WalMarkerException e) { + throw new IllegalStateException("Unable to audit WAL provider markers", e); + } + referencedIds.removeAll(registry.getIds()); + if (!referencedIds.isEmpty()) { + throw new IllegalStateException("WAL provider IDs referenced by tablet metadata or WAL " + + "markers are not configured in " + Property.INSTANCE_WAL_PROVIDERS + ": " + + referencedIds); + } + log.info("Validated configured WAL providers against all tablet metadata and WAL markers"); + } + @Override public void run() { final ServerContext context = getContext(); @@ -1208,6 +1245,11 @@ boolean canSuspendTablets() { log.info("AuthenticationTokenSecretManager is initialized"); } + // Validate that the wals in the tables use + // known providers. If that's not the case, + // then abort. + validateWalProviderReferences(); + // Replace the ServiceLockData information in the Manager lock node in ZooKeeper. // This advertises the address that clients can use to connect to the Manager // for the Coordinator, Fate, and Manager services. Do **not** do this until diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java index fb2eaa62a9f..1719198a305 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java @@ -86,6 +86,7 @@ import org.apache.accumulo.core.metrics.Metric; import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.Timer; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.core.util.threads.Threads.AccumuloDaemonThread; @@ -111,7 +112,6 @@ import org.apache.accumulo.server.manager.state.TabletManagementParameters; import org.apache.accumulo.server.manager.state.TabletStateStore; import org.apache.accumulo.server.manager.state.UnassignedTablet; -import org.apache.hadoop.fs.Path; import org.apache.thrift.TException; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -257,7 +257,7 @@ private static class TabletLists { private final List assignedToDeadServers = new ArrayList<>(); private final List suspendedToGoneServers = new ArrayList<>(); private final Map unassigned = new HashMap<>(); - private final Map> logsForDeadServers = new TreeMap<>(); + private final Map> logsForDeadServers = new TreeMap<>(); // read only list of tablet servers that are not shutting down private final SortedMap destinations; private final Map> currentTServerGrouping; @@ -1206,10 +1206,10 @@ private void replaceVolumes(List volumeReplacemen } private static void markDeadServerLogsAsClosed(WalStateManager mgr, - Map> logsForDeadServers) throws WalMarkerException { - for (Entry> server : logsForDeadServers.entrySet()) { - for (Path path : server.getValue()) { - mgr.closeWal(server.getKey(), path); + Map> logsForDeadServers) throws WalMarkerException { + for (Entry> server : logsForDeadServers.entrySet()) { + for (LogEntry log : server.getValue()) { + mgr.closeWal(server.getKey(), log); } } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java index 8e1e00988f6..f1f7dc63b0e 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java @@ -41,6 +41,7 @@ import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.manager.Manager; import org.apache.accumulo.server.fs.VolumeUtil; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.SortedLogState; import org.apache.accumulo.server.manager.recovery.HadoopLogCloser; import org.apache.accumulo.server.manager.recovery.LogCloser; @@ -87,10 +88,13 @@ private class LogSortTask implements Runnable { private final String source; private final String destination; private final String sortId; + private final String providerId; private final LogCloser closer; - public LogSortTask(LogCloser closer, String source, String destination, String sortId) { + public LogSortTask(LogCloser closer, String providerId, String source, String destination, + String sortId) { this.closer = closer; + this.providerId = providerId; this.source = source; this.destination = destination; this.sortId = sortId; @@ -100,15 +104,15 @@ public LogSortTask(LogCloser closer, String source, String destination, String s public void run() { boolean rescheduled = false; try { - long time = closer.close(manager.getConfiguration(), manager.getContext().getHadoopConf(), - manager.getVolumeManager(), new Path(source)); + long time = closer == null ? 0 : closer.close(manager.getConfiguration(), + manager.getContext().getHadoopConf(), manager.getVolumeManager(), new Path(source)); if (time > 0) { ScheduledFuture future = executor.schedule(this, time, TimeUnit.MILLISECONDS); ThreadPools.watchNonCriticalScheduledTask(future); rescheduled = true; } else { - initiateSort(sortId, source, destination); + initiateSort(sortId, providerId, source, destination); } } catch (FileNotFoundException e) { log.debug("Unable to initiate log sort for " + source + ": " + e); @@ -125,9 +129,9 @@ public void run() { } - private void initiateSort(String sortId, String source, final String destination) - throws KeeperException, InterruptedException { - String work = source + "|" + destination; + private void initiateSort(String sortId, String providerId, String source, + final String destination) throws KeeperException, InterruptedException { + String work = providerId + "|" + source + "|" + destination; new DistributedWorkQueue(Constants.ZRECOVERY, manager.getConfiguration(), manager) .addWork(sortId, work.getBytes(UTF_8)); @@ -180,8 +184,8 @@ public RecoverySession newRecoverySession() { private boolean recoverLog(LogEntry walog) throws IOException { boolean recoveryNeeded = false; - LogEntry switchedWalog = - VolumeUtil.switchVolume(walog, manager.getContext().getVolumeReplacements()); + LogEntry switchedWalog = walog.getProviderId().equals(DfsWalProvider.PROVIDER_ID) + ? VolumeUtil.switchVolume(walog, manager.getContext().getVolumeReplacements()) : null; if (switchedWalog != null) { // replaces the volume used for sorting, but do not change entry in metadata table. When // the tablet loads it will change the metadata table entry. If @@ -221,8 +225,10 @@ private boolean recoverLog(LogEntry walog) throws IOException { synchronized (this) { if (!closeTasksQueued.contains(sortId) && !sortsQueued.contains(sortId)) { AccumuloConfiguration aconf = manager.getConfiguration(); - LogCloser closer = Property.createInstanceFromPropertyName(aconf, - Property.MANAGER_WAL_CLOSER_IMPLEMENTATION, LogCloser.class, new HadoopLogCloser()); + LogCloser closer = walog.getProviderId().equals(DfsWalProvider.PROVIDER_ID) + ? Property.createInstanceFromPropertyName(aconf, + Property.MANAGER_WAL_CLOSER_IMPLEMENTATION, LogCloser.class, new HadoopLogCloser()) + : null; Long delay = recoveryDelay.get(sortId); if (delay == null) { delay = aconf.getTimeInMillis(Property.MANAGER_RECOVERY_DELAY); @@ -233,7 +239,8 @@ private boolean recoverLog(LogEntry walog) throws IOException { log.info("Starting recovery of {} (in : {}s)", filename, (delay / 1000)); ScheduledFuture future = executor.schedule( - new LogSortTask(closer, filename, dest, sortId), delay, TimeUnit.MILLISECONDS); + new LogSortTask(closer, walog.getProviderId(), filename, dest, sortId), delay, + TimeUnit.MILLISECONDS); ThreadPools.watchNonCriticalScheduledTask(future); closeTasksQueued.add(sortId); recoveryDelay.put(sortId, delay); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java index d4ea54cc55b..43483578d5e 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java @@ -80,6 +80,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily; import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMergeabilityMetadata; @@ -87,6 +88,7 @@ import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.schema.Section; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.spi.wal.WalProvider; import org.apache.accumulo.core.util.Encoding; import org.apache.accumulo.core.util.TextUtil; import org.apache.accumulo.core.util.UtilWaitThread; @@ -101,6 +103,8 @@ import org.apache.accumulo.server.conf.store.TablePropKey; import org.apache.accumulo.server.init.FileSystemInitializer; import org.apache.accumulo.server.init.InitialConfiguration; +import org.apache.accumulo.server.log.DfsWalProvider; +import org.apache.accumulo.server.log.WalStateManager; import org.apache.accumulo.server.util.FindCompactionTmpFiles.DeleteStats; import org.apache.accumulo.server.util.PropUtil; import org.apache.hadoop.fs.FileStatus; @@ -226,7 +230,7 @@ public static String getRowPrefix() { @VisibleForTesting static final Set UPGRADE_FAMILIES = Set.of(DataFileColumnFamily.NAME, CHOPPED, - ExternalCompactionColumnFamily.NAME, ScanFileColumnFamily.NAME); + ExternalCompactionColumnFamily.NAME, ScanFileColumnFamily.NAME, LogColumnFamily.NAME); @VisibleForTesting static final String ZNAMESPACE_NAME = "/name"; @@ -238,6 +242,8 @@ public static String getRowPrefix() { public void upgradeZookeeper(ServerContext context) { LOG.info("Ensuring all worker server processes are down."); validateEmptyZKWorkerServerPaths(context); + LOG.info("Upgrading WAL provider IDs in ZooKeeper markers"); + upgradeWalMarkers(context); LOG.info("Removing ZTracer node"); removeZTracersNode(context); LOG.info("Updating file references in root tablet"); @@ -430,6 +436,10 @@ private void removeCompactColumnsFromRootTabletMetadata(ServerContext context) { zrw.overwritePersistentData(ZROOT_TABLET, rtm.toJson().getBytes(UTF_8), stat.getVersion()); LOG.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson()); } + RootTabletMetadata upgraded = + new RootTabletMetadata(new String(zrw.getData(ZROOT_TABLET), UTF_8)); + upgraded.getKeyValues() + .forEach(entry -> verifyNoLegacyWalReference(entry.getKey(), "root tablet metadata")); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new IllegalStateException( @@ -862,6 +872,8 @@ void processReferences(MutationWriter batchWriter, Iterable tableName, key.getRow()); update.at().family(ExternalCompactionColumnFamily.NAME) .qualifier(key.getColumnQualifier()).delete(); + } else if (family.equals(LogColumnFamily.NAME)) { + upgradeWalReference(key, value, update); } else { throw new IllegalStateException("Processing: " + tableName + " Received unexpected column family processing references: " + family); @@ -879,6 +891,81 @@ void processReferences(MutationWriter batchWriter, Iterable } } + @VisibleForTesting + static void upgradeWalReference(Key key, Value value, Mutation mutation) { + String qualifier = key.getColumnQualifier().toString(); + if (qualifier.startsWith("-/")) { + Text upgradedQualifier = new Text(DfsWalProvider.PROVIDER_ID + qualifier.substring(1)); + mutation.at().family(LogColumnFamily.NAME).qualifier(upgradedQualifier).put(value); + mutation.at().family(LogColumnFamily.NAME).qualifier(key.getColumnQualifier()).delete(); + } + } + + private static void verifyNoLegacyWalReference(Key key, String source) { + if (key.getColumnFamily().equals(LogColumnFamily.NAME) + && key.getColumnQualifier().toString().startsWith("-/")) { + throw new IllegalStateException( + "Legacy WAL reference remains after upgrade in " + source + ": " + key); + } + } + + static String upgradeWalMarker(String markerData) { + String[] fields = markerData.split(",", 3); + String upgraded; + if (fields.length == 2) { + upgraded = fields[0] + "," + DfsWalProvider.PROVIDER_ID + "," + fields[1]; + } else if (fields.length == 3) { + upgraded = markerData; + } else { + throw new IllegalStateException("Malformed WAL marker: " + markerData); + } + + String[] upgradedFields = upgraded.split(",", 3); + try { + WalStateManager.WalState.valueOf(upgradedFields[0]); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Malformed WAL marker: " + markerData, e); + } + if (!upgradedFields[1].matches(WalProvider.PROVIDER_ID_REGEX) || upgradedFields[2].isBlank()) { + throw new IllegalStateException("Malformed WAL marker: " + markerData); + } + return upgraded; + } + + @VisibleForTesting + void upgradeWalMarkers(ServerContext context) { + var zrw = context.getZooSession().asReaderWriter(); + try { + String walRoot = WalStateManager.ZWALS; + if (!zrw.exists(walRoot)) { + return; + } + for (String server : zrw.getChildren(walRoot)) { + String serverPath = walRoot + "/" + server; + for (String marker : zrw.getChildren(serverPath)) { + String markerPath = serverPath + "/" + marker; + byte[] markerData = zrw.getData(markerPath); + String data = new String(markerData, UTF_8); + String upgraded = upgradeWalMarker(data); + if (!upgraded.equals(data)) { + zrw.putPersistentData(markerPath, upgraded.getBytes(UTF_8), + ZooUtil.NodeExistsPolicy.OVERWRITE); + String persisted = new String(zrw.getData(markerPath), UTF_8); + if (!persisted.equals(upgraded)) { + throw new IllegalStateException("Failed to upgrade WAL marker " + markerPath + + ": expected " + upgraded + " but found " + persisted); + } + } + } + } + } catch (KeeperException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new IllegalStateException("Error upgrading WAL provider IDs in ZooKeeper", e); + } + } + @VisibleForTesting void updateRootTabletFileReferences(ServerContext context) { try { @@ -957,6 +1044,19 @@ private void upgradeTabletsMetadata(@NonNull ServerContext context, String metaN LOG.warn("Constraint violations: {}", mex.getConstraintViolationSummaries()); throw new IllegalStateException("Failed to process table: " + metaName, mex); } + verifyNoLegacyWalReferences(context, metaName); + } + + private void verifyNoLegacyWalReferences(ServerContext context, String metaName) { + try (Scanner scanner = context.createScanner(metaName, Authorizations.EMPTY)) { + scanner.fetchColumnFamily(LogColumnFamily.NAME); + scanner.setRange(MetadataSchema.TabletsSection.getRange()); + for (Entry entry : scanner) { + verifyNoLegacyWalReference(entry.getKey(), metaName); + } + } catch (TableNotFoundException e) { + throw new IllegalStateException("Failed to verify WAL references in " + metaName, e); + } } @VisibleForTesting diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java index 002ffbeea80..0331873520e 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java @@ -92,6 +92,7 @@ import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.gc.AllVolumesDirectory; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.metadata.ConditionalTabletMutatorImpl; import org.apache.accumulo.server.metadata.ConditionalTabletsMutatorImpl; import org.apache.hadoop.io.Text; @@ -330,7 +331,8 @@ public void testUnexpectedColumns() { testUnexpectedColumn(tmb -> tmb.putOperation(otherOpid), "had unexpected opid", otherOpid.toString()); - var walog = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + var walog = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); testUnexpectedColumn(tmb -> tmb.putWal(walog), "has unexpected walogs 1"); FateId ucfid1 = otherFateId; diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java index 13ad3bde70e..b257c3be093 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java @@ -72,6 +72,7 @@ import org.apache.accumulo.manager.split.FileRangeCache; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.metadata.ConditionalTabletMutatorImpl; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -466,7 +467,8 @@ public void testErrors() throws Exception { assertTrue(e.getMessage().contains(opid2.toString())); // Test splitting a tablet with walogs - var walog = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + var walog = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); var tablet4 = TabletMetadata.builder(origExtent).putOperation(opid).putWal(walog) .build(ColumnType.LOCATION); e = assertThrows(IllegalStateException.class, () -> testError(origExtent, tablet4, fateId)); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java index f1f1541eb89..c1f1a773372 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java @@ -72,6 +72,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LastLocationColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily; import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; import org.apache.accumulo.core.zookeeper.ZooSession; @@ -82,6 +83,8 @@ import org.apache.accumulo.server.conf.store.SystemPropKey; import org.apache.accumulo.server.conf.store.TablePropKey; import org.apache.accumulo.server.conf.store.impl.ZooPropStore; +import org.apache.accumulo.server.log.DfsWalProvider; +import org.apache.accumulo.server.log.WalStateManager; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.zookeeper.KeeperException; @@ -189,6 +192,80 @@ public void processReferencesTest() { } + @Test + void upgradeWalReferenceTest() { + String path = "hdfs://nn/accumulo/wal/host+1234/11111111-1111-1111-1111-111111111111"; + Key key = Key.builder().row("123").family(LogColumnFamily.NAME).qualifier("-/" + path).build(); + Value value = new Value(); + ArrayList mutations = new ArrayList<>(); + new Upgrader11to12().processReferences(mutations::add, List.of(Map.entry(key, value)), + "accumulo.metadata"); + + assertTrue(UPGRADE_FAMILIES.contains(LogColumnFamily.NAME)); + assertEquals(1, mutations.size()); + Mutation mutation = mutations.get(0); + var updates = mutation.getUpdates(); + assertEquals(2, updates.size()); + var added = updates.stream().filter(update -> !update.isDeleted()).findFirst().orElseThrow(); + var deleted = updates.stream().filter(ColumnUpdate::isDeleted).findFirst().orElseThrow(); + assertEquals(DfsWalProvider.PROVIDER_ID + "/" + path, + new Text(added.getColumnQualifier()).toString()); + assertEquals("-/" + path, new Text(deleted.getColumnQualifier()).toString()); + + Key upgradedKey = Key.builder().row("123").family(LogColumnFamily.NAME) + .qualifier(DfsWalProvider.PROVIDER_ID + "/" + path).build(); + ArrayList retryMutations = new ArrayList<>(); + new Upgrader11to12().processReferences(retryMutations::add, + List.of(Map.entry(upgradedKey, value)), "accumulo.metadata"); + assertTrue(retryMutations.isEmpty()); + } + + @Test + void upgradeWalMarkerIsIdempotentAndRejectsInvalidProviderIds() { + String oldData = "CLOSED,hdfs://nn/accumulo/wal/host+1234/11111111-1111-1111-1111-111111111111"; + String upgraded = "CLOSED," + DfsWalProvider.PROVIDER_ID + + ",hdfs://nn/accumulo/wal/host+1234/11111111-1111-1111-1111-111111111111"; + assertEquals(upgraded, Upgrader11to12.upgradeWalMarker(oldData)); + assertEquals(upgraded, Upgrader11to12.upgradeWalMarker(upgraded)); + assertThrows(IllegalStateException.class, + () -> Upgrader11to12.upgradeWalMarker("CLOSED,provider-id,path")); + assertThrows(IllegalStateException.class, + () -> Upgrader11to12.upgradeWalMarker("CLOSED,-,path")); + } + + @Test + void upgradeWalMarkersTest() throws Exception { + ServerContext context = createMock(ServerContext.class); + ZooSession zk = createMock(ZooSession.class); + ZooReaderWriter zrw = createStrictMock(ZooReaderWriter.class); + String server = "localhost:9997[1]"; + String uuid = "11111111-1111-1111-1111-111111111111"; + String markerPath = WalStateManager.ZWALS + "/" + server + "/" + uuid; + String oldData = "CLOSED,hdfs://nn/accumulo/wal/host+1234/" + uuid; + String newData = "OPEN," + DfsWalProvider.PROVIDER_ID + + ",hdfs://nn/accumulo/wal/host+1234/22222222-2222-2222-2222-222222222222"; + + expect(context.getZooSession()).andReturn(zk); + expect(zk.asReaderWriter()).andReturn(zrw); + expect(zrw.exists(WalStateManager.ZWALS)).andReturn(true); + expect(zrw.getChildren(WalStateManager.ZWALS)).andReturn(List.of(server)); + expect(zrw.getChildren(WalStateManager.ZWALS + "/" + server)).andReturn(List.of(uuid, "222")); + expect(zrw.getData(markerPath)).andReturn(oldData.getBytes(UTF_8)); + expect(zrw.putPersistentData(markerPath, + ("CLOSED," + DfsWalProvider.PROVIDER_ID + "," + oldData.substring("CLOSED,".length())) + .getBytes(UTF_8), + ZooUtil.NodeExistsPolicy.OVERWRITE)).andReturn(true); + expect(zrw.getData(markerPath)).andReturn( + ("CLOSED," + DfsWalProvider.PROVIDER_ID + "," + oldData.substring("CLOSED,".length())) + .getBytes(UTF_8)); + expect(zrw.getData(WalStateManager.ZWALS + "/" + server + "/222")) + .andReturn(newData.getBytes(UTF_8)); + + replay(context, zk, zrw); + new Upgrader11to12().upgradeWalMarkers(context); + verify(context, zk, zrw); + } + @Test public void skipConvertedFileTest() { // create sample data "served" by the mocked scanner diff --git a/server/manager/src/test/resources/accumulo.properties b/server/manager/src/test/resources/accumulo.properties index 7a9d5c90663..1669b68ee68 100644 --- a/server/manager/src/test/resources/accumulo.properties +++ b/server/manager/src/test/resources/accumulo.properties @@ -18,3 +18,5 @@ # # File needs to exist for unit tests +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java index 0d3363c302c..0af8a0e1c15 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java @@ -215,6 +215,7 @@ private TabletMetadataLoader(Ample ample) { public ScanServer(ServerOpts opts, String[] args) { super(ServerId.Type.SCAN_SERVER, opts, ServerContext::new, args); context = super.getContext(); + context.getWalProviderRegistry(); LOG.info("Version " + Constants.VERSION); LOG.info("Instance " + getContext().getInstanceID()); this.sessionManager = new SessionManager(context); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index 1402c22865a..4a45714edf5 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -227,6 +227,7 @@ protected TabletServer(ServerOpts opts, String[] args) { super(ServerId.Type.TABLET_SERVER, opts, serverContextFactory, args); context = super.getContext(); + context.getWalProviderRegistry(); final AccumuloConfiguration aconf = getConfiguration(); log.info("Version " + Constants.VERSION); log.info("Instance " + getInstanceID()); @@ -1030,7 +1031,7 @@ private void markUnusedWALs() { TServerInstance session = this.getTabletSession(); for (DfsLogger candidate : eligible) { log.info("Marking " + candidate.getPath() + " as unreferenced"); - walMarker.walUnreferenced(session, candidate.getPath()); + walMarker.walUnreferenced(session, candidate.getLogEntry()); } synchronized (closedLogs) { closedLogs.removeAll(eligible); @@ -1042,7 +1043,7 @@ private void markUnusedWALs() { public void addNewLogMarker(DfsLogger copy) throws WalMarkerException { log.info("Writing log marker for " + copy.getPath()); - walMarker.addNewWalMarker(getTabletSession(), copy.getPath()); + walMarker.addNewWalMarker(getTabletSession(), copy.getLogEntry()); } public void walogClosed(DfsLogger currentLog) throws WalMarkerException { @@ -1054,7 +1055,7 @@ public void walogClosed(DfsLogger currentLog) throws WalMarkerException { clSize = closedLogs.size(); } log.info("Marking {} as closed. Total closed logs: {} ", currentLog.getPath(), clSize); - walMarker.closeWal(getTabletSession(), currentLog.getPath()); + walMarker.closeWal(getTabletSession(), currentLog.getLogEntry()); // whenever a new log is added to the set of closed logs, go through all of the tablets and // see if any need to minor compact @@ -1072,7 +1073,7 @@ public void walogClosed(DfsLogger currentLog) throws WalMarkerException { } } else { log.info("Marking {} as unreferenced (skipping closed writes == 0)", currentLog.getPath()); - walMarker.walUnreferenced(getTabletSession(), currentLog.getPath()); + walMarker.walUnreferenced(getTabletSession(), currentLog.getLogEntry()); } } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java index d021ff111bf..f0cd572378f 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java @@ -32,6 +32,7 @@ import java.io.DataOutputStream; import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.channels.ClosedChannelException; import java.time.Duration; @@ -46,7 +47,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Durability; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; @@ -59,23 +59,17 @@ import org.apache.accumulo.core.spi.crypto.CryptoService; import org.apache.accumulo.core.spi.crypto.FileDecrypter; import org.apache.accumulo.core.spi.crypto.FileEncrypter; -import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment; +import org.apache.accumulo.core.spi.wal.WalProvider; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.core.util.Timer; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.server.ServerContext; -import org.apache.accumulo.server.fs.VolumeChooserEnvironmentImpl; -import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.tserver.TabletMutations; import org.apache.accumulo.tserver.logger.LogFileKey; import org.apache.accumulo.tserver.logger.LogFileValue; import org.apache.accumulo.tserver.tablet.CommitSession; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSOutputStream; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,7 +91,6 @@ public final class DfsLogger implements Comparable { public static final String LOG_FILE_HEADER_V4 = "--- Log File Header (v4) ---"; private static final Logger log = LoggerFactory.getLogger(DfsLogger.class); - private static final DatanodeInfo[] EMPTY_PIPELINE = new DatanodeInfo[0]; public static class LogClosedException extends IOException { private static final long serialVersionUID = 1L; @@ -182,10 +175,10 @@ public void run() { try { if (shouldHSync.isPresent()) { if (shouldHSync.orElseThrow()) { - logFile.hsync(); + logFile.sync(Durability.SYNC); syncCounter.incrementAndGet(); } else { - logFile.hflush(); + logFile.sync(Durability.FLUSH); flushCounter.incrementAndGet(); } } @@ -198,7 +191,7 @@ public void run() { if (expectedReplication > 0) { int current = expectedReplication; try { - current = ((DFSOutputStream) logFile.getWrappedStream()).getCurrentBlockReplication(); + current = logFile.getCurrentReplication(); } catch (IOException e) { fail(work, e, "getting replication level"); } @@ -210,10 +203,9 @@ public void run() { } } } - if (expectedReplication == 0 && logFile.getWrappedStream() instanceof DFSOutputStream) { + if (expectedReplication == 0) { try { - expectedReplication = - ((DFSOutputStream) logFile.getWrappedStream()).getCurrentBlockReplication(); + expectedReplication = logFile.getCurrentReplication(); } catch (IOException e) { fail(work, e, "getting replication level"); } @@ -303,7 +295,7 @@ public int hashCode() { return logEntry.hashCode(); } - private FSDataOutputStream logFile; + private WalProvider.Writer logFile; private DataOutputStream encryptingLogFile = null; private final LogEntry logEntry; private Thread syncThread; @@ -313,24 +305,30 @@ public int hashCode() { /** * Create a new DfsLogger with the provided characteristics. */ - public static DfsLogger createNew(ServerContext context, AtomicLong syncCounter, - AtomicLong flushCounter, String address) throws IOException { - - String filename = UUID.randomUUID().toString(); - String addressForFilename = address.replace(':', '+'); - - var chooserEnv = - new VolumeChooserEnvironmentImpl(VolumeChooserEnvironment.Scope.LOGGER, context); - String logPath = - context.getVolumeManager().choose(chooserEnv, context.getBaseUris()) + Path.SEPARATOR - + Constants.WAL_DIR + Path.SEPARATOR + addressForFilename + Path.SEPARATOR + filename; - - LogEntry log = LogEntry.fromPath(logPath); - DfsLogger dfsLogger = new DfsLogger(log); - long slowFlushMillis = - context.getConfiguration().getTimeInMillis(Property.TSERV_SLOW_FLUSH_MILLIS); - dfsLogger.open(context, logPath, filename, address, syncCounter, flushCounter, slowFlushMillis); - return dfsLogger; + public static DfsLogger createNew(ServerContext context, WalProvider provider, + AtomicLong syncCounter, AtomicLong flushCounter, String address) throws IOException { + + WalProvider.Writer writer = provider.create(address, UUID.randomUUID()); + DfsLogger dfsLogger = null; + try { + LogEntry log = LogEntry.fromPath(provider.getId(), writer.getReference()); + dfsLogger = new DfsLogger(log); + long slowFlushMillis = + context.getConfiguration().getTimeInMillis(Property.TSERV_SLOW_FLUSH_MILLIS); + dfsLogger.open(context, writer, address, syncCounter, flushCounter, slowFlushMillis); + return dfsLogger; + } catch (IOException | RuntimeException e) { + try { + if (dfsLogger != null && dfsLogger.syncThread != null) { + dfsLogger.close(); + } else { + writer.close(); + } + } catch (IOException closeException) { + e.addSuppressed(closeException); + } + throw e; + } } /** @@ -353,33 +351,35 @@ private DfsLogger(LogEntry logEntry) { * @throws LogHeaderIncompleteException if the header cannot be fully read (can happen if the * tserver died before finishing) */ - public static DataInputStream getDecryptingStream(FSDataInputStream input, - CryptoService cryptoService) throws LogHeaderIncompleteException, IOException { + public static DataInputStream getDecryptingStream(InputStream input, CryptoService cryptoService) + throws LogHeaderIncompleteException, IOException { DataInputStream decryptingInput; + DataInputStream dataInput = + input instanceof DataInputStream dis ? dis : new DataInputStream(input); byte[] magic4 = DfsLogger.LOG_FILE_HEADER_V4.getBytes(UTF_8); byte[] magic3 = DfsLogger.LOG_FILE_HEADER_V3.getBytes(UTF_8); byte[] magicBuffer = new byte[magic4.length]; try { - input.readFully(magicBuffer); + dataInput.readFully(magicBuffer); if (Arrays.equals(magicBuffer, magic4)) { FileDecrypter decrypter = - CryptoUtils.getFileDecrypter(cryptoService, Scope.WAL, null, input); + CryptoUtils.getFileDecrypter(cryptoService, Scope.WAL, null, dataInput); log.debug("Using {} for decrypting WAL", cryptoService.getClass().getSimpleName()); - var stream = decrypter.decryptStream(input); + var stream = decrypter.decryptStream(dataInput); decryptingInput = stream instanceof DataInputStream ? (DataInputStream) stream : new DataInputStream(stream); } else if (Arrays.equals(magicBuffer, magic3)) { // Read logs files from Accumulo 1.9 and throw an error if they are encrypted - String cryptoModuleClassname = input.readUTF(); + String cryptoModuleClassname = dataInput.readUTF(); if (!cryptoModuleClassname.equals("NullCryptoModule")) { throw new IllegalArgumentException( "Old encryption modules not supported at this time. Unsupported module : " + cryptoModuleClassname); } - decryptingInput = input; + decryptingInput = dataInput; } else { throw new IllegalArgumentException( "Unsupported write ahead log version " + new String(magicBuffer, UTF_8)); @@ -401,62 +401,35 @@ public static DataInputStream getDecryptingStream(FSDataInputStream input, * * @param address The address of the host using this WAL */ - private synchronized void open(ServerContext context, String logPath, String filename, - String address, AtomicLong syncCounter, AtomicLong flushCounter, long slowFlushMillis) - throws IOException { + private synchronized void open(ServerContext context, WalProvider.Writer writer, String address, + AtomicLong syncCounter, AtomicLong flushCounter, long slowFlushMillis) throws IOException { log.debug("Address is {}", address); log.debug("DfsLogger.open() begin"); - VolumeManager fs = context.getVolumeManager(); - LoggerOperation op; - var serverConf = context.getConfiguration(); try { - Path logfilePath = new Path(logPath); - short replication = (short) serverConf.getCount(Property.TSERV_WAL_REPLICATION); - if (replication == 0) { - replication = fs.getDefaultReplication(logfilePath); - } - long blockSize = getWalBlockSize(serverConf); - if (serverConf.getBoolean(Property.TSERV_WAL_SYNC)) { - logFile = fs.createSyncable(logfilePath, 0, replication, blockSize); - } else { - logFile = fs.create(logfilePath, true, 0, replication, blockSize); - } - - // Tell the DataNode that the write ahead log does not need to be cached in the OS page cache - try { - logFile.setDropBehind(Boolean.TRUE); - } catch (UnsupportedOperationException e) { - log.debug("setDropBehind writes not enabled for wal file: {}", logFile); - } catch (IOException e) { - log.debug("IOException setting drop behind for file: {}, msg: {}", logFile, e.getMessage()); - } - - // check again that logfile can be sync'd - if (!fs.canSyncAndFlush(logfilePath)) { - log.warn("sync not supported for log file {}. Data loss may occur.", logPath); - } + logFile = writer; + OutputStream output = logFile.getOutputStream(); // Initialize the log file with a header and its encryption CryptoEnvironment env = new CryptoEnvironmentImpl(Scope.WAL); - CryptoService cryptoService = - context.getCryptoFactory().getService(env, serverConf.getAllCryptoProperties()); - logFile.write(LOG_FILE_HEADER_V4.getBytes(UTF_8)); + CryptoService cryptoService = context.getCryptoFactory().getService(env, + context.getConfiguration().getAllCryptoProperties()); + output.write(LOG_FILE_HEADER_V4.getBytes(UTF_8)); log.debug("Using {} for encrypting WAL {}", cryptoService.getClass().getSimpleName(), - filename); + writer.getReference()); FileEncrypter encrypter = cryptoService.getFileEncrypter(env); byte[] cryptoParams = encrypter.getDecryptionParameters(); - CryptoUtils.writeParams(cryptoParams, logFile); + CryptoUtils.writeParams(cryptoParams, new DataOutputStream(output)); /* * Always wrap the WAL in a NoFlushOutputStream to prevent extra flushing to HDFS. The method * write(LogFileKey, LogFileValue) will flush crypto data or do nothing when crypto is not * enabled. */ - OutputStream encryptedStream = encrypter.encryptStream(new NoFlushOutputStream(logFile)); + OutputStream encryptedStream = encrypter.encryptStream(new NoFlushOutputStream(output)); if (encryptedStream instanceof NoFlushOutputStream) { encryptingLogFile = (NoFlushOutputStream) encryptedStream; } else { @@ -465,8 +438,8 @@ private synchronized void open(ServerContext context, String logPath, String fil LogFileKey key = new LogFileKey(); key.setEvent(OPEN); - key.setTserverSession(filename); - key.setFilename(filename); + key.setTserverSession(logEntry.getUniqueID().toString()); + key.setFilename(logEntry.getUniqueID().toString()); op = logKeyData(key, Durability.SYNC); } catch (Exception ex) { if (logFile != null) { @@ -551,9 +524,9 @@ public synchronized long getWrites() { return writes; } - public synchronized long getBytesWritten() { + public synchronized long getBytesWritten() throws IOException { Objects.requireNonNull(logFile, "logFile is null"); - return logFile.getPos(); + return logFile.getLength(); } public LoggerOperation defineTablet(CommitSession cs) throws IOException { @@ -677,18 +650,10 @@ public int compareTo(DfsLogger o) { /** * This method gets the pipeline for the current walog. * - * @return non-null array of DatanodeInfo + * @return diagnostic storage pipeline */ - DatanodeInfo[] getPipeLine() { - if (logFile != null) { - OutputStream os = logFile.getWrappedStream(); - if (os instanceof DFSOutputStream) { - return ((DFSOutputStream) os).getPipeline(); - } - } - - // Don't have a pipeline or can't figure it out. - return EMPTY_PIPELINE; + String[] getPipeLine() { + return logFile == null ? new String[0] : logFile.getPipeline(); } } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java index 93a23b52e4d..abe80ddc95b 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java @@ -27,6 +27,7 @@ import java.io.DataInputStream; import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -51,18 +52,19 @@ import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.spi.crypto.CryptoEnvironment; import org.apache.accumulo.core.spi.crypto.CryptoService; +import org.apache.accumulo.core.util.CountingInputStream; import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.server.AbstractServer; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.SortedLogState; import org.apache.accumulo.server.zookeeper.DistributedWorkQueue; import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor; import org.apache.accumulo.tserver.log.DfsLogger.LogHeaderIncompleteException; import org.apache.accumulo.tserver.logger.LogFileKey; import org.apache.accumulo.tserver.logger.LogFileValue; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.zookeeper.KeeperException; @@ -84,7 +86,7 @@ public class LogSorter implements MetricsProducer { class LogProcessor implements Processor { - private FSDataInputStream input; + private CountingInputStream input; private DataInputStream decryptingInput; private long bytesCopied = -1; private long sortStart = 0; @@ -99,8 +101,9 @@ public Processor newProcessor() { public void process(String child, byte[] data) { String work = new String(data, UTF_8); String[] parts = work.split("\\|"); - String src = parts[0]; - String dest = parts[1]; + String providerId = parts.length == 2 ? DfsWalProvider.PROVIDER_ID : parts[0]; + String src = parts.length == 2 ? parts[0] : parts[1]; + String dest = parts.length == 2 ? parts[1] : parts[2]; String sortId = new Path(src).getName(); log.debug("Sorting {} to {} using sortId {}", src, dest, sortId); @@ -119,7 +122,7 @@ public void process(String child, byte[] data) { String formerThreadName = Thread.currentThread().getName(); try { - sort(fs, sortId, new Path(src), dest); + sort(fs, sortId, providerId, new Path(src), dest); } catch (Exception t) { try { // parent dir may not exist @@ -145,6 +148,11 @@ public void process(String child, byte[] data) { public void sort(VolumeManager fs, String name, Path srcPath, String destPath) throws IOException { + sort(fs, name, DfsWalProvider.PROVIDER_ID, srcPath, destPath); + } + + public void sort(VolumeManager fs, String name, String providerId, Path srcPath, + String destPath) throws IOException { int part = 0; // check for finished first since another thread may have already done the sort @@ -157,16 +165,9 @@ public void sort(VolumeManager fs, String name, Path srcPath, String destPath) // the following call does not throw an exception if the file/dir does not exist fs.deleteRecursively(new Path(destPath)); - input = fs.open(srcPath); - - // Tell the DataNode that the write ahead log does not need to be cached in the OS page cache - try { - input.setDropBehind(Boolean.TRUE); - } catch (UnsupportedOperationException e) { - log.debug("setDropBehind reads not enabled for wal file: {}", input); - } catch (IOException e) { - log.debug("IOException setting drop behind for file: {}, msg: {}", input, e.getMessage()); - } + InputStream source = providerId.equals(DfsWalProvider.PROVIDER_ID) ? fs.open(srcPath) + : context.getWalProviderRegistry().get(providerId).open(srcPath.toString()); + input = new CountingInputStream(source); try { decryptingInput = DfsLogger.getDecryptingStream(input, cryptoService); @@ -187,8 +188,8 @@ public void sort(VolumeManager fs, String name, Path srcPath, String destPath) while (true) { final ArrayList> buffer = new ArrayList<>(512); try { - long start = input.getPos(); - while (input.getPos() - start < bufferSize) { + long start = input.getCount(); + while (input.getCount() - start < bufferSize) { LogFileKey key = new LogFileKey(); LogFileValue value = new LogFileValue(); key.readFields(decryptingInput); @@ -211,7 +212,7 @@ synchronized void close() throws IOException { // If we receive an empty or malformed-header WAL, we won't // have input streams that need closing. Avoid the NPE. if (input != null) { - bytesCopied = input.getPos(); + bytesCopied = input.getCount(); input.close(); if (decryptingInput != null) { decryptingInput.close(); @@ -231,7 +232,7 @@ public synchronized long getSortTime() { } synchronized long getBytesCopied() throws IOException { - return input == null ? bytesCopied : input.getPos(); + return input == null ? bytesCopied : input.getCount(); } } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java index 6843538510b..9f511a383e0 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java @@ -275,7 +275,8 @@ private synchronized void startLogMaker() { final DfsLogger alog; try { - alog = DfsLogger.createNew(tserver.getContext(), syncCounter, flushCounter, + alog = DfsLogger.createNew(tserver.getContext(), + tserver.getContext().getWalProviderRegistry().getCurrent(), syncCounter, flushCounter, tserver.getAdvertiseAddress().toString()); } catch (IOException | RuntimeException e) { log.error("Failed to open WAL", e); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java index c41455850ff..1056c586ad3 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java @@ -45,6 +45,7 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.util.ServerKeywordExecutable; import org.apache.accumulo.start.spi.CommandGroup; import org.apache.accumulo.start.spi.CommandGroups; @@ -175,7 +176,8 @@ public void execute(JCommander cl, ReaderOpts opts) throws Exception { } else { // read the log entries in a sorted RFile. This has to be a directory that contains the // finished file. - var rsl = ResolvedSortedLog.resolve(LogEntry.fromPath(path.toString()), fs); + var rsl = ResolvedSortedLog + .resolve(LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, path.toString()), fs); try (var rli = new RecoveryLogsIterator(context, Collections.singletonList(rsl), null, null, false)) { while (rli.hasNext()) { diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java index 2e682b4231a..80c35416552 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.tserver.log.DfsLogger; import org.junit.jupiter.api.Test; @@ -34,7 +35,8 @@ public class WalRemovalOrderTest { private DfsLogger mockLogger(String filename) { - var mockLogEntry = LogEntry.fromPath(filename + "+1234/11111111-1111-1111-1111-111111111111"); + var mockLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, + filename + "+1234/11111111-1111-1111-1111-111111111111"); return DfsLogger.fromLogEntry(mockLogEntry); } diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java index 76516fbed84..618256dd723 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java @@ -48,6 +48,7 @@ import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.fs.VolumeManagerImpl; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.SortedLogState; import org.apache.accumulo.tserver.TabletServer; import org.apache.accumulo.tserver.WithTestNames; @@ -252,7 +253,8 @@ private void createRecoveryDir(Map logs, ArrayList recover(Map logs, Set files, K logSorter.writeBuffer(destPath, buffer, parts); ns.create(SortedLogState.getFinishedMarkerPath(destPath)).close(); - dirs.add(ResolvedSortedLog.resolve(LogEntry.fromPath(origPath), fs)); + dirs.add( + ResolvedSortedLog.resolve(LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, origPath), fs)); } // Recover SortedLogRecovery recovery = new SortedLogRecovery(context, fileLenCache, cacheProvider); diff --git a/server/tserver/src/test/resources/accumulo.properties b/server/tserver/src/test/resources/accumulo.properties index b63ed5c0617..2a7b4af5260 100644 --- a/server/tserver/src/test/resources/accumulo.properties +++ b/server/tserver/src/test/resources/accumulo.properties @@ -21,3 +21,5 @@ instance.dfs.dir=${project.build.directory}/instanceTest instance.secret=TEST_SYSTEM_SECRET +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider diff --git a/shell/pom.xml b/shell/pom.xml index b413fdcb4c3..08294cf7a4d 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -91,6 +91,11 @@ hadoop-client-runtime runtime + + org.apache.accumulo + accumulo-server-base + test + org.apache.logging.log4j log4j-slf4j2-impl diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java index a12c23e3c5a..48f4b3c93f2 100644 --- a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java +++ b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java @@ -48,6 +48,7 @@ import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.shell.Shell; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -191,8 +192,10 @@ public void mockTest() throws Exception { KeyExtent extent = new KeyExtent(tableId, new Text("d"), null); - LogEntry le1 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); - LogEntry le2 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); TabletMetadata tm1 = TabletMetadata.builder(extent).putTabletAvailability(TabletAvailability.ONDEMAND) diff --git a/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java index e33abf148bf..c4f01f77bef 100644 --- a/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java @@ -41,6 +41,7 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.accumulo.tserver.log.DfsLogger; import org.apache.hadoop.conf.Configuration; @@ -136,7 +137,8 @@ public void testEmptyWalRecoveryCompletes() throws Exception { TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName)); assertNotNull(tableId, "Table ID was null"); - LogEntry logEntry = LogEntry.fromPath(emptyWalog.toUri().toString()); + LogEntry logEntry = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, emptyWalog.toUri().toString()); log.info("Taking {} offline", tableName); client.tableOperations().offline(tableName, true); @@ -196,7 +198,8 @@ public void testPartialHeaderWalRecoveryCompletes() throws Exception { TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName)); assertNotNull(tableId, "Table ID was null"); - LogEntry logEntry = LogEntry.fromPath(partialHeaderWalog.toUri().toString()); + LogEntry logEntry = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, partialHeaderWalog.toUri().toString()); log.info("Taking {} offline", tableName); client.tableOperations().offline(tableName, true); diff --git a/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java b/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java index 6f39b63be11..d238d3dab7d 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java @@ -90,6 +90,7 @@ import org.apache.accumulo.manager.tableOps.split.PreSplit; import org.apache.accumulo.manager.tableOps.split.SplitInfo; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.test.LargeSplitRowIT; import org.apache.accumulo.test.ample.metadata.TestAmple; import org.apache.accumulo.test.ample.metadata.TestAmple.TestServerAmpleImpl; @@ -161,7 +162,8 @@ public void testNoWalsMergeRepos(MergeInfo.Operation operation) throws Exception // Write a WAL to the test metadata and then re-run the repo to check for an error try (TabletsMutator tm = testAmple.mutateTablets()) { var walFilePath = Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - tm.mutateTablet(extent).putWal(LogEntry.fromPath(walFilePath)).mutate(); + tm.mutateTablet(extent).putWal(LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath)) + .mutate(); } // Should not be ready due to the presence of a WAL diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java index 713eabdf82b..8ae7c688ee2 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java @@ -105,6 +105,7 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.metadata.AsyncConditionalTabletsMutatorImpl; import org.apache.accumulo.test.harness.SharedMiniClusterBase; import org.apache.accumulo.test.util.Wait; @@ -473,7 +474,7 @@ public void testWALs() { // Test adding a WAL to a tablet and verifying its presence String walFilePath = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - final LogEntry originalLogEntry = LogEntry.fromPath(walFilePath); + final LogEntry originalLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); // create a tablet metadata with no write ahead logs var tmEmptySet = TabletMetadata.builder(e1).build(LOGS); // tablet should not have any logs to start with so requireSame with the empty logs should pass @@ -491,7 +492,7 @@ public void testWALs() { // Test adding another WAL and verifying the update String walFilePath2 = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - LogEntry newLogEntry = LogEntry.fromPath(walFilePath2); + LogEntry newLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath2); try (var ctmi = context.getAmple().conditionallyMutateTablets()) { ctmi.mutateTablet(e1).requireAbsentOperation().putWal(newLogEntry).submit(tm -> false); assertEquals(Status.ACCEPTED, ctmi.process().get(e1).getStatus()); @@ -504,7 +505,7 @@ public void testWALs() { String walFilePath3 = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - LogEntry otherLogEntry = LogEntry.fromPath(walFilePath3); + LogEntry otherLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath3); // create a powerset to ensure all possible subsets fail when using requireSame except the // expected current state @@ -1236,7 +1237,7 @@ public void multipleFilters() { String server = "server1+8555"; String walFilePath = java.nio.file.Path.of(server, UUID.randomUUID().toString()).toString(); - LogEntry wal = LogEntry.fromPath(walFilePath); + LogEntry wal = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); // add wal compact and flush ID to these tablets final Set tabletsWithWalCompactFlush = Set.of(e1, e2, e3); @@ -1340,7 +1341,7 @@ public void walFilter() { // add a wal to e2 var walFilePath = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - var wal = LogEntry.fromPath(walFilePath); + var wal = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); try (var ctmi = context.getAmple().conditionallyMutateTablets()) { ctmi.mutateTablet(e2).requireAbsentOperation().putWal(wal).submit(tabletMetadata -> false); assertEquals(Status.ACCEPTED, ctmi.process().get(e2).getStatus()); @@ -1351,7 +1352,7 @@ public void walFilter() { // add wal to tablet e4 walFilePath = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - wal = LogEntry.fromPath(walFilePath); + wal = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); try (var ctmi = context.getAmple().conditionallyMutateTablets()) { ctmi.mutateTablet(e4).requireAbsentOperation().putWal(wal).submit(tabletMetadata -> false); assertEquals(Status.ACCEPTED, ctmi.process().get(e4).getStatus()); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/InMemoryWalProviderIT.java b/test/src/main/java/org/apache/accumulo/test/functional/InMemoryWalProviderIT.java new file mode 100644 index 00000000000..35ac1688a04 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/functional/InMemoryWalProviderIT.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.functional; + +import static org.apache.accumulo.minicluster.ServerType.GARBAGE_COLLECTOR; +import static org.apache.accumulo.minicluster.ServerType.TABLET_SERVER; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.metadata.SystemTables; +import org.apache.accumulo.core.spi.wal.WalProvider; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.server.log.DfsWalProvider; +import org.apache.accumulo.server.log.WalStateManager; +import org.apache.accumulo.server.log.WalStateManager.WalStatePath; +import org.apache.accumulo.test.TestIngest; +import org.apache.accumulo.test.VerifyIngest; +import org.apache.accumulo.test.VerifyIngest.VerifyParams; +import org.apache.accumulo.test.util.Wait; +import org.apache.accumulo.test.wal.InMemoryWalProvider; +import org.apache.accumulo.test.wal.InMemoryWalStore; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +public class InMemoryWalProviderIT extends ConfigurableMacBase { + + private InMemoryWalStore walStore; + + @Override + protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { + try { + if (walStore == null) { + walStore = InMemoryWalStore.start(); + } + } catch (IOException e) { + throw new IllegalStateException("Unable to start in-memory WAL store", e); + } + + var options = walStore.getProviderOptions(); + String providerClass = InMemoryWalProvider.class.getName(); + String providerConfig = "{\"" + DfsWalProvider.class.getName() + "\":{},\"" + providerClass + + "\":{\"host\":\"" + options.get(InMemoryWalProvider.HOST_OPTION) + "\",\"port\":\"" + + options.get(InMemoryWalProvider.PORT_OPTION) + "\"}}"; + cfg.setProperty(Property.INSTANCE_WAL_PROVIDER, providerClass); + cfg.setProperty(Property.INSTANCE_WAL_PROVIDERS, providerConfig); + cfg.setProperty(Property.TSERV_WAL_MAX_SIZE, "1M"); + cfg.setProperty(Property.GC_CYCLE_START, "0s"); + cfg.setProperty(Property.GC_CYCLE_DELAY, "1s"); + cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s"); + cfg.setProperty(Property.MANAGER_RECOVERY_DELAY, "1s"); + cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1); + hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); + } + + @AfterEach + @Override + public void tearDown() { + try { + super.tearDown(); + } finally { + if (walStore != null) { + walStore.close(); + walStore = null; + } + } + } + + @Test + public void recoversAndGarbageCollectsWalThroughProvider() throws Exception { + var control = getCluster().getClusterControl(); + control.stop(GARBAGE_COLLECTOR); + try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) { + String tableName = getUniqueNames(1)[0]; + client.tableOperations().create(tableName); + + VerifyParams params = new VerifyParams(getClientProperties(), tableName); + params.rows = 2_000; + params.dataSize = 1_024; + TestIngest.ingest(client, params); + VerifyIngest.verifyIngest(client, params); + + Set walStates = new WalStateManager(getServerContext()).getAllState(); + var provider = + getServerContext().getWalProviderRegistry().get(InMemoryWalProvider.PROVIDER_ID); + assertTrue(getServerContext().getWalProviderRegistry().getIds() + .containsAll(Set.of(DfsWalProvider.PROVIDER_ID, InMemoryWalProvider.PROVIDER_ID))); + Set walReferences = walStates.stream() + .filter(state -> state.providerId().equals(InMemoryWalProvider.PROVIDER_ID)) + .map(state -> state.path().toString()).collect(Collectors.toSet()); + assertFalse(walReferences.isEmpty(), "SPI-created WALs should be present"); + for (String reference : walReferences) { + assertTrue(provider.exists(reference), "SPI-created WAL should exist: " + reference); + } + + control.stopAllServers(TABLET_SERVER); + control.startAllServers(TABLET_SERVER); + VerifyIngest.verifyIngest(client, params); + client.tableOperations().flush(tableName, null, null, true); + for (String systemTable : new String[] {SystemTables.METADATA.tableName(), + SystemTables.ROOT.tableName(), SystemTables.FATE.tableName()}) { + client.tableOperations().flush(systemTable, null, null, true); + } + + control.start(GARBAGE_COLLECTOR); + Wait.waitFor(() -> anyWalDeleted(provider, walReferences), + "Garbage collector did not delete recovered WALs through their provider"); + } + } + + private static boolean anyWalDeleted(WalProvider provider, Set references) + throws IOException { + for (String reference : references) { + if (!provider.exists(reference)) { + return true; + } + } + return false; + } +} diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java index 8936f6e80bd..5516db2e9b1 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java @@ -99,6 +99,7 @@ import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.manager.LiveTServerSet; import org.apache.accumulo.server.manager.state.TabletManagementIterator; import org.apache.accumulo.server.manager.state.TabletManagementParameters; @@ -672,7 +673,7 @@ private void createLogEntry(AccumuloClient client, String table, String tableNam KeyExtent extent = new KeyExtent(tableIdToModify, new Text("some split"), null); Mutation m = new Mutation(extent.toMetaRow()); String fileName = "file:/accumulo/wal/localhost+9997/" + UUID.randomUUID().toString(); - LogEntry logEntry = LogEntry.fromPath(fileName); + LogEntry logEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, fileName); logEntry.addToMutation(m); try (BatchWriter bw = client.createBatchWriter(table)) { bw.addMutation(m); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/WalProviderSwitchIT.java b/test/src/main/java/org/apache/accumulo/test/functional/WalProviderSwitchIT.java new file mode 100644 index 00000000000..25ba6c75755 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/functional/WalProviderSwitchIT.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.functional; + +import static org.apache.accumulo.core.lock.ServiceLockPaths.AddressSelector.all; +import static org.apache.accumulo.core.lock.ServiceLockPaths.ResourceGroupPredicate.DEFAULT_RG_ONLY; +import static org.apache.accumulo.minicluster.ServerType.GARBAGE_COLLECTOR; +import static org.apache.accumulo.minicluster.ServerType.TABLET_SERVER; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.lock.ServiceLock; +import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath; +import org.apache.accumulo.core.spi.wal.WalProvider; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.miniclusterImpl.ProcessReference; +import org.apache.accumulo.server.log.DfsWalProvider; +import org.apache.accumulo.server.log.WalStateManager; +import org.apache.accumulo.server.log.WalStateManager.WalState; +import org.apache.accumulo.server.log.WalStateManager.WalStatePath; +import org.apache.accumulo.test.TestIngest; +import org.apache.accumulo.test.VerifyIngest; +import org.apache.accumulo.test.VerifyIngest.VerifyParams; +import org.apache.accumulo.test.util.Wait; +import org.apache.accumulo.test.wal.InMemoryWalProvider; +import org.apache.accumulo.test.wal.InMemoryWalStore; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +public class WalProviderSwitchIT extends ConfigurableMacBase { + + private InMemoryWalStore walStore; + + @Override + protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { + try { + if (walStore == null) { + walStore = InMemoryWalStore.start(); + } + } catch (IOException e) { + throw new IllegalStateException("Unable to start in-memory WAL store", e); + } + + var options = walStore.getProviderOptions(); + String memoryProviderClass = InMemoryWalProvider.class.getName(); + String providers = "{\"" + DfsWalProvider.class.getName() + "\":{},\"" + memoryProviderClass + + "\":{\"host\":\"" + options.get(InMemoryWalProvider.HOST_OPTION) + "\",\"port\":\"" + + options.get(InMemoryWalProvider.PORT_OPTION) + "\"}}"; + cfg.setProperty(Property.INSTANCE_WAL_PROVIDER, DfsWalProvider.class.getName()); + cfg.setProperty(Property.INSTANCE_WAL_PROVIDERS, providers); + cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s"); + cfg.setProperty(Property.MANAGER_RECOVERY_DELAY, "1s"); + cfg.setProperty(Property.GC_CYCLE_START, "0s"); + cfg.setProperty(Property.GC_CYCLE_DELAY, "1s"); + cfg.setProperty(Property.TSERV_WAL_MAX_SIZE, "2M"); + cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1); + hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); + } + + @AfterEach + @Override + public void tearDown() { + try { + super.tearDown(); + } finally { + if (walStore != null) { + walStore.close(); + walStore = null; + } + } + } + + @Test + public void recoversDfsWalAfterSwitchingCurrentProvider() throws Exception { + var control = getCluster().getClusterControl(); + control.stop(GARBAGE_COLLECTOR); + + try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) { + String tableName = getUniqueNames(1)[0]; + client.tableOperations().create(tableName); + client.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync"); + + VerifyParams params = new VerifyParams(getClientProperties(), tableName); + params.rows = 2_000; + params.dataSize = 1_024; + TestIngest.ingest(client, params); + VerifyIngest.verifyIngest(client, params); + + Set initialWals = new WalStateManager(getServerContext()).getAllState(); + Set dfsReferences = + initialWals.stream().filter(wal -> wal.providerId().equals(DfsWalProvider.PROVIDER_ID)) + .map(wal -> wal.path().toString()).collect(Collectors.toSet()); + assertFalse(dfsReferences.isEmpty(), "Initial WALs should use the DFS provider"); + + Set tabletServers = + getCluster().getProcesses().get(TABLET_SERVER).stream().collect(Collectors.toSet()); + assertFalse(tabletServers.isEmpty(), "Expected a running TabletServer process"); + ServiceLockPath tabletServerLock = getServerContext().getServerPaths() + .getTabletServer(DEFAULT_RG_ONLY, all(), true).stream().findFirst().orElseThrow(); + ServiceLock.deleteLock(getServerContext().getZooSession().asReaderWriter(), tabletServerLock); + + Wait.waitFor(() -> tabletServers.stream().noneMatch(ref -> ref.getProcess().isAlive()), + "TabletServer did not exit after its ZooKeeper lock was removed"); + + // Preserve the WALs and metadata, but stop all processes before changing the instance-wide + // site property. The WAL provider map continues to include both providers. + getCluster().stop(); + setCurrentWalProvider(InMemoryWalProvider.class.getName()); + getCluster().start(); + + VerifyIngest.verifyIngest(client, params); + + VerifyParams newRows = new VerifyParams(getClientProperties(), tableName); + newRows.startRow = params.rows; + newRows.rows = 1; + newRows.dataSize = params.dataSize; + TestIngest.ingest(client, newRows); + VerifyIngest.verifyIngest(client, newRows); + + WalProvider memoryProvider = + getServerContext().getWalProviderRegistry().get(InMemoryWalProvider.PROVIDER_ID); + Set walsAfterSwitch = new WalStateManager(getServerContext()).getAllState(); + Set memoryReferences = walsAfterSwitch.stream() + .filter(wal -> wal.state() == WalState.OPEN + && wal.providerId().equals(InMemoryWalProvider.PROVIDER_ID)) + .map(wal -> wal.path().toString()).collect(Collectors.toSet()); + assertFalse(memoryReferences.isEmpty(), "New WALs should use the in-memory provider"); + for (String reference : memoryReferences) { + assertTrue(memoryProvider.exists(reference), "In-memory WAL should exist: " + reference); + } + } + } + + private void setCurrentWalProvider(String providerClass) throws IOException { + var config = getCluster().getConfig(); + config.setProperty(Property.INSTANCE_WAL_PROVIDER, providerClass); + Properties siteProperties = new Properties(); + siteProperties.putAll(config.getSiteConfig()); + try (OutputStream output = + Files.newOutputStream(config.getConfDir().toPath().resolve("accumulo.properties"))) { + siteProperties.store(output, "Updated WAL provider for integration test restart"); + } + } +} diff --git a/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java new file mode 100644 index 00000000000..eee1b1c817e --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.wal; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; + +import org.apache.accumulo.core.client.Durability; +import org.apache.accumulo.core.spi.wal.WalProvider; + +/** + * In-memory WAL provider intended for integration tests using MiniAccumuloCluster. WAL bytes are + * held by an {@link InMemoryWalStore} running in the test process and accessed over loopback TCP, + * allowing the cluster's separate server processes to share them. Start the store before starting + * the cluster and pass {@link InMemoryWalStore#getProviderOptions()} as this provider's options. + * References use a local filesystem URI with the standard WAL path layout so Accumulo can place + * sorted recovery logs in the corresponding recovery directory; the WAL bytes themselves remain in + * the in-memory store. Writes are buffered in the provider process and sent to the store on + * {@code sync} or {@code close}; therefore synced bytes remain available for recovery after a + * TabletServer process exits, while unsynced bytes may be lost with that process. + */ +public class InMemoryWalProvider implements WalProvider { + + public static final String PROVIDER_ID = "mem"; + public static final String HOST_OPTION = "host"; + public static final String PORT_OPTION = "port"; + private static final String SCHEME = "file"; + + private String host; + private int port = -1; + + @Override + public String getId() { + return PROVIDER_ID; + } + + @Override + public void init(InitParameters params) { + Map options = params.getOptions(); + host = options.get(HOST_OPTION); + if (host == null || host.isBlank()) { + throw new IllegalArgumentException( + "Missing in-memory WAL store option '" + HOST_OPTION + "'"); + } + String portOption = options.get(PORT_OPTION); + if (portOption == null) { + throw new IllegalArgumentException( + "Missing in-memory WAL store option '" + PORT_OPTION + "'"); + } + try { + port = Integer.parseInt(portOption); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid in-memory WAL store port: " + portOption, e); + } + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("Invalid in-memory WAL store port: " + port); + } + } + + @Override + public Writer create(String serverAddress, UUID id) throws IOException { + requireInitialized(); + String hostForPath = Objects.requireNonNull(serverAddress).replace(':', '+'); + String reference; + try { + reference = + new URI(SCHEME, null, referenceRoot() + hostForPath + "/" + id, null, null).toString(); + } catch (URISyntaxException e) { + throw new IOException("Invalid TabletServer address for WAL reference: " + serverAddress, e); + } + validateReference(reference); + try (var socket = InMemoryWalStore.connect(host, port); + var output = new DataOutputStream(socket.getOutputStream()); + var input = new DataInputStream(socket.getInputStream())) { + InMemoryWalStore.sendRequest(output, InMemoryWalStore.CREATE, reference); + output.flush(); + InMemoryWalStore.checkResponse(input); + } + return new MemoryWriter(reference); + } + + @Override + public InputStream open(String reference) throws IOException { + requireInitialized(); + validateReference(reference); + try (var socket = InMemoryWalStore.connect(host, port); + var output = new DataOutputStream(socket.getOutputStream()); + var input = new DataInputStream(socket.getInputStream())) { + InMemoryWalStore.sendRequest(output, InMemoryWalStore.READ, reference); + output.flush(); + InMemoryWalStore.checkResponse(input); + int length = input.readInt(); + if (length < 0) { + throw new IOException("Invalid in-memory WAL length: " + length); + } + byte[] bytes = input.readNBytes(length); + if (bytes.length != length) { + throw new IOException("Incomplete WAL contents for " + reference); + } + return new ByteArrayInputStream(bytes); + } + } + + @Override + public boolean exists(String reference) throws IOException { + requireInitialized(); + validateReference(reference); + try (var socket = InMemoryWalStore.connect(host, port); + var output = new DataOutputStream(socket.getOutputStream()); + var input = new DataInputStream(socket.getInputStream())) { + InMemoryWalStore.sendRequest(output, InMemoryWalStore.EXISTS, reference); + output.flush(); + InMemoryWalStore.checkResponse(input); + return input.readBoolean(); + } + } + + @Override + public void delete(String reference) throws IOException { + requireInitialized(); + validateReference(reference); + try (var socket = InMemoryWalStore.connect(host, port); + var output = new DataOutputStream(socket.getOutputStream()); + var input = new DataInputStream(socket.getInputStream())) { + InMemoryWalStore.sendRequest(output, InMemoryWalStore.DELETE, reference); + output.flush(); + InMemoryWalStore.checkResponse(input); + } + } + + private void requireInitialized() throws IOException { + if (host == null || port < 1) { + throw new IOException("InMemoryWalProvider has not been initialized"); + } + } + + private void validateReference(String reference) throws IOException { + try { + URI uri = new URI(reference); + if (!SCHEME.equals(uri.getScheme()) || uri.getAuthority() != null || uri.getQuery() != null + || uri.getFragment() != null) { + throw new IOException("Invalid in-memory WAL reference: " + reference); + } + String path = uri.getPath(); + if (path == null || !path.startsWith(referenceRoot())) { + throw new IOException("Invalid in-memory WAL reference: " + reference); + } + String[] parts = path.substring(referenceRoot().length()).split("/", -1); + if (parts.length != 2 || parts[0].isBlank() || parts[1].isBlank()) { + throw new IOException("Invalid in-memory WAL reference: " + reference); + } + UUID.fromString(parts[1]); + } catch (URISyntaxException | IllegalArgumentException e) { + throw new IOException("Invalid in-memory WAL reference: " + reference, e); + } + } + + private String referenceRoot() { + return "/tmp/accumulo-inmemory-wal-" + port + "/wal/"; + } + + private final class MemoryWriter implements Writer { + private final String reference; + private final ByteArrayOutputStream pending = new ByteArrayOutputStream(); + private final OutputStream output = new OutputStream() { + @Override + public void write(int b) throws IOException { + append(new byte[] {(byte) b}, 0, 1); + } + + @Override + public void write(byte[] bytes, int offset, int length) throws IOException { + Objects.checkFromIndexSize(offset, length, bytes.length); + append(bytes, offset, length); + } + + @Override + public void close() throws IOException { + MemoryWriter.this.close(); + } + }; + private long length; + private boolean closed; + + private MemoryWriter(String reference) { + this.reference = reference; + } + + private synchronized void append(byte[] bytes, int offset, int count) throws IOException { + ensureOpen(); + if (count > 0) { + pending.write(bytes, offset, count); + length += count; + } + } + + private synchronized void flushPending() throws IOException { + if (pending.size() > 0) { + byte[] bytes = pending.toByteArray(); + try (var socket = InMemoryWalStore.connect(host, port); + var output = new DataOutputStream(socket.getOutputStream()); + var input = new DataInputStream(socket.getInputStream())) { + InMemoryWalStore.sendRequest(output, InMemoryWalStore.APPEND, reference); + output.writeInt(bytes.length); + output.write(bytes); + output.flush(); + InMemoryWalStore.checkResponse(input); + } + pending.reset(); + } + } + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("WAL writer is closed: " + reference); + } + } + + @Override + public String getReference() { + return reference; + } + + @Override + public OutputStream getOutputStream() { + return output; + } + + @Override + public synchronized void sync(Durability durability) throws IOException { + ensureOpen(); + if (durability != Durability.SYNC && durability != Durability.FLUSH) { + throw new IllegalArgumentException("Unexpected WAL sync durability " + durability); + } + flushPending(); + } + + @Override + public synchronized long getLength() throws IOException { + return length; + } + + @Override + public synchronized void close() throws IOException { + if (!closed) { + flushPending(); + closed = true; + } + } + } +} diff --git a/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalStore.java b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalStore.java new file mode 100644 index 00000000000..0f6448be6dc --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalStore.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.wal; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * An in-memory WAL store shared by {@link InMemoryWalProvider} instances in separate + * MiniAccumuloCluster processes. It listens on an available loopback TCP port, keeps WAL contents + * in the test process heap, and provides the host and port options needed to connect providers. + * Start the store before starting the cluster and keep it running until the cluster has stopped. + * Bytes appended by a provider are retained here after its sync or close, so they remain available + * to other provider processes if a TabletServer exits. + */ +public final class InMemoryWalStore implements AutoCloseable { + + static final byte CREATE = 1; + static final byte APPEND = 2; + static final byte READ = 3; + static final byte EXISTS = 4; + static final byte DELETE = 5; + private static final int CONNECT_TIMEOUT_MILLIS = 10_000; + + private final ServerSocket serverSocket; + private final ExecutorService handlers = Executors.newCachedThreadPool(r -> { + Thread thread = new Thread(r, "in-memory-wal-handler"); + thread.setDaemon(true); + return thread; + }); + private final Set connections = ConcurrentHashMap.newKeySet(); + private final Map wals = new ConcurrentHashMap<>(); + private final Thread acceptThread; + private volatile boolean closed; + + private InMemoryWalStore(ServerSocket serverSocket) { + this.serverSocket = serverSocket; + acceptThread = new Thread(this::acceptConnections, "in-memory-wal-acceptor"); + acceptThread.setDaemon(true); + acceptThread.start(); + } + + /** Starts a store listening on an available loopback port. */ + @SuppressFBWarnings(value = "UNENCRYPTED_SERVER_SOCKET", + justification = "This class is only used in tests.") + public static InMemoryWalStore start() throws IOException { + ServerSocket serverSocket = new ServerSocket(); + serverSocket.setReuseAddress(true); + serverSocket.bind(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0)); + return new InMemoryWalStore(serverSocket); + } + + /** Returns provider options that connect to this store. */ + public Map getProviderOptions() { + return Map.of(InMemoryWalProvider.HOST_OPTION, "127.0.0.1", InMemoryWalProvider.PORT_OPTION, + Integer.toString(serverSocket.getLocalPort())); + } + + private void acceptConnections() { + while (!closed) { + try { + Socket socket = serverSocket.accept(); + connections.add(socket); + handlers.execute(() -> handle(socket)); + } catch (IOException e) { + if (!closed) { + close(); + } + } + } + } + + private void handle(Socket socket) { + try (socket; var input = new DataInputStream(socket.getInputStream()); + var output = new DataOutputStream(socket.getOutputStream())) { + byte command = input.readByte(); + String reference = input.readUTF(); + try { + switch (command) { + case CREATE -> { + if (wals.putIfAbsent(reference, new WalData()) != null) { + throw new IOException("WAL already exists: " + reference); + } + output.writeBoolean(true); + } + case APPEND -> { + int length = input.readInt(); + if (length < 0) { + throw new IOException("Invalid append length: " + length); + } + byte[] bytes = input.readNBytes(length); + if (bytes.length != length) { + throw new EOFException("Incomplete WAL append"); + } + getWal(reference).append(bytes); + output.writeBoolean(true); + } + case READ -> { + byte[] bytes = getWal(reference).snapshot(); + output.writeBoolean(true); + output.writeInt(bytes.length); + output.write(bytes); + } + case EXISTS -> { + output.writeBoolean(true); + output.writeBoolean(wals.containsKey(reference)); + } + case DELETE -> { + wals.remove(reference); + output.writeBoolean(true); + } + default -> throw new IOException("Unknown in-memory WAL operation: " + command); + } + output.flush(); + } catch (IOException | RuntimeException e) { + output.writeBoolean(false); + output.writeUTF(e.getMessage() == null ? e.getClass().getName() : e.getMessage()); + output.flush(); + } + } catch (IOException e) { + // The client may have disconnected before the response was sent. + } finally { + connections.remove(socket); + } + } + + private WalData getWal(String reference) throws IOException { + WalData wal = wals.get(reference); + if (wal == null) { + throw new IOException("WAL does not exist: " + reference); + } + return wal; + } + + private static final class WalData { + private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + synchronized void append(byte[] append) { + bytes.writeBytes(append); + } + + synchronized byte[] snapshot() { + return bytes.toByteArray(); + } + } + + @SuppressFBWarnings(value = "UNENCRYPTED_SOCKET", + justification = "This class is only used in tests.") + static Socket connect(String host, int port) throws IOException { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress(host, port), CONNECT_TIMEOUT_MILLIS); + return socket; + } + + static void sendRequest(DataOutputStream output, byte command, String reference) + throws IOException { + output.writeByte(command); + output.writeUTF(reference); + } + + static void checkResponse(DataInputStream input) throws IOException { + if (!input.readBoolean()) { + throw new IOException(input.readUTF()); + } + } + + @Override + public synchronized void close() { + if (!closed) { + closed = true; + try { + serverSocket.close(); + } catch (IOException e) { + // Closing the listener is best effort during teardown. + } + for (Socket socket : connections) { + try { + socket.close(); + } catch (IOException e) { + // Closing active requests is best effort during teardown. + } + } + handlers.shutdownNow(); + if (Thread.currentThread() != acceptThread) { + try { + acceptThread.join(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + } +} diff --git a/test/src/main/resources/accumulo.properties b/test/src/main/resources/accumulo.properties index b63ed5c0617..2a7b4af5260 100644 --- a/test/src/main/resources/accumulo.properties +++ b/test/src/main/resources/accumulo.properties @@ -21,3 +21,5 @@ instance.dfs.dir=${project.build.directory}/instanceTest instance.secret=TEST_SYSTEM_SECRET +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider diff --git a/test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java b/test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java new file mode 100644 index 00000000000..b7852b44ba0 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.wal; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.UUID; + +import org.apache.accumulo.core.client.Durability; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; +import org.apache.accumulo.core.spi.wal.WalProvider; +import org.junit.jupiter.api.Test; + +class InMemoryWalProviderTest { + + @Test + void sharesSyncedAndClosedWalBytesAcrossProviders() throws Exception { + try (var store = InMemoryWalStore.start()) { + WalProvider writerProvider = newProvider(store.getProviderOptions()); + WalProvider readerProvider = newProvider(store.getProviderOptions()); + UUID id = UUID.randomUUID(); + WalProvider.Writer writer = writerProvider.create("tserver:1234", id); + String reference = writer.getReference(); + + assertEquals("file:/tmp/accumulo-inmemory-wal-" + + store.getProviderOptions().get(InMemoryWalProvider.PORT_OPTION) + "/wal/tserver+1234/" + + id, reference); + assertTrue(readerProvider.exists(reference)); + + writer.getOutputStream().write("header-and-synced-data".getBytes(UTF_8)); + assertEquals(22, writer.getLength()); + writer.sync(Durability.FLUSH); + try (InputStream input = readerProvider.open(reference)) { + assertArrayEquals("header-and-synced-data".getBytes(UTF_8), input.readAllBytes()); + } + + writer.getOutputStream().write("-tail".getBytes(UTF_8)); + writer.close(); + writer.close(); + try (InputStream input = readerProvider.open(reference)) { + assertArrayEquals("header-and-synced-data-tail".getBytes(UTF_8), input.readAllBytes()); + } + assertEquals(27, writer.getLength()); + + readerProvider.delete(reference); + readerProvider.delete(reference); + assertFalse(writerProvider.exists(reference)); + assertThrows(IOException.class, () -> readerProvider.open(reference)); + assertThrows(IOException.class, () -> writer.getOutputStream().write(1)); + } + } + + @Test + void rejectsInvalidProviderOptionsAndReferences() throws Exception { + InMemoryWalProvider uninitializedProvider = new InMemoryWalProvider(); + assertThrows(IllegalArgumentException.class, + () -> uninitializedProvider.init(initParameters(Map.of("host", "127.0.0.1")))); + + try (var store = InMemoryWalStore.start()) { + WalProvider provider = newProvider(store.getProviderOptions()); + assertThrows(IOException.class, () -> provider.exists("file:///tmp/not-a-wal")); + } + } + + @Test + void rejectsLogDurabilitySyncRequests() throws Exception { + try (var store = InMemoryWalStore.start()) { + WalProvider provider = newProvider(store.getProviderOptions()); + try (WalProvider.Writer writer = provider.create("tserver:1234", UUID.randomUUID())) { + assertThrows(IllegalArgumentException.class, () -> writer.sync(Durability.LOG)); + } + } + } + + private static WalProvider newProvider(Map options) { + InMemoryWalProvider provider = new InMemoryWalProvider(); + provider.init(initParameters(options)); + return provider; + } + + private static WalProvider.InitParameters initParameters(Map options) { + return new WalProvider.InitParameters() { + @Override + public ServiceEnvironment getServiceEnvironment() { + return null; + } + + @Override + public Map getOptions() { + return options; + } + }; + } +} From 1be2c1909c76fa8d0c343e8e065933bf188706a3 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Mon, 19 Oct 2026 16:28:08 +0000 Subject: [PATCH 2/3] Minor updates from review --- assemble/conf/accumulo.properties | 8 +++----- .../java/org/apache/accumulo/core/conf/Property.java | 10 ++++------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/assemble/conf/accumulo.properties b/assemble/conf/accumulo.properties index 5a9532f1d02..9bca9f94da5 100644 --- a/assemble/conf/accumulo.properties +++ b/assemble/conf/accumulo.properties @@ -23,11 +23,9 @@ ## Sets location in HDFS where Accumulo will store data instance.volumes=hdfs://localhost:8020/accumulo -## Configure the WAL provider implementation consistently on every server -## For the built-in filesystem-backed implementation, configure: -## org.apache.accumulo.server.log.DfsWalProvider -# instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} -# instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider +# Configure the WAL provider implementation +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider ## Sets location of Zookeepers instance.zookeeper.host=localhost:2181 diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index aafef25bef4..b3ca5746db8 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -261,16 +261,14 @@ temporary files (for example, when creating a pre-split table). \ + "org.apache.accumulo.core.spi.crypto.CryptoFactory interface.", "2.1.0"), INSTANCE_WAL_PROVIDERS("instance.wal.providers", "{}", PropertyType.JSON, - "Required JSON object mapping fully qualified class names implementing " - + "org.apache.accumulo.core.spi.wal.WalProvider to string-valued option maps. Use an " + "Map of WAL provider class names and their configuration. Keys are class names that implement " + + "org.apache.accumulo.core.spi.wal.WalProvider. Values are string-valued option maps. Use an " + "empty map when a provider has no options. All configured providers are validated at startup, must be " + "available to Managers, TabletServers, ScanServers, and GC, and must include providers " - + "referenced by existing WALs. This must be consistent across the instance.", + + "referenced by existing WALs.", "4.0.0"), INSTANCE_WAL_PROVIDER("instance.wal.provider", "", PropertyType.CLASSNAME, - "Required. The class implementing org.apache.accumulo.core.spi.wal.WalProvider used to create new " - + "WALs. It must also be listed in instance.wal.providers and must be consistent " - + "across the instance.", + "Class implementing org.apache.accumulo.core.spi.wal.WalProvider, will be used to create new WALs.", "4.0.0"), // general properties GENERAL_PREFIX("general.", null, PropertyType.PREFIX, From 5d4a8f3df6b7655cd9ef7fc885e873d66d36217c Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Wed, 21 Oct 2026 15:43:56 +0000 Subject: [PATCH 3/3] Removed overloaded WalProvider.init method --- WAL-SPI.md | 16 +++++++--------- .../accumulo/core/spi/wal/WalProvider.java | 19 ++++--------------- .../accumulo/server/log/DfsWalProvider.java | 4 ++-- 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/WAL-SPI.md b/WAL-SPI.md index 1f872a67f69..feeb4447c33 100644 --- a/WAL-SPI.md +++ b/WAL-SPI.md @@ -53,8 +53,6 @@ Each implementation needs a public no-argument constructor and a stable `getId() `[A-Za-z0-9]+`. The ID `dfs` is reserved for `DfsWalProvider` and legacy filesystem WALs. The [registry][registry] loads each configured class, passes its options to `init(InitParameters)` once per server process, rejects duplicate provider IDs, and selects the current class by class name. The -default `init(InitParameters)` implementation calls `init(ServiceEnvironment)`, preserving providers -that implement the older initialization method. A [memoized registry][context] is accessed by the Manager, TabletServers, ScanServers, and GC. The [Manager][manager] also checks WAL IDs in tablet metadata and ZooKeeper markers against the configured IDs at startup. @@ -89,13 +87,13 @@ raw WAL format or the sorted-log recovery process. [`WalProvider`][spi] exposes: -| Method | Use | -| ------------------------------- | ----------------------------------------------------- | -| `create(serverAddress, UUID)` | Exclusively create a WAL and return its `Writer`. | -| `open(reference)` | Read a WAL from byte zero for sorting/recovery. | -| `exists(reference)` | Check a WAL, including while it may still be written. | -| `delete(reference)` | Remove an unreferenced WAL; should be idempotent. | -| `getId()` / `init(environment)` | Supply the persisted ID and initialize the provider. | +| Method | Use | +| ----------------------------------- | ----------------------------------------------------- | +| `create(serverAddress, UUID)` | Exclusively create a WAL and return its `Writer`. | +| `open(reference)` | Read a WAL from byte zero for sorting/recovery. | +| `exists(reference)` | Check a WAL, including while it may still be written. | +| `delete(reference)` | Remove an unreferenced WAL; should be idempotent. | +| `getId()` / `init(InitParameters)` | Supply the persisted ID and initialize the provider. | The nested `Writer` supplies `getReference()`, `getOutputStream()`, `sync(Durability)`, `getLength()`, and `close()`. It can optionally report `getCurrentReplication()` and diff --git a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java index 842c3dca1b2..689e1e390df 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java @@ -60,25 +60,14 @@ public interface WalProvider { String getId(); /** - * Initializes the provider once after construction. Implementations may retain the environment - * for accessing system configuration and must not assume that properties change during this - * process lifetime. Throwing an exception prevents the server process from starting. - * - * @param environment system configuration and Accumulo service information - */ - default void init(ServiceEnvironment environment) {} - - /** - * Initializes this provider with its configured options. The default implementation calls - * {@link #init(ServiceEnvironment)} for compatibility with providers that only implement that - * method. + * Initializes this provider with its configured options. Implementations may retain the service + * environment for accessing system configuration and must not assume that properties change + * during this process lifetime. Throwing an exception prevents the server process from starting. * * @param params the service environment and options configured for this provider * @since 4.0.0 */ - default void init(InitParameters params) { - init(params.getServiceEnvironment()); - } + default void init(InitParameters params) {} /** Parameters passed to {@link #init(InitParameters)}. */ interface InitParameters { diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java index 096c978b215..d78bdad2773 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java @@ -25,7 +25,6 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Durability; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.spi.common.ServiceEnvironment; import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment; import org.apache.accumulo.core.spi.wal.WalProvider; import org.apache.accumulo.server.ServerContext; @@ -51,7 +50,8 @@ public String getId() { } @Override - public void init(ServiceEnvironment environment) { + public void init(InitParameters params) { + var environment = params.getServiceEnvironment(); if (!(environment instanceof ServiceEnvironmentImpl serviceEnvironment)) { throw new IllegalArgumentException("DfsWalProvider requires the server service environment"); }