From d8fee0c7382669312345c26497479608bbbd53ac Mon Sep 17 00:00:00 2001 From: Siddhi Agrawal <101979598+Siddhi-agg@users.noreply.github.com> Date: Tue, 22 Sep 2026 10:19:37 +0530 Subject: [PATCH 1/3] Add retained-state fallback for KILL actions Keep active-DAG cancellation unchanged and allow missing-DAG KILL actions to delegate to an optional backend using retained job states. Preserve opaque execution metadata, scope it to the requested flow and job, and recheck for active DAGs during fallback. Add handler configuration and lifecycle hooks with focused unit coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../gobblin/service/ServiceConfigKeys.java | 1 + .../metastore/MysqlJobStatusStateStore.java | 3 +- ...MysqlJobStatusStateStoreFlowScopeTest.java | 48 +++ .../monitoring/JobStatusRetriever.java | 24 ++ .../core/GobblinServiceGuiceModule.java | 10 + .../modules/core/GobblinServiceManager.java | 48 ++- .../modules/orchestration/DagProcFactory.java | 21 +- .../orchestration/ForceKillHandler.java | 68 ++++ .../orchestration/proc/KillDagProc.java | 89 +++++ .../monitoring/MysqlJobStatusRetriever.java | 9 + ...blinServiceForceKillConfigurationTest.java | 89 +++++ .../GobblinServiceForceKillLifecycleTest.java | 140 +++++++ .../DagProcFactoryForceKillTest.java | 141 +++++++ .../proc/KillDagProcForceKillTest.java | 355 ++++++++++++++++++ .../monitoring/RawJobStatusRetrieverTest.java | 107 ++++++ 15 files changed, 1148 insertions(+), 5 deletions(-) create mode 100644 gobblin-metastore/src/test/java/org/apache/gobblin/metastore/MysqlJobStatusStateStoreFlowScopeTest.java create mode 100644 gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ForceKillHandler.java create mode 100644 gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java create mode 100644 gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillLifecycleTest.java create mode 100644 gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagProcFactoryForceKillTest.java create mode 100644 gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java create mode 100644 gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java diff --git a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java index 35bbd8c2e82..729621b3551 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java @@ -37,6 +37,7 @@ public class ServiceConfigKeys { public static final String GOBBLIN_SERVICE_FLOWSPEC = GOBBLIN_SERVICE_PREFIX + "flowSpec"; public static final String GOBBLIN_SERVICE_FLOWGRAPH_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "flowGraph.class"; public static final String GOBBLIN_SERVICE_FLOWGRAPH_HELPER_KEY = GOBBLIN_SERVICE_PREFIX + "flowGraphHelper.class"; + public static final String FORCE_KILL_HANDLER_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "forceKillHandler.class"; // Flow Compiler Keys public static final String GOBBLIN_SERVICE_FLOWCOMPILER_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "flowCompiler.class"; diff --git a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlJobStatusStateStore.java b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlJobStatusStateStore.java index 8715d9f388b..04164430438 100644 --- a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlJobStatusStateStore.java +++ b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlJobStatusStateStore.java @@ -68,7 +68,8 @@ public MysqlJobStatusStateStore(DataSource dataSource, String stateStoreTableNam * @throws IOException in case of failures */ public List getAll(String storeName, long flowExecutionId) throws IOException { - return getAll(storeName, flowExecutionId + "%", JobStateSearchColumns.TABLE_NAME_ONLY); + return getAll(storeName, flowExecutionId + ServiceConfigKeys.STATE_STORE_KEY_SEPARATION_CHARACTER + "%", + JobStateSearchColumns.TABLE_NAME_ONLY); } /** diff --git a/gobblin-metastore/src/test/java/org/apache/gobblin/metastore/MysqlJobStatusStateStoreFlowScopeTest.java b/gobblin-metastore/src/test/java/org/apache/gobblin/metastore/MysqlJobStatusStateStoreFlowScopeTest.java new file mode 100644 index 00000000000..925d797228a --- /dev/null +++ b/gobblin-metastore/src/test/java/org/apache/gobblin/metastore/MysqlJobStatusStateStoreFlowScopeTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.metastore; + +import java.io.IOException; +import java.util.Collections; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.State; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + + +public class MysqlJobStatusStateStoreFlowScopeTest { + @Test + @SuppressWarnings("unchecked") + public void testExecutionPrefixIncludesSeparator() throws IOException { + MysqlJobStatusStateStore store = mock(MysqlJobStatusStateStore.class, CALLS_REAL_METHODS); + doReturn(Collections.emptyList()).when(store).getAll(eq("group.flow"), anyString(), + eq(MysqlStateStore.JobStateSearchColumns.TABLE_NAME_ONLY)); + + Assert.assertTrue(store.getAll("group.flow", 1234L).isEmpty()); + + verify(store).getAll("group.flow", "1234.%", MysqlStateStore.JobStateSearchColumns.TABLE_NAME_ONLY); + } +} diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java b/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java index c82e806c641..7ba5f92c527 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java @@ -17,6 +17,7 @@ package org.apache.gobblin.service.monitoring; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -49,6 +50,7 @@ import org.apache.gobblin.runtime.troubleshooter.TroubleshooterException; import org.apache.gobblin.runtime.troubleshooter.TroubleshooterUtils; import org.apache.gobblin.service.ExecutionStatus; +import org.apache.gobblin.service.ServiceConfigKeys; import org.apache.gobblin.util.ConfigUtils; @@ -86,6 +88,28 @@ public abstract Iterator getJobStatusesForFlowExecution(String flowNa public abstract Iterator getJobStatusesForFlowExecution(String flowName, String flowGroup, long flowExecutionId, String jobName, String jobGroup); + /** + * Read retained states without discarding backend metadata during conversion to {@link JobStatus}. + * Includes the flow-summary record, if present. Callers must validate the decoded identity before using + * execution handles: storage keys alone do not establish the identity of the serialized contents. + * + *

Unlike status-display APIs, read failures propagate rather than appearing as an empty result.

+ */ + public List getJobStatusStatesForFlowExecution(String flowName, String flowGroup, long flowExecutionId) + throws IOException { + Objects.requireNonNull(flowName, "flowName"); + Objects.requireNonNull(flowGroup, "flowGroup"); + String separator = ServiceConfigKeys.STATE_STORE_KEY_SEPARATION_CHARACTER; + String storeName = flowGroup + separator + flowName; + String tablePrefix = flowExecutionId + separator; + StateStore stateStore = getStateStore(); + List states = new ArrayList<>(); + for (String tableName : stateStore.getTableNames(storeName, name -> name.startsWith(tablePrefix))) { + states.addAll(stateStore.getAll(storeName, tableName)); + } + return states; + } + /** * Get the latest {@link FlowStatus}es of executions of flows belonging to this flow group. Currently, latest flow execution * is decided by comparing {@link JobStatus#getFlowExecutionId()}. diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java index 9e2ba6f79e6..19a17b1cc39 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java @@ -68,6 +68,7 @@ import org.apache.gobblin.service.modules.orchestration.DagTaskStream; import org.apache.gobblin.service.modules.orchestration.FlowLaunchHandler; import org.apache.gobblin.service.modules.orchestration.FlowLaunchMultiActiveLeaseArbiterFactory; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; import org.apache.gobblin.service.modules.orchestration.MultiActiveLeaseArbiter; import org.apache.gobblin.service.modules.orchestration.MySqlDagManagementStateStore; import org.apache.gobblin.service.modules.orchestration.MysqlDagActionStore; @@ -164,6 +165,7 @@ binding time (optionally bound classes cannot have names associated with them), binder.bind(DagManagement.class).to(DagManagementTaskStreamImpl.class); binder.bind(DagManagementStateStore.class).to(MySqlDagManagementStateStore.class); binder.bind(DagTaskStream.class).to(DagManagementTaskStreamImpl.class); + configureForceKillHandler(binder, serviceConfig.getInnerConfig()); binder.bind(DagProcFactory.class); binder.bind(DagProcessingEngine.class); binder.bind(DagProcessingEngineMetrics.class); @@ -257,6 +259,14 @@ protected static Class getClassByNameOrAlias(Class baseClass } } + static void configureForceKillHandler(Binder binder, Config config) { + OptionalBinder optional = OptionalBinder.newOptionalBinder(binder, ForceKillHandler.class); + if (config.hasPath(ServiceConfigKeys.FORCE_KILL_HANDLER_CLASS_KEY)) { + optional.setBinding().to(getClassByNameOrAlias(ForceKillHandler.class, config, + ServiceConfigKeys.FORCE_KILL_HANDLER_CLASS_KEY, null)).in(Singleton.class); + } + } + public static class EmbeddedRestliServerProvider implements Provider { Injector injector; diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java index 9bab575350c..9165783972d 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java @@ -38,6 +38,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.base.Optional; +import com.google.common.util.concurrent.AbstractIdleService; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; @@ -71,6 +73,7 @@ import org.apache.gobblin.service.ServiceConfigKeys; import org.apache.gobblin.service.modules.db.ServiceDatabaseManager; import org.apache.gobblin.service.modules.orchestration.DagProcessingEngine; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; import org.apache.gobblin.service.modules.orchestration.Orchestrator; import org.apache.gobblin.service.modules.orchestration.UserQuotaManager; import org.apache.gobblin.service.modules.restli.FlowConfigsV2ResourceHandler; @@ -174,6 +177,9 @@ public class GobblinServiceManager implements ApplicationLauncher { @Inject(optional = true) protected DagProcessingEngine dagProcessingEngine; + @Inject + private Optional forceKillHandler = Optional.absent(); + @Inject protected GobblinServiceManager(GobblinServiceConfiguration configuration) throws Exception { this.configuration = Objects.requireNonNull(configuration); @@ -256,6 +262,19 @@ private void registerServicesInLauncher(){ } this.serviceLauncher.addService(dagProcessingEngine); + if (this.forceKillHandler.isPresent()) { + // The launcher's own failure/shutdown paths must also close the backend, not just this manager's stop(). + this.serviceLauncher.addService(new AbstractIdleService() { + @Override + protected void startUp() { + } + + @Override + protected void shutDown() { + forceKillHandler.get().close(); + } + }); + } this.serviceLauncher.addService(databaseManager); this.serviceLauncher.addService(issueRepository); @@ -297,6 +316,13 @@ private void configureServices(){ @Override public void start() throws ApplicationException { + try (ForceKillStartupGuard guard = new ForceKillStartupGuard(this.forceKillHandler.orNull())) { + startServices(); + guard.started = true; + } + } + + private void startServices() throws ApplicationException { LOGGER.info("[Init] Starting the Gobblin Service Manager"); configureServices(); @@ -346,7 +372,7 @@ public void stop() throws ApplicationException { LOGGER.info("Stopping the Gobblin Service Manager"); this.stopInProgress = true; - try { + try (ForceKillHandler handler = this.forceKillHandler.orNull()) { // Stop announcing GaaS instances to d2 when services are stopped this.d2Announcer.markDownServer(); this.serviceLauncher.stop(); @@ -357,7 +383,25 @@ public void stop() throws ApplicationException { @Override public void close() throws IOException { - this.serviceLauncher.close(); + try (ForceKillHandler handler = this.forceKillHandler.orNull()) { + this.serviceLauncher.close(); + } + } + + static final class ForceKillStartupGuard implements AutoCloseable { + private final ForceKillHandler handler; + boolean started; + + ForceKillStartupGuard(ForceKillHandler handler) { + this.handler = handler; + } + + @Override + public void close() { + if (!this.started && this.handler != null) { + this.handler.close(); + } + } } private static String getServiceId(CommandLine cmd) { diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcFactory.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcFactory.java index 10b9bf27a81..98843510582 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcFactory.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcFactory.java @@ -17,7 +17,11 @@ package org.apache.gobblin.service.modules.orchestration; +import java.util.Objects; + +import com.google.common.base.Optional; import com.google.inject.Inject; +import com.google.inject.Provider; import com.google.inject.Singleton; import com.typesafe.config.Config; @@ -37,6 +41,7 @@ import org.apache.gobblin.service.modules.orchestration.task.ReevaluateDagTask; import org.apache.gobblin.service.modules.orchestration.task.ResumeDagTask; import org.apache.gobblin.service.modules.utils.FlowCompilationValidationHelper; +import org.apache.gobblin.service.monitoring.JobStatusRetriever; /** * {@link DagTaskVisitor} for transforming a specific {@link DagTask} derived class to its companion {@link DagProc} derived class. @@ -50,11 +55,21 @@ public class DagProcFactory implements DagTaskVisitor> { private final Config config; private final FlowCompilationValidationHelper flowCompilationValidationHelper; + private final Optional> forceKillHandler; + private final Provider jobStatusRetriever; - @Inject public DagProcFactory(Config config, FlowCompilationValidationHelper flowCompilationValidationHelper) { + this(config, flowCompilationValidationHelper, Optional.absent(), null); + } + + @Inject + public DagProcFactory(Config config, FlowCompilationValidationHelper flowCompilationValidationHelper, + Optional> forceKillHandler, Provider jobStatusRetriever) { this.config = config; this.flowCompilationValidationHelper = flowCompilationValidationHelper; + this.forceKillHandler = Objects.requireNonNull(forceKillHandler, "forceKillHandler"); + this.jobStatusRetriever = forceKillHandler.isPresent() + ? Objects.requireNonNull(jobStatusRetriever, "jobStatusRetriever") : jobStatusRetriever; } @Override @@ -79,6 +94,9 @@ public ReevaluateDagProc meet(ReevaluateDagTask reEvaluateDagTask) { @Override public KillDagProc meet(KillDagTask killDagTask) { + if (this.forceKillHandler.isPresent()) { + return new KillDagProc(killDagTask, this.config, this.forceKillHandler.get(), this.jobStatusRetriever); + } return new KillDagProc(killDagTask, this.config); } @@ -87,4 +105,3 @@ public ResumeDagProc meet(ResumeDagTask resumeDagTask) { return new ResumeDagProc(resumeDagTask, this.config); } } - diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ForceKillHandler.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ForceKillHandler.java new file mode 100644 index 00000000000..66914a6cabe --- /dev/null +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ForceKillHandler.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; + +import org.apache.gobblin.configuration.State; + + +/** + * Backend-specific handling of an existing KILL action after its active DAG has been removed. + * + *

The supplied states are the only retained execution metadata source. Implementations must validate backend + * handles, destinations and their original submission identities, without consulting archived DAGs or guessing + * destinations from current routing. A flow-summary state is context, not an execution to stop. Implementations + * must not rewrite terminal job status or emit normal flow/job cancellation events. Bind implementations as + * singletons so the service manager can close the same instance used by KILL processing.

+ */ +@FunctionalInterface +public interface ForceKillHandler extends Closeable { + enum Result { + /** All identified stop requests were acknowledged; this does not establish physical process termination. */ + ACKNOWLEDGED, + /** No execution could be resolved, or at least one stop was not acknowledged. */ + NOT_ACKNOWLEDGED, + /** An active DAG was observed; stop retained processing and let normal DAG cancellation take over. */ + ACTIVE_DAG_PRESENT + } + + @FunctionalInterface + interface ActiveDagCheck { + boolean isActive() throws IOException; + } + + /** + * Attempt bounded, backend-specific stopping using defensive copies of the retained job states. + * + * @param action the exact flow execution and optional job requested for cancellation + * @param jobStates scoped job states and, if retained, the flow-summary state; the list is unmodifiable + * @param activeDagCheck must be checked before each backend stop; return {@link Result#ACTIVE_DAG_PRESENT} if true. + * This reduces, but does not atomically exclude, concurrent resume. + * @throws IOException when the operation cannot be completed; failures must not be reported as acknowledgements + */ + Result forceKill(DagActionStore.DagAction action, List jobStates, ActiveDagCheck activeDagCheck) + throws IOException; + + /** Stop accepting work and release local resources without waiting indefinitely for remote calls. Must be idempotent. */ + @Override + default void close() { + } +} diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java index d181c2996db..2e5ba837f23 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java @@ -18,20 +18,28 @@ package org.apache.gobblin.service.modules.orchestration.proc; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; import java.util.Optional; +import com.google.inject.Provider; import com.typesafe.config.Config; import lombok.extern.slf4j.Slf4j; +import org.apache.gobblin.configuration.State; import org.apache.gobblin.metrics.event.TimingEvent; import org.apache.gobblin.runtime.troubleshooter.IssueSeverity; import org.apache.gobblin.service.modules.flowgraph.Dag; import org.apache.gobblin.service.modules.orchestration.DagActionStore; import org.apache.gobblin.service.modules.orchestration.DagManagementStateStore; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; import org.apache.gobblin.service.modules.orchestration.task.DagProcessingEngineMetrics; import org.apache.gobblin.service.modules.orchestration.task.KillDagTask; import org.apache.gobblin.service.modules.spec.JobExecutionPlan; +import org.apache.gobblin.service.monitoring.JobStatusRetriever; /** @@ -41,10 +49,20 @@ @Slf4j public class KillDagProc extends DagProc>> { private final boolean shouldKillSpecificJob; + private final Provider forceKillHandler; + private final Provider jobStatusRetriever; public KillDagProc(KillDagTask killDagTask, Config config) { + this(killDagTask, config, null, null); + } + + public KillDagProc(KillDagTask killDagTask, Config config, Provider forceKillHandler, + Provider jobStatusRetriever) { super(killDagTask, config); this.shouldKillSpecificJob = !getDagNodeId().getJobName().equals(DagActionStore.NO_JOB_NAME_DEFAULT); + this.forceKillHandler = forceKillHandler; + this.jobStatusRetriever = forceKillHandler == null ? jobStatusRetriever + : Objects.requireNonNull(jobStatusRetriever, "jobStatusRetriever"); } @Override @@ -59,6 +77,10 @@ protected void act(DagManagementStateStore dagManagementStateStore, Optional>"); if (!dag.isPresent()) { + if (this.forceKillHandler != null) { + forceKillFromJobStatus(dagManagementStateStore, dagProcEngineMetrics); + return; + } dagProcEngineMetrics.markDagActionsAct(getDagActionType(), false); log.error("Did not find Dag with id {}, it might be already cancelled/finished and thus cleaned up from the store.", getDagId()); OrchestratorIssueEmitter.emitFlowIssue(eventSubmitter, getDagId(), IssueSeverity.WARN, @@ -84,4 +106,71 @@ protected void act(DagManagementStateStore dagManagementStateStore, Optional retainedStates = this.jobStatusRetriever.get().getJobStatusStatesForFlowExecution( + getDagId().getFlowName(), getDagId().getFlowGroup(), getDagId().getFlowExecutionId()); + List scopedStates = new ArrayList<>(); + boolean hasJob = false; + for (State state : retainedStates) { + if (!matchesFlow(state)) { + log.warn("Ignoring out-of-scope job status during force kill of {}", getDagId()); + continue; + } + boolean flowSummary = isFlowSummary(state); + if (flowSummary || !this.shouldKillSpecificJob + || (getDagNodeId().getJobGroup().equals(state.getProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD)) + && getDagNodeId().getJobName().equals(state.getProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD)))) { + scopedStates.add(new State(state)); + hasJob |= !flowSummary; + } + } + + Optional> activeDag = store.getDag(getDagId()); + if (activeDag.isPresent()) { + act(store, activeDag, metrics); + return; + } + if (!hasJob) { + reportForceKillFailure(metrics, "No retained job status found for force kill"); + return; + } + + ForceKillHandler.Result result = Objects.requireNonNull(this.forceKillHandler.get().forceKill( + this.dagTask.getDagAction(), Collections.unmodifiableList(scopedStates), + () -> store.getDag(getDagId()).isPresent()), "forceKill result"); + if (result == ForceKillHandler.Result.ACTIVE_DAG_PRESENT) { + activeDag = store.getDag(getDagId()); + if (activeDag.isPresent()) { + act(store, activeDag, metrics); + } else { + reportForceKillFailure(metrics, "Active DAG changed during force kill; retained stopping was abandoned"); + } + } else if (result == ForceKillHandler.Result.ACKNOWLEDGED) { + metrics.markDagActionsAct(getDagActionType(), true); + log.info("Force-kill requests acknowledged for {}; process termination is not confirmed", getDagId()); + } else { + reportForceKillFailure(metrics, "Force kill did not acknowledge all retained executions"); + } + } + + private boolean matchesFlow(State state) { + return getDagId().getFlowGroup().equals(state.getProp(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD)) + && getDagId().getFlowName().equals(state.getProp(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD)) + && Long.toString(getDagId().getFlowExecutionId()) + .equals(state.getProp(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD)); + } + + private static boolean isFlowSummary(State state) { + return JobStatusRetriever.NA_KEY.equals(state.getProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD)) + && JobStatusRetriever.NA_KEY.equals(state.getProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD)); + } + + private void reportForceKillFailure(DagProcessingEngineMetrics metrics, String message) { + metrics.markDagActionsAct(getDagActionType(), false); + log.warn("{}: {} (node: {})", message, getDagId(), getDagNodeId()); + OrchestratorIssueEmitter.emitFlowIssue(eventSubmitter, getDagId(), IssueSeverity.WARN, + message + ": " + getDagNodeId()); + } } diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/MysqlJobStatusRetriever.java b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/MysqlJobStatusRetriever.java index 5678fef6aab..4fba36f6698 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/MysqlJobStatusRetriever.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/MysqlJobStatusRetriever.java @@ -76,6 +76,15 @@ public Iterator getJobStatusesForFlowExecution(String flowName, Strin return asJobStatuses(jobStatusStates); } + @Override + public List getJobStatusStatesForFlowExecution(String flowName, String flowGroup, long flowExecutionId) + throws IOException { + String storeName = KafkaJobStatusMonitor.jobStatusStoreName(flowGroup, flowName); + try (Timer.Context context = this.metricContext.contextAwareTimer(GET_LATEST_FLOW_STATUS_METRIC).time()) { + return this.stateStore.getAll(storeName, flowExecutionId); + } + } + @Override public Iterator getJobStatusesForFlowExecution(String flowName, String flowGroup, long flowExecutionId, String jobName, String jobGroup) { diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java new file mode 100644 index 00000000000..913c34fbea1 --- /dev/null +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gobblin.service.modules.core; + +import java.util.List; + +import javax.inject.Inject; + +import com.google.common.base.Optional; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provider; +import com.google.inject.TypeLiteral; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigValueFactory; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.service.ServiceConfigKeys; +import org.apache.gobblin.service.modules.orchestration.DagActionStore; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; + + +public class GobblinServiceForceKillConfigurationTest { + @Test + public void testNoConfiguredBackendLeavesOptionalAbsent() { + Injector injector = injector(ConfigFactory.empty()); + + Assert.assertFalse(injector.getInstance(Key.get(new TypeLiteral>() { })).isPresent()); + Assert.assertFalse(injector.getInstance(Key.get( + new TypeLiteral>>() { })).isPresent()); + } + + @Test + public void testConfiguredBackendIsInjectedAsOneSingletonForManagerAndFactory() { + Config config = ConfigFactory.empty().withValue(ServiceConfigKeys.FORCE_KILL_HANDLER_CLASS_KEY, + ConfigValueFactory.fromAnyRef(TestHandler.class.getName())); + Injector injector = injector(config); + + ForceKillHandler managerHandler = + injector.getInstance(Key.get(new TypeLiteral>() { })).get(); + Provider factoryProvider = + injector.getInstance(Key.get(new TypeLiteral>>() { })).get(); + + Assert.assertSame(managerHandler, factoryProvider.get()); + Assert.assertSame(managerHandler, factoryProvider.get()); + Assert.assertSame(((TestHandler) managerHandler).config, config); + } + + private static Injector injector(Config config) { + return Guice.createInjector(binder -> { + binder.requireExplicitBindings(); + binder.bind(Config.class).toInstance(config); + GobblinServiceGuiceModule.configureForceKillHandler(binder, config); + }); + } + + public static class TestHandler implements ForceKillHandler { + final Config config; + + @Inject + public TestHandler(Config config) { + this.config = config; + } + + @Override + public Result forceKill(DagActionStore.DagAction action, List states, ActiveDagCheck guard) { + return Result.ACKNOWLEDGED; + } + } +} diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillLifecycleTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillLifecycleTest.java new file mode 100644 index 00000000000..79b62ed8955 --- /dev/null +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillLifecycleTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gobblin.service.modules.core; + +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Optional; +import com.google.common.util.concurrent.Service; + +import org.mockito.ArgumentCaptor; +import org.testng.Assert; +import org.testng.annotations.Test; + +import org.apache.gobblin.runtime.app.ServiceBasedAppLauncher; +import org.apache.gobblin.runtime.spec_catalog.FlowCatalog; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; + +import static org.mockito.Mockito.*; +import static org.powermock.reflect.Whitebox.setInternalState; + + +public class GobblinServiceForceKillLifecycleTest { + @Test + public void testManagerStopClosesBackendAndIsIdempotent() throws Exception { + Fixture fixture = new Fixture(); + + fixture.manager.stop(); + fixture.manager.stop(); + + verify(fixture.launcher).stop(); + verify(fixture.handler).close(); + } + + @Test + public void testBackendClosesWhenShutdownFailsBeforeLauncher() { + Fixture fixture = new Fixture(); + IllegalStateException failure = new IllegalStateException("d2 failure"); + doThrow(failure).when(fixture.manager.d2Announcer).markDownServer(); + + Assert.assertSame(Assert.expectThrows(IllegalStateException.class, fixture.manager::stop), failure); + verify(fixture.handler).close(); + verifyNoInteractions(fixture.launcher); + } + + @Test + public void testBackendClosesWhenLauncherCloseFails() throws Exception { + Fixture fixture = new Fixture(); + IOException failure = new IOException("launcher close failed"); + doThrow(failure).when(fixture.launcher).close(); + + Assert.assertSame(Assert.expectThrows(IOException.class, fixture.manager::close), failure); + verify(fixture.handler).close(); + } + + @Test + public void testStartupFailureClosesBackendWithoutHidingOriginalFailure() { + Fixture fixture = new Fixture(); + IllegalStateException failure = new IllegalStateException("configuration failed"); + when(fixture.configuration.isRestLIServerEnabled()).thenThrow(failure); + IllegalStateException closeFailure = new IllegalStateException("close failed"); + doThrow(closeFailure).when(fixture.handler).close(); + + Assert.assertSame(Assert.expectThrows(IllegalStateException.class, fixture.manager::start), failure); + Assert.assertEquals(failure.getSuppressed(), new Throwable[] {closeFailure}); + verify(fixture.handler).close(); + } + + @Test + public void testLauncherOwnShutdownPathAlsoClosesBackend() throws Exception { + Fixture fixture = new Fixture(); + fixture.manager.flowCatalog = mock(FlowCatalog.class); + IllegalStateException failure = new IllegalStateException("launcher start failed"); + doThrow(failure).when(fixture.launcher).start(); + Assert.assertSame(Assert.expectThrows(IllegalStateException.class, fixture.manager::start), failure); + verify(fixture.handler).close(); + + ArgumentCaptor registered = ArgumentCaptor.forClass(Service.class); + verify(fixture.launcher, atLeastOnce()).addService(registered.capture()); + Service backendLifecycle = registered.getAllValues().stream().filter(Objects::nonNull) + .filter(service -> service != fixture.manager.flowCatalog).findFirst().get(); + clearInvocations(fixture.handler); + + backendLifecycle.startAsync().awaitRunning(5, TimeUnit.SECONDS); + backendLifecycle.stopAsync().awaitTerminated(5, TimeUnit.SECONDS); + + verify(fixture.handler).close(); + } + + @Test + public void testNoBackendPreservesManagerShutdown() throws Exception { + Fixture fixture = new Fixture(); + setInternalState(fixture.manager, "forceKillHandler", Optional.absent()); + + fixture.manager.stop(); + fixture.manager.close(); + + verify(fixture.launcher).stop(); + verify(fixture.launcher).close(); + verifyNoInteractions(fixture.handler); + } + + @Test + public void testSuccessfulStartupGuardDoesNotCloseBackend() { + ForceKillHandler handler = mock(ForceKillHandler.class); + try (GobblinServiceManager.ForceKillStartupGuard guard = new GobblinServiceManager.ForceKillStartupGuard(handler)) { + guard.started = true; + } + verifyNoInteractions(handler); + } + + private static final class Fixture { + final ForceKillHandler handler = mock(ForceKillHandler.class); + final ServiceBasedAppLauncher launcher = mock(ServiceBasedAppLauncher.class); + final GobblinServiceConfiguration configuration = mock(GobblinServiceConfiguration.class); + final GobblinServiceManager manager = mock(GobblinServiceManager.class, CALLS_REAL_METHODS); + + Fixture() { + setInternalState(this.manager, "forceKillHandler", Optional.of(this.handler)); + setInternalState(this.manager, "serviceLauncher", this.launcher); + setInternalState(this.manager, "configuration", this.configuration); + this.manager.d2Announcer = mock(D2Announcer.class); + } + } +} diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagProcFactoryForceKillTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagProcFactoryForceKillTest.java new file mode 100644 index 00000000000..9ddc1eba18c --- /dev/null +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagProcFactoryForceKillTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration; + +import java.io.IOException; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provider; +import com.google.inject.multibindings.OptionalBinder; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import org.mockito.MockedStatic; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.metrics.event.TimingEvent; +import org.apache.gobblin.service.modules.orchestration.proc.KillDagProc; +import org.apache.gobblin.service.modules.orchestration.proc.ReevaluateDagProc; +import org.apache.gobblin.service.modules.orchestration.proc.OrchestratorIssueEmitter; +import org.apache.gobblin.service.modules.orchestration.task.DagProcessingEngineMetrics; +import org.apache.gobblin.service.modules.orchestration.task.KillDagTask; +import org.apache.gobblin.service.modules.orchestration.task.ReevaluateDagTask; +import org.apache.gobblin.service.modules.utils.FlowCompilationValidationHelper; +import org.apache.gobblin.service.monitoring.JobStatusRetriever; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + + +public class DagProcFactoryForceKillTest { + @DataProvider + public Object[][] handlerBindings() { + return new Object[][] {{false}, {true}}; + } + + @Test(dataProvider = "handlerBindings") + public void testOptionalGuiceBindingIsLazyAndUsedOnlyForMissingDagKill(boolean bindHandler) throws IOException { + AtomicInteger handlerRequests = new AtomicInteger(); + AtomicInteger retrieverRequests = new AtomicInteger(); + ForceKillHandler handler = mock(ForceKillHandler.class); + JobStatusRetriever retriever = mock(JobStatusRetriever.class); + Provider handlerProvider = () -> { + handlerRequests.incrementAndGet(); + return handler; + }; + Provider retrieverProvider = () -> { + retrieverRequests.incrementAndGet(); + return retriever; + }; + Injector injector = Guice.createInjector(binder -> { + binder.requireExplicitBindings(); + binder.bind(Config.class).toInstance(ConfigFactory.empty()); + binder.bind(FlowCompilationValidationHelper.class).toInstance(mock(FlowCompilationValidationHelper.class)); + binder.bind(JobStatusRetriever.class).toProvider(retrieverProvider); + OptionalBinder optional = OptionalBinder.newOptionalBinder(binder, ForceKillHandler.class); + if (bindHandler) { + optional.setBinding().toProvider(handlerProvider); + } + binder.bind(DagProcFactory.class); + }); + DagProcFactory factory = injector.getInstance(DagProcFactory.class); + DagManagementStateStore store = mock(DagManagementStateStore.class); + DagProcessingEngineMetrics metrics = mock(DagProcessingEngineMetrics.class); + when(store.getDag(any())).thenReturn(Optional.empty()); + when(handler.forceKill(any(), anyList(), any())).thenReturn(ForceKillHandler.Result.ACKNOWLEDGED); + State state = new State(); + state.setProp(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD, "group"); + state.setProp(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, "flow"); + state.setProp(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD, "1234"); + state.setProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD, "group"); + state.setProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD, "job"); + when(retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L)) + .thenReturn(Collections.singletonList(state)); + + ReevaluateDagTask reevaluate = mock(ReevaluateDagTask.class); + when(reevaluate.getDagAction()).thenReturn(action(DagActionStore.DagActionType.REEVALUATE)); + Assert.assertEquals(factory.meet(reevaluate).getClass(), ReevaluateDagProc.class); + KillDagProc kill = factory.meet(new KillDagTask(action(DagActionStore.DagActionType.KILL), null, store, metrics)); + Assert.assertEquals(handlerRequests.get(), 0); + Assert.assertEquals(retrieverRequests.get(), 0); + + try (MockedStatic ignored = mockStatic(OrchestratorIssueEmitter.class)) { + kill.process(store, metrics); + } + + Assert.assertEquals(handlerRequests.get(), bindHandler ? 1 : 0); + Assert.assertEquals(retrieverRequests.get(), bindHandler ? 1 : 0); + verify(metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, bindHandler); + if (bindHandler) { + verify(handler).forceKill(any(), anyList(), any()); + } else { + verifyNoInteractions(handler, retriever); + } + } + + @Test + public void testLegacyConstructorStillCreatesOrdinaryKillProcessor() throws IOException { + DagProcFactory factory = new DagProcFactory(ConfigFactory.empty(), null); + DagManagementStateStore store = mock(DagManagementStateStore.class); + DagProcessingEngineMetrics metrics = mock(DagProcessingEngineMetrics.class); + when(store.getDag(any())).thenReturn(Optional.empty()); + + try (MockedStatic ignored = mockStatic(OrchestratorIssueEmitter.class)) { + factory.meet(new KillDagTask(action(DagActionStore.DagActionType.KILL), null, store, metrics)) + .process(store, metrics); + } + + verify(metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, false); + } + + private static DagActionStore.DagAction action(DagActionStore.DagActionType type) { + return DagActionStore.DagAction.forFlow("group", "flow", 1234L, type); + } +} diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java new file mode 100644 index 00000000000..64ec85d6518 --- /dev/null +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java @@ -0,0 +1,355 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration.proc; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import com.google.inject.Provider; +import com.typesafe.config.ConfigFactory; + +import org.apache.commons.lang3.tuple.Pair; +import org.mockito.MockedStatic; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.metrics.event.TimingEvent; +import org.apache.gobblin.service.ExecutionStatus; +import org.apache.gobblin.service.modules.flowgraph.Dag; +import org.apache.gobblin.service.modules.orchestration.DagActionStore; +import org.apache.gobblin.service.modules.orchestration.DagManagementStateStore; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; +import org.apache.gobblin.service.modules.orchestration.task.DagProcessingEngineMetrics; +import org.apache.gobblin.service.modules.orchestration.task.KillDagTask; +import org.apache.gobblin.service.modules.spec.JobExecutionPlan; +import org.apache.gobblin.service.monitoring.JobStatusRetriever; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + + +public class KillDagProcForceKillTest { + private static final String GROUP = "group"; + private static final String FLOW = "flow"; + private static final long EXECUTION_ID = 1234L; + + private DagManagementStateStore store; + private JobStatusRetriever retriever; + private ForceKillHandler handler; + private Provider retrieverProvider; + private Provider handlerProvider; + private DagProcessingEngineMetrics metrics; + private MockedStatic dagUtils; + private MockedStatic issues; + + @BeforeMethod + @SuppressWarnings("unchecked") + public void setUp() throws IOException { + this.store = mock(DagManagementStateStore.class); + this.retriever = mock(JobStatusRetriever.class); + this.handler = mock(ForceKillHandler.class); + this.retrieverProvider = mock(Provider.class); + this.handlerProvider = mock(Provider.class); + this.metrics = mock(DagProcessingEngineMetrics.class); + when(this.store.getDag(any())).thenReturn(Optional.empty()); + when(this.retrieverProvider.get()).thenReturn(this.retriever); + when(this.handlerProvider.get()).thenReturn(this.handler); + when(this.handler.forceKill(any(), anyList(), any())).thenReturn(ForceKillHandler.Result.ACKNOWLEDGED); + statuses(); + this.dagUtils = mockStatic(DagProcUtils.class); + this.issues = mockStatic(OrchestratorIssueEmitter.class); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + if (this.issues != null) { + this.issues.close(); + } + if (this.dagUtils != null) { + this.dagUtils.close(); + } + } + + @DataProvider + public Object[][] activeScopes() { + return new Object[][] {{DagActionStore.NO_JOB_NAME_DEFAULT}, {"job"}}; + } + + @Test(dataProvider = "activeScopes") + @SuppressWarnings("unchecked") + public void testActiveDagUsesOnlyNormalCancellation(String jobName) throws IOException { + Dag dag = mock(Dag.class); + Dag.DagNode node = mock(Dag.DagNode.class); + when(this.store.getDag(any())).thenReturn(Optional.of(dag)); + when(this.store.getDagNodeWithJobStatus(any())).thenReturn(Pair.of(Optional.of(node), Optional.empty())); + + processor(jobName).process(this.store, this.metrics); + + this.dagUtils.verify(() -> DagProcUtils.setAndEmitFlowEvent(any(), eq(dag), + eq(TimingEvent.FlowTimings.FLOW_CANCELLED))); + if (jobName.equals(DagActionStore.NO_JOB_NAME_DEFAULT)) { + this.dagUtils.verify(() -> DagProcUtils.cancelDag(dag, this.store)); + } else { + this.dagUtils.verify(() -> DagProcUtils.cancelDagNode(node, this.store)); + } + verifyNoInteractions(this.handlerProvider, this.retrieverProvider, this.handler, this.retriever); + verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, true); + verify(this.store, never()).getFailedDag(any()); + } + + @Test + public void testAbsentHandlerPreservesMissingDagBehavior() throws IOException { + new KillDagProc(task(DagActionStore.NO_JOB_NAME_DEFAULT), ConfigFactory.empty()).process(this.store, this.metrics); + + verifyNoInteractions(this.handlerProvider, this.retrieverProvider, this.handler, this.retriever); + assertFailedWithoutCancellation(); + verify(this.store, never()).getFailedDag(any()); + } + + @Test + public void testAcknowledgedFallbackGetsImmutableDefensiveCopiesAndOriginalMetadata() throws IOException { + State original = state(GROUP, "job"); + original.setProp("backend.handle", "{\"generation\":1,\"attempt\":1,\"id\":\"original\"}"); + original.setProp(TimingEvent.FlowEventConstants.CURRENT_GENERATION_FIELD, "3"); + original.setProp(TimingEvent.FlowEventConstants.CURRENT_ATTEMPTS_FIELD, "4"); + statuses(original); + doAnswer(invocation -> { + List states = invocation.getArgument(1); + Assert.assertEquals(states.size(), 1); + Assert.assertNotSame(states.get(0), original); + Assert.assertEquals(states.get(0).getProp("backend.handle"), original.getProp("backend.handle")); + Assert.assertEquals(states.get(0).getProp(TimingEvent.FlowEventConstants.CURRENT_GENERATION_FIELD), "3"); + Assert.expectThrows(UnsupportedOperationException.class, () -> states.add(new State())); + states.get(0).setProp("backend.handle", "changed by handler"); + ForceKillHandler.ActiveDagCheck active = invocation.getArgument(2); + Assert.assertFalse(active.isActive()); + return ForceKillHandler.Result.ACKNOWLEDGED; + }).when(this.handler).forceKill(any(), anyList(), any()); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + Assert.assertEquals(original.getProp("backend.handle"), "{\"generation\":1,\"attempt\":1,\"id\":\"original\"}"); + verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, true); + this.dagUtils.verifyNoInteractions(); + this.issues.verifyNoInteractions(); + verify(this.store, never()).getFailedDag(any()); + verify(this.store, never()).updateDagNode(any()); + verify(this.store, never()).deleteDag(any()); + } + + @DataProvider + public Object[][] absentOrOutOfScopeStates() { + return new Object[][] { + {Collections.emptyList()}, + {Collections.singletonList(state(JobStatusRetriever.NA_KEY, JobStatusRetriever.NA_KEY))}, + {Collections.singletonList(withFlowField(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD, "12340"))}, + {Collections.singletonList(withFlowField(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD, "other"))}, + {Collections.singletonList(withFlowField(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, "other"))} + }; + } + + @Test(dataProvider = "absentOrOutOfScopeStates") + public void testMissingOrOutOfScopeStateCannotTriggerHandler(List states) throws IOException { + when(this.retriever.getJobStatusStatesForFlowExecution(FLOW, GROUP, EXECUTION_ID)).thenReturn(states); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + verifyNoInteractions(this.handlerProvider, this.handler); + assertFailedWithoutCancellation(); + verify(this.store, never()).getFailedDag(any()); + } + + @Test + public void testJobSpecificScopeKeepsOnlyTargetAndSummary() throws IOException { + State summary = state(JobStatusRetriever.NA_KEY, JobStatusRetriever.NA_KEY); + statuses(summary, state(GROUP, "job"), state(GROUP, "other-job"), state("other-group", "job")); + doAnswer(invocation -> { + List states = invocation.getArgument(1); + Assert.assertEquals(states.size(), 2); + Assert.assertEquals(states.get(0).getProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD), JobStatusRetriever.NA_KEY); + Assert.assertEquals(states.get(1).getProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD), "job"); + Assert.assertEquals(states.get(1).getProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD), GROUP); + return ForceKillHandler.Result.ACKNOWLEDGED; + }).when(this.handler).forceKill(eq(action("job")), anyList(), any()); + + processor("job").process(this.store, this.metrics); + + verify(this.handler).forceKill(eq(action("job")), anyList(), any()); + this.dagUtils.verifyNoInteractions(); + } + + @Test + public void testUnacknowledgedHandlerDoesNotReportSuccess() throws IOException { + statuses(state(GROUP, "job")); + when(this.handler.forceKill(any(), anyList(), any())).thenReturn(ForceKillHandler.Result.NOT_ACKNOWLEDGED); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + assertFailedWithoutCancellation(); + } + + @Test + @SuppressWarnings("unchecked") + public void testDagReappearingAfterStatusReadUsesNormalCancellation() throws IOException { + Dag dag = mock(Dag.class); + statuses(state(GROUP, "job")); + when(this.store.getDag(any())).thenReturn(Optional.empty(), Optional.of(dag)); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + verifyNoInteractions(this.handlerProvider, this.handler); + this.dagUtils.verify(() -> DagProcUtils.cancelDag(dag, this.store)); + verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, true); + } + + @Test + @SuppressWarnings("unchecked") + public void testDagReappearingDuringHandlerUsesNormalCancellation() throws IOException { + Dag dag = mock(Dag.class); + statuses(state(GROUP, "job")); + when(this.store.getDag(any())).thenReturn(Optional.empty(), Optional.empty(), Optional.of(dag)); + doAnswer(invocation -> { + ForceKillHandler.ActiveDagCheck active = invocation.getArgument(2); + Assert.assertTrue(active.isActive()); + return ForceKillHandler.Result.ACTIVE_DAG_PRESENT; + }).when(this.handler).forceKill(any(), anyList(), any()); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + this.dagUtils.verify(() -> DagProcUtils.cancelDag(dag, this.store)); + verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, true); + } + + @Test + public void testReappearedDagDisappearingAgainDoesNotResumeRetainedStopping() throws IOException { + statuses(state(GROUP, "job")); + when(this.handler.forceKill(any(), anyList(), any())).thenReturn(ForceKillHandler.Result.ACTIVE_DAG_PRESENT); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + assertFailedWithoutCancellation(); + verify(this.handler).forceKill(any(), anyList(), any()); + } + + @Test + public void testReadFailurePropagatesWithoutCallingHandler() throws IOException { + IOException failure = new IOException("Status read failed"); + doThrow(failure).when(this.retriever).getJobStatusStatesForFlowExecution(FLOW, GROUP, EXECUTION_ID); + + Assert.assertSame(Assert.expectThrows(IOException.class, + () -> processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics)), failure); + + verifyNoInteractions(this.handlerProvider, this.handler); + assertFailedWithoutCancellation(); + } + + @Test + public void testHandlerFailurePropagatesWithoutSuccess() throws IOException { + statuses(state(GROUP, "job")); + IOException failure = new IOException("Backend unavailable"); + doThrow(failure).when(this.handler).forceKill(any(), anyList(), any()); + + Assert.assertSame(Assert.expectThrows(IOException.class, + () -> processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics)), failure); + + assertFailedWithoutCancellation(); + } + + @Test + public void testActiveDagCheckFailurePropagates() throws IOException { + statuses(state(GROUP, "job")); + IOException failure = new IOException("Active DAG read failed"); + when(this.store.getDag(any())).thenReturn(Optional.empty(), Optional.empty()).thenThrow(failure); + doAnswer(invocation -> { + ForceKillHandler.ActiveDagCheck active = invocation.getArgument(2); + active.isActive(); + return ForceKillHandler.Result.ACKNOWLEDGED; + }).when(this.handler).forceKill(any(), anyList(), any()); + + Assert.assertSame(Assert.expectThrows(IOException.class, + () -> processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics)), failure); + + assertFailedWithoutCancellation(); + } + + @Test + public void testNullHandlerResultIsNotSuccess() throws IOException { + statuses(state(GROUP, "job")); + when(this.handler.forceKill(any(), anyList(), any())).thenReturn(null); + + Assert.expectThrows(NullPointerException.class, + () -> processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics)); + + assertFailedWithoutCancellation(); + } + + private void assertFailedWithoutCancellation() { + verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, false); + verify(this.metrics, never()).markDagActionsAct(DagActionStore.DagActionType.KILL, true); + this.dagUtils.verifyNoInteractions(); + } + + private void statuses(State... states) throws IOException { + when(this.retriever.getJobStatusStatesForFlowExecution(FLOW, GROUP, EXECUTION_ID)).thenReturn(Arrays.asList(states)); + } + + private KillDagProc processor(String jobName) { + return new KillDagProc(task(jobName), ConfigFactory.empty(), this.handlerProvider, this.retrieverProvider); + } + + private KillDagTask task(String jobName) { + return new KillDagTask(action(jobName), null, this.store, this.metrics); + } + + private static DagActionStore.DagAction action(String jobName) { + return new DagActionStore.DagAction(GROUP, FLOW, EXECUTION_ID, jobName, DagActionStore.DagActionType.KILL); + } + + private static State withFlowField(String field, String value) { + State state = state(GROUP, "job"); + state.setProp(field, value); + return state; + } + + private static State state(String jobGroup, String jobName) { + State state = new State(); + state.setProp(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD, GROUP); + state.setProp(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, FLOW); + state.setProp(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD, Long.toString(EXECUTION_ID)); + state.setProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD, jobGroup); + state.setProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD, jobName); + state.setProp(JobStatusRetriever.EVENT_NAME_FIELD, ExecutionStatus.COMPLETE.name()); + return state; + } +} diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java new file mode 100644 index 00000000000..fbdaf68dd33 --- /dev/null +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.monitoring; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import com.google.common.base.Predicate; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.metastore.MysqlJobStatusStateStore; +import org.apache.gobblin.metastore.StateStore; +import org.apache.gobblin.metrics.RootMetricContext; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.powermock.reflect.Whitebox.setInternalState; + + +public class RawJobStatusRetrieverTest { + @Test + @SuppressWarnings("unchecked") + public void testGenericReaderUsesExactExecutionPrefixAndPreservesOpaqueMetadata() throws IOException { + StateStore store = mock(StateStore.class); + JobStatusRetriever retriever = mock(JobStatusRetriever.class, CALLS_REAL_METHODS); + doReturn(store).when(retriever).getStateStore(); + when(store.getTableNames(eq("group.flow"), any())).thenAnswer(invocation -> { + Predicate predicate = invocation.getArgument(1); + return Arrays.asList("1234.group.job.gst", "1234.NA.NA.gst", "12340.group.other.gst") + .stream().filter(predicate::apply).collect(Collectors.toList()); + }); + State job = new State(); + job.setProp("backend.handle", "opaque-original-handle"); + State summary = new State(); + when(store.getAll("group.flow", "1234.group.job.gst")).thenReturn(Collections.singletonList(job)); + when(store.getAll("group.flow", "1234.NA.NA.gst")).thenReturn(Collections.singletonList(summary)); + + List states = retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L); + + Assert.assertEquals(states, Arrays.asList(job, summary)); + Assert.assertEquals(states.get(0).getProp("backend.handle"), "opaque-original-handle"); + verify(store, never()).getAll("group.flow", "12340.group.other.gst"); + verify(store, never()).getAll("group.flow"); + } + + @Test + @SuppressWarnings("unchecked") + public void testGenericReaderPropagatesReadFailureInsteadOfReturningNoRecords() throws IOException { + StateStore store = mock(StateStore.class); + JobStatusRetriever retriever = mock(JobStatusRetriever.class, CALLS_REAL_METHODS); + doReturn(store).when(retriever).getStateStore(); + IOException failure = new IOException("Unavailable"); + doThrow(failure).when(store).getTableNames(eq("group.flow"), any()); + + Assert.assertSame(Assert.expectThrows(IOException.class, + () -> retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L)), failure); + } + + @Test + @SuppressWarnings("unchecked") + public void testMysqlReaderReusesRawStoreAndPropagatesIOException() throws IOException { + MysqlJobStatusStateStore store = mock(MysqlJobStatusStateStore.class); + MysqlJobStatusRetriever retriever = mock(MysqlJobStatusRetriever.class, CALLS_REAL_METHODS); + setInternalState(retriever, "stateStore", store); + setInternalState(retriever, "metricContext", RootMetricContext.get()); + State state = new State(); + state.setProp("backend.handle", "opaque"); + when(store.getAll("group.flow", 1234L)).thenReturn(Collections.singletonList(state)); + + Assert.assertEquals(retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L), + Collections.singletonList(state)); + verify(store).getAll("group.flow", 1234L); + + IOException failure = new IOException("Database unavailable"); + doThrow(failure).when(store).getAll("group.flow", 1234L); + Assert.assertSame(Assert.expectThrows(IOException.class, + () -> retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L)), failure); + } +} From 9c72ce8f38ec26e944945d315adf398a71e6452a Mon Sep 17 00:00:00 2001 From: Siddhi Agrawal <101979598+Siddhi-agg@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:51:44 +0530 Subject: [PATCH 2/3] Fix retained-state KILL CI and review issues Declare the metastore Mockito test dependency so the CI test and static-check jobs can compile their shared test fixtures. Reject raw-state reads with an IOException when the retriever has no state store. Snapshot merged properties before invoking the kill handler so neither common nor specific properties share mutable maps. Cover unsupported local-file retrieval and property isolation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- gobblin-metastore/build.gradle | 1 + .../service/monitoring/JobStatusRetriever.java | 5 +++++ .../modules/orchestration/proc/KillDagProc.java | 2 +- .../proc/KillDagProcForceKillTest.java | 8 ++++++++ .../monitoring/RawJobStatusRetrieverTest.java | 16 ++++++++++++++++ 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/gobblin-metastore/build.gradle b/gobblin-metastore/build.gradle index 063a0c0a53d..769bd27fea6 100644 --- a/gobblin-metastore/build.gradle +++ b/gobblin-metastore/build.gradle @@ -39,6 +39,7 @@ dependencies { runtimeOnly externalDependency.mysqlConnector testCompile externalDependency.testng + testCompile externalDependency.mockito testCompile externalDependency.embeddedProcess testCompile externalDependency.testContainers testCompile externalDependency.testContainersMysql diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java b/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java index 7ba5f92c527..58b535bc59e 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java @@ -94,6 +94,8 @@ public abstract Iterator getJobStatusesForFlowExecution(String flowNa * execution handles: storage keys alone do not establish the identity of the serialized contents. * *

Unlike status-display APIs, read failures propagate rather than appearing as an empty result.

+ * + * @throws IOException if this retriever has no state store or retained state cannot be read */ public List getJobStatusStatesForFlowExecution(String flowName, String flowGroup, long flowExecutionId) throws IOException { @@ -103,6 +105,9 @@ public List getJobStatusStatesForFlowExecution(String flowName, String fl String storeName = flowGroup + separator + flowName; String tablePrefix = flowExecutionId + separator; StateStore stateStore = getStateStore(); + if (stateStore == null) { + throw new IOException("Raw job status retrieval requires a state store: " + getClass().getName()); + } List states = new ArrayList<>(); for (String tableName : stateStore.getTableNames(storeName, name -> name.startsWith(tablePrefix))) { states.addAll(stateStore.getAll(storeName, tableName)); diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java index 2e5ba837f23..cd668d001d8 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java @@ -122,7 +122,7 @@ private void forceKillFromJobStatus(DagManagementStateStore store, DagProcessing if (flowSummary || !this.shouldKillSpecificJob || (getDagNodeId().getJobGroup().equals(state.getProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD)) && getDagNodeId().getJobName().equals(state.getProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD)))) { - scopedStates.add(new State(state)); + scopedStates.add(new State(state.getProperties())); hasJob |= !flowSummary; } } diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java index 64ec85d6518..79205860af0 100644 --- a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java @@ -140,6 +140,9 @@ public void testAbsentHandlerPreservesMissingDagBehavior() throws IOException { @Test public void testAcknowledgedFallbackGetsImmutableDefensiveCopiesAndOriginalMetadata() throws IOException { State original = state(GROUP, "job"); + original.getCommonProperties().setProperty("backend.common.handle", "original-common-handle"); + original.getCommonProperties().setProperty("backend.target", "original-target"); + original.getCommonProperties().setProperty("backend.handle", "overridden-common-handle"); original.setProp("backend.handle", "{\"generation\":1,\"attempt\":1,\"id\":\"original\"}"); original.setProp(TimingEvent.FlowEventConstants.CURRENT_GENERATION_FIELD, "3"); original.setProp(TimingEvent.FlowEventConstants.CURRENT_ATTEMPTS_FIELD, "4"); @@ -149,9 +152,13 @@ public void testAcknowledgedFallbackGetsImmutableDefensiveCopiesAndOriginalMetad Assert.assertEquals(states.size(), 1); Assert.assertNotSame(states.get(0), original); Assert.assertEquals(states.get(0).getProp("backend.handle"), original.getProp("backend.handle")); + Assert.assertEquals(states.get(0).getProp("backend.common.handle"), "original-common-handle"); Assert.assertEquals(states.get(0).getProp(TimingEvent.FlowEventConstants.CURRENT_GENERATION_FIELD), "3"); Assert.expectThrows(UnsupportedOperationException.class, () -> states.add(new State())); states.get(0).setProp("backend.handle", "changed by handler"); + states.get(0).getCommonProperties().setProperty("backend.common.handle", "changed by handler"); + original.getCommonProperties().setProperty("backend.target", "changed in source"); + Assert.assertEquals(states.get(0).getProp("backend.target"), "original-target"); ForceKillHandler.ActiveDagCheck active = invocation.getArgument(2); Assert.assertFalse(active.isActive()); return ForceKillHandler.Result.ACKNOWLEDGED; @@ -160,6 +167,7 @@ public void testAcknowledgedFallbackGetsImmutableDefensiveCopiesAndOriginalMetad processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); Assert.assertEquals(original.getProp("backend.handle"), "{\"generation\":1,\"attempt\":1,\"id\":\"original\"}"); + Assert.assertEquals(original.getProp("backend.common.handle"), "original-common-handle"); verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, true); this.dagUtils.verifyNoInteractions(); this.issues.verifyNoInteractions(); diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java index fbdaf68dd33..e0bb0c29d86 100644 --- a/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java @@ -24,6 +24,7 @@ import java.util.stream.Collectors; import com.google.common.base.Predicate; +import com.typesafe.config.ConfigFactory; import org.testng.Assert; import org.testng.annotations.Test; @@ -32,6 +33,8 @@ import org.apache.gobblin.metastore.MysqlJobStatusStateStore; import org.apache.gobblin.metastore.StateStore; import org.apache.gobblin.metrics.RootMetricContext; +import org.apache.gobblin.runtime.spec_executorInstance.LocalFsSpecProducer; +import org.apache.gobblin.runtime.troubleshooter.MultiContextIssueRepository; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -84,6 +87,19 @@ public void testGenericReaderPropagatesReadFailureInsteadOfReturningNoRecords() () -> retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L)), failure); } + @Test + public void testLocalFsReaderRejectsRawRetrievalWithoutStateStore() { + JobStatusRetriever retriever = new LocalFsJobStatusRetriever(ConfigFactory.parseMap(Collections.singletonMap( + LocalFsJobStatusRetriever.CONF_PREFIX + LocalFsSpecProducer.LOCAL_FS_PRODUCER_PATH_KEY, "unused")), + mock(MultiContextIssueRepository.class)); + + IOException failure = Assert.expectThrows(IOException.class, + () -> retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L)); + + Assert.assertTrue(failure.getMessage().contains("Raw job status retrieval requires a state store")); + Assert.assertTrue(failure.getMessage().contains(LocalFsJobStatusRetriever.class.getName())); + } + @Test @SuppressWarnings("unchecked") public void testMysqlReaderReusesRawStoreAndPropagatesIOException() throws IOException { From b990965598496058cc00af46f4c5cb9d5e77130f Mon Sep 17 00:00:00 2001 From: Siddhi Agrawal Date: Thu, 24 Sep 2026 12:59:54 +0530 Subject: [PATCH 3/3] [GOBBLIN-2270] Fix FORCE_KILL_HANDLER_CLASS_KEY config name to GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY --- .../java/org/apache/gobblin/service/ServiceConfigKeys.java | 2 +- .../service/modules/core/GobblinServiceGuiceModule.java | 4 ++-- .../core/GobblinServiceForceKillConfigurationTest.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java index 729621b3551..2d6f013533a 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java @@ -37,7 +37,7 @@ public class ServiceConfigKeys { public static final String GOBBLIN_SERVICE_FLOWSPEC = GOBBLIN_SERVICE_PREFIX + "flowSpec"; public static final String GOBBLIN_SERVICE_FLOWGRAPH_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "flowGraph.class"; public static final String GOBBLIN_SERVICE_FLOWGRAPH_HELPER_KEY = GOBBLIN_SERVICE_PREFIX + "flowGraphHelper.class"; - public static final String FORCE_KILL_HANDLER_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "forceKillHandler.class"; + public static final String GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "forceKillHandler.class"; // Flow Compiler Keys public static final String GOBBLIN_SERVICE_FLOWCOMPILER_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "flowCompiler.class"; diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java index 19a17b1cc39..4967725f624 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java @@ -261,9 +261,9 @@ protected static Class getClassByNameOrAlias(Class baseClass static void configureForceKillHandler(Binder binder, Config config) { OptionalBinder optional = OptionalBinder.newOptionalBinder(binder, ForceKillHandler.class); - if (config.hasPath(ServiceConfigKeys.FORCE_KILL_HANDLER_CLASS_KEY)) { + if (config.hasPath(ServiceConfigKeys.GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY)) { optional.setBinding().to(getClassByNameOrAlias(ForceKillHandler.class, config, - ServiceConfigKeys.FORCE_KILL_HANDLER_CLASS_KEY, null)).in(Singleton.class); + ServiceConfigKeys.GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY, null)).in(Singleton.class); } } diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java index 913c34fbea1..0e8da8f74cd 100644 --- a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java @@ -51,7 +51,7 @@ public void testNoConfiguredBackendLeavesOptionalAbsent() { @Test public void testConfiguredBackendIsInjectedAsOneSingletonForManagerAndFactory() { - Config config = ConfigFactory.empty().withValue(ServiceConfigKeys.FORCE_KILL_HANDLER_CLASS_KEY, + Config config = ConfigFactory.empty().withValue(ServiceConfigKeys.GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY, ConfigValueFactory.fromAnyRef(TestHandler.class.getName())); Injector injector = injector(config);