diff --git a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java index 35bbd8c2e82..2d6f013533a 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java @@ -37,6 +37,7 @@ public class ServiceConfigKeys { public static final String GOBBLIN_SERVICE_FLOWSPEC = GOBBLIN_SERVICE_PREFIX + "flowSpec"; public static final String GOBBLIN_SERVICE_FLOWGRAPH_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "flowGraph.class"; public static final String GOBBLIN_SERVICE_FLOWGRAPH_HELPER_KEY = GOBBLIN_SERVICE_PREFIX + "flowGraphHelper.class"; + public static final String GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "forceKillHandler.class"; // Flow Compiler Keys public static final String GOBBLIN_SERVICE_FLOWCOMPILER_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "flowCompiler.class"; diff --git a/gobblin-metastore/build.gradle b/gobblin-metastore/build.gradle index 063a0c0a53d..769bd27fea6 100644 --- a/gobblin-metastore/build.gradle +++ b/gobblin-metastore/build.gradle @@ -39,6 +39,7 @@ dependencies { runtimeOnly externalDependency.mysqlConnector testCompile externalDependency.testng + testCompile externalDependency.mockito testCompile externalDependency.embeddedProcess testCompile externalDependency.testContainers testCompile externalDependency.testContainersMysql diff --git a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlJobStatusStateStore.java b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlJobStatusStateStore.java index 8715d9f388b..04164430438 100644 --- a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlJobStatusStateStore.java +++ b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlJobStatusStateStore.java @@ -68,7 +68,8 @@ public MysqlJobStatusStateStore(DataSource dataSource, String stateStoreTableNam * @throws IOException in case of failures */ public List getAll(String storeName, long flowExecutionId) throws IOException { - return getAll(storeName, flowExecutionId + "%", JobStateSearchColumns.TABLE_NAME_ONLY); + return getAll(storeName, flowExecutionId + ServiceConfigKeys.STATE_STORE_KEY_SEPARATION_CHARACTER + "%", + JobStateSearchColumns.TABLE_NAME_ONLY); } /** diff --git a/gobblin-metastore/src/test/java/org/apache/gobblin/metastore/MysqlJobStatusStateStoreFlowScopeTest.java b/gobblin-metastore/src/test/java/org/apache/gobblin/metastore/MysqlJobStatusStateStoreFlowScopeTest.java new file mode 100644 index 00000000000..925d797228a --- /dev/null +++ b/gobblin-metastore/src/test/java/org/apache/gobblin/metastore/MysqlJobStatusStateStoreFlowScopeTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.metastore; + +import java.io.IOException; +import java.util.Collections; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.State; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + + +public class MysqlJobStatusStateStoreFlowScopeTest { + @Test + @SuppressWarnings("unchecked") + public void testExecutionPrefixIncludesSeparator() throws IOException { + MysqlJobStatusStateStore store = mock(MysqlJobStatusStateStore.class, CALLS_REAL_METHODS); + doReturn(Collections.emptyList()).when(store).getAll(eq("group.flow"), anyString(), + eq(MysqlStateStore.JobStateSearchColumns.TABLE_NAME_ONLY)); + + Assert.assertTrue(store.getAll("group.flow", 1234L).isEmpty()); + + verify(store).getAll("group.flow", "1234.%", MysqlStateStore.JobStateSearchColumns.TABLE_NAME_ONLY); + } +} diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java b/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java index c82e806c641..58b535bc59e 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java @@ -17,6 +17,7 @@ package org.apache.gobblin.service.monitoring; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -49,6 +50,7 @@ import org.apache.gobblin.runtime.troubleshooter.TroubleshooterException; import org.apache.gobblin.runtime.troubleshooter.TroubleshooterUtils; import org.apache.gobblin.service.ExecutionStatus; +import org.apache.gobblin.service.ServiceConfigKeys; import org.apache.gobblin.util.ConfigUtils; @@ -86,6 +88,33 @@ public abstract Iterator getJobStatusesForFlowExecution(String flowNa public abstract Iterator getJobStatusesForFlowExecution(String flowName, String flowGroup, long flowExecutionId, String jobName, String jobGroup); + /** + * Read retained states without discarding backend metadata during conversion to {@link JobStatus}. + * Includes the flow-summary record, if present. Callers must validate the decoded identity before using + * execution handles: storage keys alone do not establish the identity of the serialized contents. + * + *

Unlike status-display APIs, read failures propagate rather than appearing as an empty result.

+ * + * @throws IOException if this retriever has no state store or retained state cannot be read + */ + public List getJobStatusStatesForFlowExecution(String flowName, String flowGroup, long flowExecutionId) + throws IOException { + Objects.requireNonNull(flowName, "flowName"); + Objects.requireNonNull(flowGroup, "flowGroup"); + String separator = ServiceConfigKeys.STATE_STORE_KEY_SEPARATION_CHARACTER; + String storeName = flowGroup + separator + flowName; + String tablePrefix = flowExecutionId + separator; + StateStore stateStore = getStateStore(); + if (stateStore == null) { + throw new IOException("Raw job status retrieval requires a state store: " + getClass().getName()); + } + List states = new ArrayList<>(); + for (String tableName : stateStore.getTableNames(storeName, name -> name.startsWith(tablePrefix))) { + states.addAll(stateStore.getAll(storeName, tableName)); + } + return states; + } + /** * Get the latest {@link FlowStatus}es of executions of flows belonging to this flow group. Currently, latest flow execution * is decided by comparing {@link JobStatus#getFlowExecutionId()}. diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java index 9e2ba6f79e6..4967725f624 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java @@ -68,6 +68,7 @@ import org.apache.gobblin.service.modules.orchestration.DagTaskStream; import org.apache.gobblin.service.modules.orchestration.FlowLaunchHandler; import org.apache.gobblin.service.modules.orchestration.FlowLaunchMultiActiveLeaseArbiterFactory; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; import org.apache.gobblin.service.modules.orchestration.MultiActiveLeaseArbiter; import org.apache.gobblin.service.modules.orchestration.MySqlDagManagementStateStore; import org.apache.gobblin.service.modules.orchestration.MysqlDagActionStore; @@ -164,6 +165,7 @@ binding time (optionally bound classes cannot have names associated with them), binder.bind(DagManagement.class).to(DagManagementTaskStreamImpl.class); binder.bind(DagManagementStateStore.class).to(MySqlDagManagementStateStore.class); binder.bind(DagTaskStream.class).to(DagManagementTaskStreamImpl.class); + configureForceKillHandler(binder, serviceConfig.getInnerConfig()); binder.bind(DagProcFactory.class); binder.bind(DagProcessingEngine.class); binder.bind(DagProcessingEngineMetrics.class); @@ -257,6 +259,14 @@ protected static Class getClassByNameOrAlias(Class baseClass } } + static void configureForceKillHandler(Binder binder, Config config) { + OptionalBinder optional = OptionalBinder.newOptionalBinder(binder, ForceKillHandler.class); + if (config.hasPath(ServiceConfigKeys.GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY)) { + optional.setBinding().to(getClassByNameOrAlias(ForceKillHandler.class, config, + ServiceConfigKeys.GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY, null)).in(Singleton.class); + } + } + public static class EmbeddedRestliServerProvider implements Provider { Injector injector; diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java index 9bab575350c..9165783972d 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java @@ -38,6 +38,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.base.Optional; +import com.google.common.util.concurrent.AbstractIdleService; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; @@ -71,6 +73,7 @@ import org.apache.gobblin.service.ServiceConfigKeys; import org.apache.gobblin.service.modules.db.ServiceDatabaseManager; import org.apache.gobblin.service.modules.orchestration.DagProcessingEngine; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; import org.apache.gobblin.service.modules.orchestration.Orchestrator; import org.apache.gobblin.service.modules.orchestration.UserQuotaManager; import org.apache.gobblin.service.modules.restli.FlowConfigsV2ResourceHandler; @@ -174,6 +177,9 @@ public class GobblinServiceManager implements ApplicationLauncher { @Inject(optional = true) protected DagProcessingEngine dagProcessingEngine; + @Inject + private Optional forceKillHandler = Optional.absent(); + @Inject protected GobblinServiceManager(GobblinServiceConfiguration configuration) throws Exception { this.configuration = Objects.requireNonNull(configuration); @@ -256,6 +262,19 @@ private void registerServicesInLauncher(){ } this.serviceLauncher.addService(dagProcessingEngine); + if (this.forceKillHandler.isPresent()) { + // The launcher's own failure/shutdown paths must also close the backend, not just this manager's stop(). + this.serviceLauncher.addService(new AbstractIdleService() { + @Override + protected void startUp() { + } + + @Override + protected void shutDown() { + forceKillHandler.get().close(); + } + }); + } this.serviceLauncher.addService(databaseManager); this.serviceLauncher.addService(issueRepository); @@ -297,6 +316,13 @@ private void configureServices(){ @Override public void start() throws ApplicationException { + try (ForceKillStartupGuard guard = new ForceKillStartupGuard(this.forceKillHandler.orNull())) { + startServices(); + guard.started = true; + } + } + + private void startServices() throws ApplicationException { LOGGER.info("[Init] Starting the Gobblin Service Manager"); configureServices(); @@ -346,7 +372,7 @@ public void stop() throws ApplicationException { LOGGER.info("Stopping the Gobblin Service Manager"); this.stopInProgress = true; - try { + try (ForceKillHandler handler = this.forceKillHandler.orNull()) { // Stop announcing GaaS instances to d2 when services are stopped this.d2Announcer.markDownServer(); this.serviceLauncher.stop(); @@ -357,7 +383,25 @@ public void stop() throws ApplicationException { @Override public void close() throws IOException { - this.serviceLauncher.close(); + try (ForceKillHandler handler = this.forceKillHandler.orNull()) { + this.serviceLauncher.close(); + } + } + + static final class ForceKillStartupGuard implements AutoCloseable { + private final ForceKillHandler handler; + boolean started; + + ForceKillStartupGuard(ForceKillHandler handler) { + this.handler = handler; + } + + @Override + public void close() { + if (!this.started && this.handler != null) { + this.handler.close(); + } + } } private static String getServiceId(CommandLine cmd) { diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcFactory.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcFactory.java index 10b9bf27a81..98843510582 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcFactory.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcFactory.java @@ -17,7 +17,11 @@ package org.apache.gobblin.service.modules.orchestration; +import java.util.Objects; + +import com.google.common.base.Optional; import com.google.inject.Inject; +import com.google.inject.Provider; import com.google.inject.Singleton; import com.typesafe.config.Config; @@ -37,6 +41,7 @@ import org.apache.gobblin.service.modules.orchestration.task.ReevaluateDagTask; import org.apache.gobblin.service.modules.orchestration.task.ResumeDagTask; import org.apache.gobblin.service.modules.utils.FlowCompilationValidationHelper; +import org.apache.gobblin.service.monitoring.JobStatusRetriever; /** * {@link DagTaskVisitor} for transforming a specific {@link DagTask} derived class to its companion {@link DagProc} derived class. @@ -50,11 +55,21 @@ public class DagProcFactory implements DagTaskVisitor> { private final Config config; private final FlowCompilationValidationHelper flowCompilationValidationHelper; + private final Optional> forceKillHandler; + private final Provider jobStatusRetriever; - @Inject public DagProcFactory(Config config, FlowCompilationValidationHelper flowCompilationValidationHelper) { + this(config, flowCompilationValidationHelper, Optional.absent(), null); + } + + @Inject + public DagProcFactory(Config config, FlowCompilationValidationHelper flowCompilationValidationHelper, + Optional> forceKillHandler, Provider jobStatusRetriever) { this.config = config; this.flowCompilationValidationHelper = flowCompilationValidationHelper; + this.forceKillHandler = Objects.requireNonNull(forceKillHandler, "forceKillHandler"); + this.jobStatusRetriever = forceKillHandler.isPresent() + ? Objects.requireNonNull(jobStatusRetriever, "jobStatusRetriever") : jobStatusRetriever; } @Override @@ -79,6 +94,9 @@ public ReevaluateDagProc meet(ReevaluateDagTask reEvaluateDagTask) { @Override public KillDagProc meet(KillDagTask killDagTask) { + if (this.forceKillHandler.isPresent()) { + return new KillDagProc(killDagTask, this.config, this.forceKillHandler.get(), this.jobStatusRetriever); + } return new KillDagProc(killDagTask, this.config); } @@ -87,4 +105,3 @@ public ResumeDagProc meet(ResumeDagTask resumeDagTask) { return new ResumeDagProc(resumeDagTask, this.config); } } - diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ForceKillHandler.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ForceKillHandler.java new file mode 100644 index 00000000000..66914a6cabe --- /dev/null +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ForceKillHandler.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; + +import org.apache.gobblin.configuration.State; + + +/** + * Backend-specific handling of an existing KILL action after its active DAG has been removed. + * + *

The supplied states are the only retained execution metadata source. Implementations must validate backend + * handles, destinations and their original submission identities, without consulting archived DAGs or guessing + * destinations from current routing. A flow-summary state is context, not an execution to stop. Implementations + * must not rewrite terminal job status or emit normal flow/job cancellation events. Bind implementations as + * singletons so the service manager can close the same instance used by KILL processing.

+ */ +@FunctionalInterface +public interface ForceKillHandler extends Closeable { + enum Result { + /** All identified stop requests were acknowledged; this does not establish physical process termination. */ + ACKNOWLEDGED, + /** No execution could be resolved, or at least one stop was not acknowledged. */ + NOT_ACKNOWLEDGED, + /** An active DAG was observed; stop retained processing and let normal DAG cancellation take over. */ + ACTIVE_DAG_PRESENT + } + + @FunctionalInterface + interface ActiveDagCheck { + boolean isActive() throws IOException; + } + + /** + * Attempt bounded, backend-specific stopping using defensive copies of the retained job states. + * + * @param action the exact flow execution and optional job requested for cancellation + * @param jobStates scoped job states and, if retained, the flow-summary state; the list is unmodifiable + * @param activeDagCheck must be checked before each backend stop; return {@link Result#ACTIVE_DAG_PRESENT} if true. + * This reduces, but does not atomically exclude, concurrent resume. + * @throws IOException when the operation cannot be completed; failures must not be reported as acknowledgements + */ + Result forceKill(DagActionStore.DagAction action, List jobStates, ActiveDagCheck activeDagCheck) + throws IOException; + + /** Stop accepting work and release local resources without waiting indefinitely for remote calls. Must be idempotent. */ + @Override + default void close() { + } +} diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java index d181c2996db..cd668d001d8 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProc.java @@ -18,20 +18,28 @@ package org.apache.gobblin.service.modules.orchestration.proc; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; import java.util.Optional; +import com.google.inject.Provider; import com.typesafe.config.Config; import lombok.extern.slf4j.Slf4j; +import org.apache.gobblin.configuration.State; import org.apache.gobblin.metrics.event.TimingEvent; import org.apache.gobblin.runtime.troubleshooter.IssueSeverity; import org.apache.gobblin.service.modules.flowgraph.Dag; import org.apache.gobblin.service.modules.orchestration.DagActionStore; import org.apache.gobblin.service.modules.orchestration.DagManagementStateStore; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; import org.apache.gobblin.service.modules.orchestration.task.DagProcessingEngineMetrics; import org.apache.gobblin.service.modules.orchestration.task.KillDagTask; import org.apache.gobblin.service.modules.spec.JobExecutionPlan; +import org.apache.gobblin.service.monitoring.JobStatusRetriever; /** @@ -41,10 +49,20 @@ @Slf4j public class KillDagProc extends DagProc>> { private final boolean shouldKillSpecificJob; + private final Provider forceKillHandler; + private final Provider jobStatusRetriever; public KillDagProc(KillDagTask killDagTask, Config config) { + this(killDagTask, config, null, null); + } + + public KillDagProc(KillDagTask killDagTask, Config config, Provider forceKillHandler, + Provider jobStatusRetriever) { super(killDagTask, config); this.shouldKillSpecificJob = !getDagNodeId().getJobName().equals(DagActionStore.NO_JOB_NAME_DEFAULT); + this.forceKillHandler = forceKillHandler; + this.jobStatusRetriever = forceKillHandler == null ? jobStatusRetriever + : Objects.requireNonNull(jobStatusRetriever, "jobStatusRetriever"); } @Override @@ -59,6 +77,10 @@ protected void act(DagManagementStateStore dagManagementStateStore, Optional>"); if (!dag.isPresent()) { + if (this.forceKillHandler != null) { + forceKillFromJobStatus(dagManagementStateStore, dagProcEngineMetrics); + return; + } dagProcEngineMetrics.markDagActionsAct(getDagActionType(), false); log.error("Did not find Dag with id {}, it might be already cancelled/finished and thus cleaned up from the store.", getDagId()); OrchestratorIssueEmitter.emitFlowIssue(eventSubmitter, getDagId(), IssueSeverity.WARN, @@ -84,4 +106,71 @@ protected void act(DagManagementStateStore dagManagementStateStore, Optional retainedStates = this.jobStatusRetriever.get().getJobStatusStatesForFlowExecution( + getDagId().getFlowName(), getDagId().getFlowGroup(), getDagId().getFlowExecutionId()); + List scopedStates = new ArrayList<>(); + boolean hasJob = false; + for (State state : retainedStates) { + if (!matchesFlow(state)) { + log.warn("Ignoring out-of-scope job status during force kill of {}", getDagId()); + continue; + } + boolean flowSummary = isFlowSummary(state); + if (flowSummary || !this.shouldKillSpecificJob + || (getDagNodeId().getJobGroup().equals(state.getProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD)) + && getDagNodeId().getJobName().equals(state.getProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD)))) { + scopedStates.add(new State(state.getProperties())); + hasJob |= !flowSummary; + } + } + + Optional> activeDag = store.getDag(getDagId()); + if (activeDag.isPresent()) { + act(store, activeDag, metrics); + return; + } + if (!hasJob) { + reportForceKillFailure(metrics, "No retained job status found for force kill"); + return; + } + + ForceKillHandler.Result result = Objects.requireNonNull(this.forceKillHandler.get().forceKill( + this.dagTask.getDagAction(), Collections.unmodifiableList(scopedStates), + () -> store.getDag(getDagId()).isPresent()), "forceKill result"); + if (result == ForceKillHandler.Result.ACTIVE_DAG_PRESENT) { + activeDag = store.getDag(getDagId()); + if (activeDag.isPresent()) { + act(store, activeDag, metrics); + } else { + reportForceKillFailure(metrics, "Active DAG changed during force kill; retained stopping was abandoned"); + } + } else if (result == ForceKillHandler.Result.ACKNOWLEDGED) { + metrics.markDagActionsAct(getDagActionType(), true); + log.info("Force-kill requests acknowledged for {}; process termination is not confirmed", getDagId()); + } else { + reportForceKillFailure(metrics, "Force kill did not acknowledge all retained executions"); + } + } + + private boolean matchesFlow(State state) { + return getDagId().getFlowGroup().equals(state.getProp(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD)) + && getDagId().getFlowName().equals(state.getProp(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD)) + && Long.toString(getDagId().getFlowExecutionId()) + .equals(state.getProp(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD)); + } + + private static boolean isFlowSummary(State state) { + return JobStatusRetriever.NA_KEY.equals(state.getProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD)) + && JobStatusRetriever.NA_KEY.equals(state.getProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD)); + } + + private void reportForceKillFailure(DagProcessingEngineMetrics metrics, String message) { + metrics.markDagActionsAct(getDagActionType(), false); + log.warn("{}: {} (node: {})", message, getDagId(), getDagNodeId()); + OrchestratorIssueEmitter.emitFlowIssue(eventSubmitter, getDagId(), IssueSeverity.WARN, + message + ": " + getDagNodeId()); + } } diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/MysqlJobStatusRetriever.java b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/MysqlJobStatusRetriever.java index 5678fef6aab..4fba36f6698 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/MysqlJobStatusRetriever.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/MysqlJobStatusRetriever.java @@ -76,6 +76,15 @@ public Iterator getJobStatusesForFlowExecution(String flowName, Strin return asJobStatuses(jobStatusStates); } + @Override + public List getJobStatusStatesForFlowExecution(String flowName, String flowGroup, long flowExecutionId) + throws IOException { + String storeName = KafkaJobStatusMonitor.jobStatusStoreName(flowGroup, flowName); + try (Timer.Context context = this.metricContext.contextAwareTimer(GET_LATEST_FLOW_STATUS_METRIC).time()) { + return this.stateStore.getAll(storeName, flowExecutionId); + } + } + @Override public Iterator getJobStatusesForFlowExecution(String flowName, String flowGroup, long flowExecutionId, String jobName, String jobGroup) { diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java new file mode 100644 index 00000000000..0e8da8f74cd --- /dev/null +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillConfigurationTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gobblin.service.modules.core; + +import java.util.List; + +import javax.inject.Inject; + +import com.google.common.base.Optional; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provider; +import com.google.inject.TypeLiteral; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigValueFactory; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.service.ServiceConfigKeys; +import org.apache.gobblin.service.modules.orchestration.DagActionStore; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; + + +public class GobblinServiceForceKillConfigurationTest { + @Test + public void testNoConfiguredBackendLeavesOptionalAbsent() { + Injector injector = injector(ConfigFactory.empty()); + + Assert.assertFalse(injector.getInstance(Key.get(new TypeLiteral>() { })).isPresent()); + Assert.assertFalse(injector.getInstance(Key.get( + new TypeLiteral>>() { })).isPresent()); + } + + @Test + public void testConfiguredBackendIsInjectedAsOneSingletonForManagerAndFactory() { + Config config = ConfigFactory.empty().withValue(ServiceConfigKeys.GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY, + ConfigValueFactory.fromAnyRef(TestHandler.class.getName())); + Injector injector = injector(config); + + ForceKillHandler managerHandler = + injector.getInstance(Key.get(new TypeLiteral>() { })).get(); + Provider factoryProvider = + injector.getInstance(Key.get(new TypeLiteral>>() { })).get(); + + Assert.assertSame(managerHandler, factoryProvider.get()); + Assert.assertSame(managerHandler, factoryProvider.get()); + Assert.assertSame(((TestHandler) managerHandler).config, config); + } + + private static Injector injector(Config config) { + return Guice.createInjector(binder -> { + binder.requireExplicitBindings(); + binder.bind(Config.class).toInstance(config); + GobblinServiceGuiceModule.configureForceKillHandler(binder, config); + }); + } + + public static class TestHandler implements ForceKillHandler { + final Config config; + + @Inject + public TestHandler(Config config) { + this.config = config; + } + + @Override + public Result forceKill(DagActionStore.DagAction action, List states, ActiveDagCheck guard) { + return Result.ACKNOWLEDGED; + } + } +} diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillLifecycleTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillLifecycleTest.java new file mode 100644 index 00000000000..79b62ed8955 --- /dev/null +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceForceKillLifecycleTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gobblin.service.modules.core; + +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Optional; +import com.google.common.util.concurrent.Service; + +import org.mockito.ArgumentCaptor; +import org.testng.Assert; +import org.testng.annotations.Test; + +import org.apache.gobblin.runtime.app.ServiceBasedAppLauncher; +import org.apache.gobblin.runtime.spec_catalog.FlowCatalog; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; + +import static org.mockito.Mockito.*; +import static org.powermock.reflect.Whitebox.setInternalState; + + +public class GobblinServiceForceKillLifecycleTest { + @Test + public void testManagerStopClosesBackendAndIsIdempotent() throws Exception { + Fixture fixture = new Fixture(); + + fixture.manager.stop(); + fixture.manager.stop(); + + verify(fixture.launcher).stop(); + verify(fixture.handler).close(); + } + + @Test + public void testBackendClosesWhenShutdownFailsBeforeLauncher() { + Fixture fixture = new Fixture(); + IllegalStateException failure = new IllegalStateException("d2 failure"); + doThrow(failure).when(fixture.manager.d2Announcer).markDownServer(); + + Assert.assertSame(Assert.expectThrows(IllegalStateException.class, fixture.manager::stop), failure); + verify(fixture.handler).close(); + verifyNoInteractions(fixture.launcher); + } + + @Test + public void testBackendClosesWhenLauncherCloseFails() throws Exception { + Fixture fixture = new Fixture(); + IOException failure = new IOException("launcher close failed"); + doThrow(failure).when(fixture.launcher).close(); + + Assert.assertSame(Assert.expectThrows(IOException.class, fixture.manager::close), failure); + verify(fixture.handler).close(); + } + + @Test + public void testStartupFailureClosesBackendWithoutHidingOriginalFailure() { + Fixture fixture = new Fixture(); + IllegalStateException failure = new IllegalStateException("configuration failed"); + when(fixture.configuration.isRestLIServerEnabled()).thenThrow(failure); + IllegalStateException closeFailure = new IllegalStateException("close failed"); + doThrow(closeFailure).when(fixture.handler).close(); + + Assert.assertSame(Assert.expectThrows(IllegalStateException.class, fixture.manager::start), failure); + Assert.assertEquals(failure.getSuppressed(), new Throwable[] {closeFailure}); + verify(fixture.handler).close(); + } + + @Test + public void testLauncherOwnShutdownPathAlsoClosesBackend() throws Exception { + Fixture fixture = new Fixture(); + fixture.manager.flowCatalog = mock(FlowCatalog.class); + IllegalStateException failure = new IllegalStateException("launcher start failed"); + doThrow(failure).when(fixture.launcher).start(); + Assert.assertSame(Assert.expectThrows(IllegalStateException.class, fixture.manager::start), failure); + verify(fixture.handler).close(); + + ArgumentCaptor registered = ArgumentCaptor.forClass(Service.class); + verify(fixture.launcher, atLeastOnce()).addService(registered.capture()); + Service backendLifecycle = registered.getAllValues().stream().filter(Objects::nonNull) + .filter(service -> service != fixture.manager.flowCatalog).findFirst().get(); + clearInvocations(fixture.handler); + + backendLifecycle.startAsync().awaitRunning(5, TimeUnit.SECONDS); + backendLifecycle.stopAsync().awaitTerminated(5, TimeUnit.SECONDS); + + verify(fixture.handler).close(); + } + + @Test + public void testNoBackendPreservesManagerShutdown() throws Exception { + Fixture fixture = new Fixture(); + setInternalState(fixture.manager, "forceKillHandler", Optional.absent()); + + fixture.manager.stop(); + fixture.manager.close(); + + verify(fixture.launcher).stop(); + verify(fixture.launcher).close(); + verifyNoInteractions(fixture.handler); + } + + @Test + public void testSuccessfulStartupGuardDoesNotCloseBackend() { + ForceKillHandler handler = mock(ForceKillHandler.class); + try (GobblinServiceManager.ForceKillStartupGuard guard = new GobblinServiceManager.ForceKillStartupGuard(handler)) { + guard.started = true; + } + verifyNoInteractions(handler); + } + + private static final class Fixture { + final ForceKillHandler handler = mock(ForceKillHandler.class); + final ServiceBasedAppLauncher launcher = mock(ServiceBasedAppLauncher.class); + final GobblinServiceConfiguration configuration = mock(GobblinServiceConfiguration.class); + final GobblinServiceManager manager = mock(GobblinServiceManager.class, CALLS_REAL_METHODS); + + Fixture() { + setInternalState(this.manager, "forceKillHandler", Optional.of(this.handler)); + setInternalState(this.manager, "serviceLauncher", this.launcher); + setInternalState(this.manager, "configuration", this.configuration); + this.manager.d2Announcer = mock(D2Announcer.class); + } + } +} diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagProcFactoryForceKillTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagProcFactoryForceKillTest.java new file mode 100644 index 00000000000..9ddc1eba18c --- /dev/null +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagProcFactoryForceKillTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration; + +import java.io.IOException; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provider; +import com.google.inject.multibindings.OptionalBinder; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import org.mockito.MockedStatic; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.metrics.event.TimingEvent; +import org.apache.gobblin.service.modules.orchestration.proc.KillDagProc; +import org.apache.gobblin.service.modules.orchestration.proc.ReevaluateDagProc; +import org.apache.gobblin.service.modules.orchestration.proc.OrchestratorIssueEmitter; +import org.apache.gobblin.service.modules.orchestration.task.DagProcessingEngineMetrics; +import org.apache.gobblin.service.modules.orchestration.task.KillDagTask; +import org.apache.gobblin.service.modules.orchestration.task.ReevaluateDagTask; +import org.apache.gobblin.service.modules.utils.FlowCompilationValidationHelper; +import org.apache.gobblin.service.monitoring.JobStatusRetriever; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + + +public class DagProcFactoryForceKillTest { + @DataProvider + public Object[][] handlerBindings() { + return new Object[][] {{false}, {true}}; + } + + @Test(dataProvider = "handlerBindings") + public void testOptionalGuiceBindingIsLazyAndUsedOnlyForMissingDagKill(boolean bindHandler) throws IOException { + AtomicInteger handlerRequests = new AtomicInteger(); + AtomicInteger retrieverRequests = new AtomicInteger(); + ForceKillHandler handler = mock(ForceKillHandler.class); + JobStatusRetriever retriever = mock(JobStatusRetriever.class); + Provider handlerProvider = () -> { + handlerRequests.incrementAndGet(); + return handler; + }; + Provider retrieverProvider = () -> { + retrieverRequests.incrementAndGet(); + return retriever; + }; + Injector injector = Guice.createInjector(binder -> { + binder.requireExplicitBindings(); + binder.bind(Config.class).toInstance(ConfigFactory.empty()); + binder.bind(FlowCompilationValidationHelper.class).toInstance(mock(FlowCompilationValidationHelper.class)); + binder.bind(JobStatusRetriever.class).toProvider(retrieverProvider); + OptionalBinder optional = OptionalBinder.newOptionalBinder(binder, ForceKillHandler.class); + if (bindHandler) { + optional.setBinding().toProvider(handlerProvider); + } + binder.bind(DagProcFactory.class); + }); + DagProcFactory factory = injector.getInstance(DagProcFactory.class); + DagManagementStateStore store = mock(DagManagementStateStore.class); + DagProcessingEngineMetrics metrics = mock(DagProcessingEngineMetrics.class); + when(store.getDag(any())).thenReturn(Optional.empty()); + when(handler.forceKill(any(), anyList(), any())).thenReturn(ForceKillHandler.Result.ACKNOWLEDGED); + State state = new State(); + state.setProp(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD, "group"); + state.setProp(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, "flow"); + state.setProp(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD, "1234"); + state.setProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD, "group"); + state.setProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD, "job"); + when(retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L)) + .thenReturn(Collections.singletonList(state)); + + ReevaluateDagTask reevaluate = mock(ReevaluateDagTask.class); + when(reevaluate.getDagAction()).thenReturn(action(DagActionStore.DagActionType.REEVALUATE)); + Assert.assertEquals(factory.meet(reevaluate).getClass(), ReevaluateDagProc.class); + KillDagProc kill = factory.meet(new KillDagTask(action(DagActionStore.DagActionType.KILL), null, store, metrics)); + Assert.assertEquals(handlerRequests.get(), 0); + Assert.assertEquals(retrieverRequests.get(), 0); + + try (MockedStatic ignored = mockStatic(OrchestratorIssueEmitter.class)) { + kill.process(store, metrics); + } + + Assert.assertEquals(handlerRequests.get(), bindHandler ? 1 : 0); + Assert.assertEquals(retrieverRequests.get(), bindHandler ? 1 : 0); + verify(metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, bindHandler); + if (bindHandler) { + verify(handler).forceKill(any(), anyList(), any()); + } else { + verifyNoInteractions(handler, retriever); + } + } + + @Test + public void testLegacyConstructorStillCreatesOrdinaryKillProcessor() throws IOException { + DagProcFactory factory = new DagProcFactory(ConfigFactory.empty(), null); + DagManagementStateStore store = mock(DagManagementStateStore.class); + DagProcessingEngineMetrics metrics = mock(DagProcessingEngineMetrics.class); + when(store.getDag(any())).thenReturn(Optional.empty()); + + try (MockedStatic ignored = mockStatic(OrchestratorIssueEmitter.class)) { + factory.meet(new KillDagTask(action(DagActionStore.DagActionType.KILL), null, store, metrics)) + .process(store, metrics); + } + + verify(metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, false); + } + + private static DagActionStore.DagAction action(DagActionStore.DagActionType type) { + return DagActionStore.DagAction.forFlow("group", "flow", 1234L, type); + } +} diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java new file mode 100644 index 00000000000..79205860af0 --- /dev/null +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/proc/KillDagProcForceKillTest.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration.proc; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import com.google.inject.Provider; +import com.typesafe.config.ConfigFactory; + +import org.apache.commons.lang3.tuple.Pair; +import org.mockito.MockedStatic; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.metrics.event.TimingEvent; +import org.apache.gobblin.service.ExecutionStatus; +import org.apache.gobblin.service.modules.flowgraph.Dag; +import org.apache.gobblin.service.modules.orchestration.DagActionStore; +import org.apache.gobblin.service.modules.orchestration.DagManagementStateStore; +import org.apache.gobblin.service.modules.orchestration.ForceKillHandler; +import org.apache.gobblin.service.modules.orchestration.task.DagProcessingEngineMetrics; +import org.apache.gobblin.service.modules.orchestration.task.KillDagTask; +import org.apache.gobblin.service.modules.spec.JobExecutionPlan; +import org.apache.gobblin.service.monitoring.JobStatusRetriever; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + + +public class KillDagProcForceKillTest { + private static final String GROUP = "group"; + private static final String FLOW = "flow"; + private static final long EXECUTION_ID = 1234L; + + private DagManagementStateStore store; + private JobStatusRetriever retriever; + private ForceKillHandler handler; + private Provider retrieverProvider; + private Provider handlerProvider; + private DagProcessingEngineMetrics metrics; + private MockedStatic dagUtils; + private MockedStatic issues; + + @BeforeMethod + @SuppressWarnings("unchecked") + public void setUp() throws IOException { + this.store = mock(DagManagementStateStore.class); + this.retriever = mock(JobStatusRetriever.class); + this.handler = mock(ForceKillHandler.class); + this.retrieverProvider = mock(Provider.class); + this.handlerProvider = mock(Provider.class); + this.metrics = mock(DagProcessingEngineMetrics.class); + when(this.store.getDag(any())).thenReturn(Optional.empty()); + when(this.retrieverProvider.get()).thenReturn(this.retriever); + when(this.handlerProvider.get()).thenReturn(this.handler); + when(this.handler.forceKill(any(), anyList(), any())).thenReturn(ForceKillHandler.Result.ACKNOWLEDGED); + statuses(); + this.dagUtils = mockStatic(DagProcUtils.class); + this.issues = mockStatic(OrchestratorIssueEmitter.class); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + if (this.issues != null) { + this.issues.close(); + } + if (this.dagUtils != null) { + this.dagUtils.close(); + } + } + + @DataProvider + public Object[][] activeScopes() { + return new Object[][] {{DagActionStore.NO_JOB_NAME_DEFAULT}, {"job"}}; + } + + @Test(dataProvider = "activeScopes") + @SuppressWarnings("unchecked") + public void testActiveDagUsesOnlyNormalCancellation(String jobName) throws IOException { + Dag dag = mock(Dag.class); + Dag.DagNode node = mock(Dag.DagNode.class); + when(this.store.getDag(any())).thenReturn(Optional.of(dag)); + when(this.store.getDagNodeWithJobStatus(any())).thenReturn(Pair.of(Optional.of(node), Optional.empty())); + + processor(jobName).process(this.store, this.metrics); + + this.dagUtils.verify(() -> DagProcUtils.setAndEmitFlowEvent(any(), eq(dag), + eq(TimingEvent.FlowTimings.FLOW_CANCELLED))); + if (jobName.equals(DagActionStore.NO_JOB_NAME_DEFAULT)) { + this.dagUtils.verify(() -> DagProcUtils.cancelDag(dag, this.store)); + } else { + this.dagUtils.verify(() -> DagProcUtils.cancelDagNode(node, this.store)); + } + verifyNoInteractions(this.handlerProvider, this.retrieverProvider, this.handler, this.retriever); + verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, true); + verify(this.store, never()).getFailedDag(any()); + } + + @Test + public void testAbsentHandlerPreservesMissingDagBehavior() throws IOException { + new KillDagProc(task(DagActionStore.NO_JOB_NAME_DEFAULT), ConfigFactory.empty()).process(this.store, this.metrics); + + verifyNoInteractions(this.handlerProvider, this.retrieverProvider, this.handler, this.retriever); + assertFailedWithoutCancellation(); + verify(this.store, never()).getFailedDag(any()); + } + + @Test + public void testAcknowledgedFallbackGetsImmutableDefensiveCopiesAndOriginalMetadata() throws IOException { + State original = state(GROUP, "job"); + original.getCommonProperties().setProperty("backend.common.handle", "original-common-handle"); + original.getCommonProperties().setProperty("backend.target", "original-target"); + original.getCommonProperties().setProperty("backend.handle", "overridden-common-handle"); + original.setProp("backend.handle", "{\"generation\":1,\"attempt\":1,\"id\":\"original\"}"); + original.setProp(TimingEvent.FlowEventConstants.CURRENT_GENERATION_FIELD, "3"); + original.setProp(TimingEvent.FlowEventConstants.CURRENT_ATTEMPTS_FIELD, "4"); + statuses(original); + doAnswer(invocation -> { + List states = invocation.getArgument(1); + Assert.assertEquals(states.size(), 1); + Assert.assertNotSame(states.get(0), original); + Assert.assertEquals(states.get(0).getProp("backend.handle"), original.getProp("backend.handle")); + Assert.assertEquals(states.get(0).getProp("backend.common.handle"), "original-common-handle"); + Assert.assertEquals(states.get(0).getProp(TimingEvent.FlowEventConstants.CURRENT_GENERATION_FIELD), "3"); + Assert.expectThrows(UnsupportedOperationException.class, () -> states.add(new State())); + states.get(0).setProp("backend.handle", "changed by handler"); + states.get(0).getCommonProperties().setProperty("backend.common.handle", "changed by handler"); + original.getCommonProperties().setProperty("backend.target", "changed in source"); + Assert.assertEquals(states.get(0).getProp("backend.target"), "original-target"); + ForceKillHandler.ActiveDagCheck active = invocation.getArgument(2); + Assert.assertFalse(active.isActive()); + return ForceKillHandler.Result.ACKNOWLEDGED; + }).when(this.handler).forceKill(any(), anyList(), any()); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + Assert.assertEquals(original.getProp("backend.handle"), "{\"generation\":1,\"attempt\":1,\"id\":\"original\"}"); + Assert.assertEquals(original.getProp("backend.common.handle"), "original-common-handle"); + verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, true); + this.dagUtils.verifyNoInteractions(); + this.issues.verifyNoInteractions(); + verify(this.store, never()).getFailedDag(any()); + verify(this.store, never()).updateDagNode(any()); + verify(this.store, never()).deleteDag(any()); + } + + @DataProvider + public Object[][] absentOrOutOfScopeStates() { + return new Object[][] { + {Collections.emptyList()}, + {Collections.singletonList(state(JobStatusRetriever.NA_KEY, JobStatusRetriever.NA_KEY))}, + {Collections.singletonList(withFlowField(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD, "12340"))}, + {Collections.singletonList(withFlowField(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD, "other"))}, + {Collections.singletonList(withFlowField(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, "other"))} + }; + } + + @Test(dataProvider = "absentOrOutOfScopeStates") + public void testMissingOrOutOfScopeStateCannotTriggerHandler(List states) throws IOException { + when(this.retriever.getJobStatusStatesForFlowExecution(FLOW, GROUP, EXECUTION_ID)).thenReturn(states); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + verifyNoInteractions(this.handlerProvider, this.handler); + assertFailedWithoutCancellation(); + verify(this.store, never()).getFailedDag(any()); + } + + @Test + public void testJobSpecificScopeKeepsOnlyTargetAndSummary() throws IOException { + State summary = state(JobStatusRetriever.NA_KEY, JobStatusRetriever.NA_KEY); + statuses(summary, state(GROUP, "job"), state(GROUP, "other-job"), state("other-group", "job")); + doAnswer(invocation -> { + List states = invocation.getArgument(1); + Assert.assertEquals(states.size(), 2); + Assert.assertEquals(states.get(0).getProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD), JobStatusRetriever.NA_KEY); + Assert.assertEquals(states.get(1).getProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD), "job"); + Assert.assertEquals(states.get(1).getProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD), GROUP); + return ForceKillHandler.Result.ACKNOWLEDGED; + }).when(this.handler).forceKill(eq(action("job")), anyList(), any()); + + processor("job").process(this.store, this.metrics); + + verify(this.handler).forceKill(eq(action("job")), anyList(), any()); + this.dagUtils.verifyNoInteractions(); + } + + @Test + public void testUnacknowledgedHandlerDoesNotReportSuccess() throws IOException { + statuses(state(GROUP, "job")); + when(this.handler.forceKill(any(), anyList(), any())).thenReturn(ForceKillHandler.Result.NOT_ACKNOWLEDGED); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + assertFailedWithoutCancellation(); + } + + @Test + @SuppressWarnings("unchecked") + public void testDagReappearingAfterStatusReadUsesNormalCancellation() throws IOException { + Dag dag = mock(Dag.class); + statuses(state(GROUP, "job")); + when(this.store.getDag(any())).thenReturn(Optional.empty(), Optional.of(dag)); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + verifyNoInteractions(this.handlerProvider, this.handler); + this.dagUtils.verify(() -> DagProcUtils.cancelDag(dag, this.store)); + verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, true); + } + + @Test + @SuppressWarnings("unchecked") + public void testDagReappearingDuringHandlerUsesNormalCancellation() throws IOException { + Dag dag = mock(Dag.class); + statuses(state(GROUP, "job")); + when(this.store.getDag(any())).thenReturn(Optional.empty(), Optional.empty(), Optional.of(dag)); + doAnswer(invocation -> { + ForceKillHandler.ActiveDagCheck active = invocation.getArgument(2); + Assert.assertTrue(active.isActive()); + return ForceKillHandler.Result.ACTIVE_DAG_PRESENT; + }).when(this.handler).forceKill(any(), anyList(), any()); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + this.dagUtils.verify(() -> DagProcUtils.cancelDag(dag, this.store)); + verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, true); + } + + @Test + public void testReappearedDagDisappearingAgainDoesNotResumeRetainedStopping() throws IOException { + statuses(state(GROUP, "job")); + when(this.handler.forceKill(any(), anyList(), any())).thenReturn(ForceKillHandler.Result.ACTIVE_DAG_PRESENT); + + processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics); + + assertFailedWithoutCancellation(); + verify(this.handler).forceKill(any(), anyList(), any()); + } + + @Test + public void testReadFailurePropagatesWithoutCallingHandler() throws IOException { + IOException failure = new IOException("Status read failed"); + doThrow(failure).when(this.retriever).getJobStatusStatesForFlowExecution(FLOW, GROUP, EXECUTION_ID); + + Assert.assertSame(Assert.expectThrows(IOException.class, + () -> processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics)), failure); + + verifyNoInteractions(this.handlerProvider, this.handler); + assertFailedWithoutCancellation(); + } + + @Test + public void testHandlerFailurePropagatesWithoutSuccess() throws IOException { + statuses(state(GROUP, "job")); + IOException failure = new IOException("Backend unavailable"); + doThrow(failure).when(this.handler).forceKill(any(), anyList(), any()); + + Assert.assertSame(Assert.expectThrows(IOException.class, + () -> processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics)), failure); + + assertFailedWithoutCancellation(); + } + + @Test + public void testActiveDagCheckFailurePropagates() throws IOException { + statuses(state(GROUP, "job")); + IOException failure = new IOException("Active DAG read failed"); + when(this.store.getDag(any())).thenReturn(Optional.empty(), Optional.empty()).thenThrow(failure); + doAnswer(invocation -> { + ForceKillHandler.ActiveDagCheck active = invocation.getArgument(2); + active.isActive(); + return ForceKillHandler.Result.ACKNOWLEDGED; + }).when(this.handler).forceKill(any(), anyList(), any()); + + Assert.assertSame(Assert.expectThrows(IOException.class, + () -> processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics)), failure); + + assertFailedWithoutCancellation(); + } + + @Test + public void testNullHandlerResultIsNotSuccess() throws IOException { + statuses(state(GROUP, "job")); + when(this.handler.forceKill(any(), anyList(), any())).thenReturn(null); + + Assert.expectThrows(NullPointerException.class, + () -> processor(DagActionStore.NO_JOB_NAME_DEFAULT).process(this.store, this.metrics)); + + assertFailedWithoutCancellation(); + } + + private void assertFailedWithoutCancellation() { + verify(this.metrics).markDagActionsAct(DagActionStore.DagActionType.KILL, false); + verify(this.metrics, never()).markDagActionsAct(DagActionStore.DagActionType.KILL, true); + this.dagUtils.verifyNoInteractions(); + } + + private void statuses(State... states) throws IOException { + when(this.retriever.getJobStatusStatesForFlowExecution(FLOW, GROUP, EXECUTION_ID)).thenReturn(Arrays.asList(states)); + } + + private KillDagProc processor(String jobName) { + return new KillDagProc(task(jobName), ConfigFactory.empty(), this.handlerProvider, this.retrieverProvider); + } + + private KillDagTask task(String jobName) { + return new KillDagTask(action(jobName), null, this.store, this.metrics); + } + + private static DagActionStore.DagAction action(String jobName) { + return new DagActionStore.DagAction(GROUP, FLOW, EXECUTION_ID, jobName, DagActionStore.DagActionType.KILL); + } + + private static State withFlowField(String field, String value) { + State state = state(GROUP, "job"); + state.setProp(field, value); + return state; + } + + private static State state(String jobGroup, String jobName) { + State state = new State(); + state.setProp(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD, GROUP); + state.setProp(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, FLOW); + state.setProp(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD, Long.toString(EXECUTION_ID)); + state.setProp(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD, jobGroup); + state.setProp(TimingEvent.FlowEventConstants.JOB_NAME_FIELD, jobName); + state.setProp(JobStatusRetriever.EVENT_NAME_FIELD, ExecutionStatus.COMPLETE.name()); + return state; + } +} diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java new file mode 100644 index 00000000000..e0bb0c29d86 --- /dev/null +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/RawJobStatusRetrieverTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.monitoring; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import com.google.common.base.Predicate; +import com.typesafe.config.ConfigFactory; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.metastore.MysqlJobStatusStateStore; +import org.apache.gobblin.metastore.StateStore; +import org.apache.gobblin.metrics.RootMetricContext; +import org.apache.gobblin.runtime.spec_executorInstance.LocalFsSpecProducer; +import org.apache.gobblin.runtime.troubleshooter.MultiContextIssueRepository; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.powermock.reflect.Whitebox.setInternalState; + + +public class RawJobStatusRetrieverTest { + @Test + @SuppressWarnings("unchecked") + public void testGenericReaderUsesExactExecutionPrefixAndPreservesOpaqueMetadata() throws IOException { + StateStore store = mock(StateStore.class); + JobStatusRetriever retriever = mock(JobStatusRetriever.class, CALLS_REAL_METHODS); + doReturn(store).when(retriever).getStateStore(); + when(store.getTableNames(eq("group.flow"), any())).thenAnswer(invocation -> { + Predicate predicate = invocation.getArgument(1); + return Arrays.asList("1234.group.job.gst", "1234.NA.NA.gst", "12340.group.other.gst") + .stream().filter(predicate::apply).collect(Collectors.toList()); + }); + State job = new State(); + job.setProp("backend.handle", "opaque-original-handle"); + State summary = new State(); + when(store.getAll("group.flow", "1234.group.job.gst")).thenReturn(Collections.singletonList(job)); + when(store.getAll("group.flow", "1234.NA.NA.gst")).thenReturn(Collections.singletonList(summary)); + + List states = retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L); + + Assert.assertEquals(states, Arrays.asList(job, summary)); + Assert.assertEquals(states.get(0).getProp("backend.handle"), "opaque-original-handle"); + verify(store, never()).getAll("group.flow", "12340.group.other.gst"); + verify(store, never()).getAll("group.flow"); + } + + @Test + @SuppressWarnings("unchecked") + public void testGenericReaderPropagatesReadFailureInsteadOfReturningNoRecords() throws IOException { + StateStore store = mock(StateStore.class); + JobStatusRetriever retriever = mock(JobStatusRetriever.class, CALLS_REAL_METHODS); + doReturn(store).when(retriever).getStateStore(); + IOException failure = new IOException("Unavailable"); + doThrow(failure).when(store).getTableNames(eq("group.flow"), any()); + + Assert.assertSame(Assert.expectThrows(IOException.class, + () -> retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L)), failure); + } + + @Test + public void testLocalFsReaderRejectsRawRetrievalWithoutStateStore() { + JobStatusRetriever retriever = new LocalFsJobStatusRetriever(ConfigFactory.parseMap(Collections.singletonMap( + LocalFsJobStatusRetriever.CONF_PREFIX + LocalFsSpecProducer.LOCAL_FS_PRODUCER_PATH_KEY, "unused")), + mock(MultiContextIssueRepository.class)); + + IOException failure = Assert.expectThrows(IOException.class, + () -> retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L)); + + Assert.assertTrue(failure.getMessage().contains("Raw job status retrieval requires a state store")); + Assert.assertTrue(failure.getMessage().contains(LocalFsJobStatusRetriever.class.getName())); + } + + @Test + @SuppressWarnings("unchecked") + public void testMysqlReaderReusesRawStoreAndPropagatesIOException() throws IOException { + MysqlJobStatusStateStore store = mock(MysqlJobStatusStateStore.class); + MysqlJobStatusRetriever retriever = mock(MysqlJobStatusRetriever.class, CALLS_REAL_METHODS); + setInternalState(retriever, "stateStore", store); + setInternalState(retriever, "metricContext", RootMetricContext.get()); + State state = new State(); + state.setProp("backend.handle", "opaque"); + when(store.getAll("group.flow", 1234L)).thenReturn(Collections.singletonList(state)); + + Assert.assertEquals(retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L), + Collections.singletonList(state)); + verify(store).getAll("group.flow", 1234L); + + IOException failure = new IOException("Database unavailable"); + doThrow(failure).when(store).getAll("group.flow", 1234L); + Assert.assertSame(Assert.expectThrows(IOException.class, + () -> retriever.getJobStatusStatesForFlowExecution("flow", "group", 1234L)), failure); + } +}