Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public class ServiceConfigKeys {
public static final String GOBBLIN_SERVICE_FLOWSPEC = GOBBLIN_SERVICE_PREFIX + "flowSpec";
public static final String GOBBLIN_SERVICE_FLOWGRAPH_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "flowGraph.class";
public static final String GOBBLIN_SERVICE_FLOWGRAPH_HELPER_KEY = GOBBLIN_SERVICE_PREFIX + "flowGraphHelper.class";
public static final String GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "forceKillHandler.class";

// Flow Compiler Keys
public static final String GOBBLIN_SERVICE_FLOWCOMPILER_CLASS_KEY = GOBBLIN_SERVICE_PREFIX + "flowCompiler.class";
Expand Down
1 change: 1 addition & 0 deletions gobblin-metastore/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ dependencies {
runtimeOnly externalDependency.mysqlConnector

testCompile externalDependency.testng
testCompile externalDependency.mockito
testCompile externalDependency.embeddedProcess
testCompile externalDependency.testContainers
testCompile externalDependency.testContainersMysql
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ public MysqlJobStatusStateStore(DataSource dataSource, String stateStoreTableNam
* @throws IOException in case of failures
*/
public List<T> getAll(String storeName, long flowExecutionId) throws IOException {
return getAll(storeName, flowExecutionId + "%", JobStateSearchColumns.TABLE_NAME_ONLY);
return getAll(storeName, flowExecutionId + ServiceConfigKeys.STATE_STORE_KEY_SEPARATION_CHARACTER + "%",
JobStateSearchColumns.TABLE_NAME_ONLY);
}
Comment thread
Siddhi-agg marked this conversation as resolved.
Resolved

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.gobblin.metastore;

import java.io.IOException;
import java.util.Collections;

import org.testng.Assert;
import org.testng.annotations.Test;

import org.apache.gobblin.configuration.State;

import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;


public class MysqlJobStatusStateStoreFlowScopeTest {
@Test
@SuppressWarnings("unchecked")
public void testExecutionPrefixIncludesSeparator() throws IOException {
MysqlJobStatusStateStore<State> store = mock(MysqlJobStatusStateStore.class, CALLS_REAL_METHODS);
doReturn(Collections.emptyList()).when(store).getAll(eq("group.flow"), anyString(),
eq(MysqlStateStore.JobStateSearchColumns.TABLE_NAME_ONLY));

Assert.assertTrue(store.getAll("group.flow", 1234L).isEmpty());

verify(store).getAll("group.flow", "1234.%", MysqlStateStore.JobStateSearchColumns.TABLE_NAME_ONLY);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.gobblin.service.monitoring;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
Expand Down Expand Up @@ -49,6 +50,7 @@
import org.apache.gobblin.runtime.troubleshooter.TroubleshooterException;
import org.apache.gobblin.runtime.troubleshooter.TroubleshooterUtils;
import org.apache.gobblin.service.ExecutionStatus;
import org.apache.gobblin.service.ServiceConfigKeys;
import org.apache.gobblin.util.ConfigUtils;


Expand Down Expand Up @@ -86,6 +88,33 @@ public abstract Iterator<JobStatus> getJobStatusesForFlowExecution(String flowNa
public abstract Iterator<JobStatus> getJobStatusesForFlowExecution(String flowName, String flowGroup,
long flowExecutionId, String jobName, String jobGroup);

/**
* Read retained states without discarding backend metadata during conversion to {@link JobStatus}.
* Includes the flow-summary record, if present. Callers must validate the decoded identity before using
* execution handles: storage keys alone do not establish the identity of the serialized contents.
*
* <p>Unlike status-display APIs, read failures propagate rather than appearing as an empty result.</p>
*
* @throws IOException if this retriever has no state store or retained state cannot be read
*/
public List<State> getJobStatusStatesForFlowExecution(String flowName, String flowGroup, long flowExecutionId)
throws IOException {
Objects.requireNonNull(flowName, "flowName");
Objects.requireNonNull(flowGroup, "flowGroup");
String separator = ServiceConfigKeys.STATE_STORE_KEY_SEPARATION_CHARACTER;
String storeName = flowGroup + separator + flowName;
String tablePrefix = flowExecutionId + separator;
StateStore<State> stateStore = getStateStore();
if (stateStore == null) {
throw new IOException("Raw job status retrieval requires a state store: " + getClass().getName());
}
List<State> states = new ArrayList<>();
for (String tableName : stateStore.getTableNames(storeName, name -> name.startsWith(tablePrefix))) {
states.addAll(stateStore.getAll(storeName, tableName));
}
return states;
}

/**
* Get the latest {@link FlowStatus}es of executions of flows belonging to this flow group. Currently, latest flow execution
* is decided by comparing {@link JobStatus#getFlowExecutionId()}.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
import org.apache.gobblin.service.modules.orchestration.DagTaskStream;
import org.apache.gobblin.service.modules.orchestration.FlowLaunchHandler;
import org.apache.gobblin.service.modules.orchestration.FlowLaunchMultiActiveLeaseArbiterFactory;
import org.apache.gobblin.service.modules.orchestration.ForceKillHandler;
import org.apache.gobblin.service.modules.orchestration.MultiActiveLeaseArbiter;
import org.apache.gobblin.service.modules.orchestration.MySqlDagManagementStateStore;
import org.apache.gobblin.service.modules.orchestration.MysqlDagActionStore;
Expand Down Expand Up @@ -164,6 +165,7 @@ binding time (optionally bound classes cannot have names associated with them),
binder.bind(DagManagement.class).to(DagManagementTaskStreamImpl.class);
binder.bind(DagManagementStateStore.class).to(MySqlDagManagementStateStore.class);
binder.bind(DagTaskStream.class).to(DagManagementTaskStreamImpl.class);
configureForceKillHandler(binder, serviceConfig.getInnerConfig());
binder.bind(DagProcFactory.class);
binder.bind(DagProcessingEngine.class);
binder.bind(DagProcessingEngineMetrics.class);
Expand Down Expand Up @@ -257,6 +259,14 @@ protected static <T> Class<? extends T> getClassByNameOrAlias(Class<T> baseClass
}
}

static void configureForceKillHandler(Binder binder, Config config) {
OptionalBinder<ForceKillHandler> optional = OptionalBinder.newOptionalBinder(binder, ForceKillHandler.class);
if (config.hasPath(ServiceConfigKeys.GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY)) {
optional.setBinding().to(getClassByNameOrAlias(ForceKillHandler.class, config,
ServiceConfigKeys.GOBBLIN_SERVICE_FORCE_KILL_HANDLER_CLASS_KEY, null)).in(Singleton.class);
}
}

public static class EmbeddedRestliServerProvider implements Provider<EmbeddedRestliServer> {
Injector injector;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.base.Optional;
import com.google.common.util.concurrent.AbstractIdleService;
import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
Expand Down Expand Up @@ -71,6 +73,7 @@
import org.apache.gobblin.service.ServiceConfigKeys;
import org.apache.gobblin.service.modules.db.ServiceDatabaseManager;
import org.apache.gobblin.service.modules.orchestration.DagProcessingEngine;
import org.apache.gobblin.service.modules.orchestration.ForceKillHandler;
import org.apache.gobblin.service.modules.orchestration.Orchestrator;
import org.apache.gobblin.service.modules.orchestration.UserQuotaManager;
import org.apache.gobblin.service.modules.restli.FlowConfigsV2ResourceHandler;
Expand Down Expand Up @@ -174,6 +177,9 @@ public class GobblinServiceManager implements ApplicationLauncher {
@Inject(optional = true)
protected DagProcessingEngine dagProcessingEngine;

@Inject
private Optional<ForceKillHandler> forceKillHandler = Optional.absent();

@Inject
protected GobblinServiceManager(GobblinServiceConfiguration configuration) throws Exception {
this.configuration = Objects.requireNonNull(configuration);
Expand Down Expand Up @@ -256,6 +262,19 @@ private void registerServicesInLauncher(){
}

this.serviceLauncher.addService(dagProcessingEngine);
if (this.forceKillHandler.isPresent()) {
// The launcher's own failure/shutdown paths must also close the backend, not just this manager's stop().
this.serviceLauncher.addService(new AbstractIdleService() {
@Override
protected void startUp() {
}

@Override
protected void shutDown() {
forceKillHandler.get().close();
}
});
}

this.serviceLauncher.addService(databaseManager);
this.serviceLauncher.addService(issueRepository);
Expand Down Expand Up @@ -297,6 +316,13 @@ private void configureServices(){

@Override
public void start() throws ApplicationException {
try (ForceKillStartupGuard guard = new ForceKillStartupGuard(this.forceKillHandler.orNull())) {
startServices();
guard.started = true;
}
}

private void startServices() throws ApplicationException {
LOGGER.info("[Init] Starting the Gobblin Service Manager");

configureServices();
Expand Down Expand Up @@ -346,7 +372,7 @@ public void stop() throws ApplicationException {

LOGGER.info("Stopping the Gobblin Service Manager");
this.stopInProgress = true;
try {
try (ForceKillHandler handler = this.forceKillHandler.orNull()) {
// Stop announcing GaaS instances to d2 when services are stopped
this.d2Announcer.markDownServer();
this.serviceLauncher.stop();
Expand All @@ -357,7 +383,25 @@ public void stop() throws ApplicationException {

@Override
public void close() throws IOException {
this.serviceLauncher.close();
try (ForceKillHandler handler = this.forceKillHandler.orNull()) {
this.serviceLauncher.close();
}
}

static final class ForceKillStartupGuard implements AutoCloseable {
private final ForceKillHandler handler;
boolean started;

ForceKillStartupGuard(ForceKillHandler handler) {
this.handler = handler;
}

@Override
public void close() {
if (!this.started && this.handler != null) {
this.handler.close();
}
}
}

private static String getServiceId(CommandLine cmd) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,11 @@

package org.apache.gobblin.service.modules.orchestration;

import java.util.Objects;

import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import com.typesafe.config.Config;

Expand All @@ -37,6 +41,7 @@
import org.apache.gobblin.service.modules.orchestration.task.ReevaluateDagTask;
import org.apache.gobblin.service.modules.orchestration.task.ResumeDagTask;
import org.apache.gobblin.service.modules.utils.FlowCompilationValidationHelper;
import org.apache.gobblin.service.monitoring.JobStatusRetriever;

/**
* {@link DagTaskVisitor} for transforming a specific {@link DagTask} derived class to its companion {@link DagProc} derived class.
Expand All @@ -50,11 +55,21 @@ public class DagProcFactory implements DagTaskVisitor<DagProc<?>> {

private final Config config;
private final FlowCompilationValidationHelper flowCompilationValidationHelper;
private final Optional<Provider<ForceKillHandler>> forceKillHandler;
private final Provider<JobStatusRetriever> jobStatusRetriever;

@Inject
public DagProcFactory(Config config, FlowCompilationValidationHelper flowCompilationValidationHelper) {
this(config, flowCompilationValidationHelper, Optional.absent(), null);
}

@Inject
public DagProcFactory(Config config, FlowCompilationValidationHelper flowCompilationValidationHelper,
Optional<Provider<ForceKillHandler>> forceKillHandler, Provider<JobStatusRetriever> jobStatusRetriever) {
this.config = config;
this.flowCompilationValidationHelper = flowCompilationValidationHelper;
this.forceKillHandler = Objects.requireNonNull(forceKillHandler, "forceKillHandler");
this.jobStatusRetriever = forceKillHandler.isPresent()
? Objects.requireNonNull(jobStatusRetriever, "jobStatusRetriever") : jobStatusRetriever;
}

@Override
Expand All @@ -79,6 +94,9 @@ public ReevaluateDagProc meet(ReevaluateDagTask reEvaluateDagTask) {

@Override
public KillDagProc meet(KillDagTask killDagTask) {
if (this.forceKillHandler.isPresent()) {
return new KillDagProc(killDagTask, this.config, this.forceKillHandler.get(), this.jobStatusRetriever);
}
return new KillDagProc(killDagTask, this.config);
}

Expand All @@ -87,4 +105,3 @@ public ResumeDagProc meet(ResumeDagTask resumeDagTask) {
return new ResumeDagProc(resumeDagTask, this.config);
}
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.gobblin.service.modules.orchestration;

import java.io.Closeable;
import java.io.IOException;
import java.util.List;

import org.apache.gobblin.configuration.State;


/**
* Backend-specific handling of an existing KILL action after its active DAG has been removed.
*
* <p>The supplied states are the only retained execution metadata source. Implementations must validate backend
* handles, destinations and their original submission identities, without consulting archived DAGs or guessing
* destinations from current routing. A flow-summary state is context, not an execution to stop. Implementations
* must not rewrite terminal job status or emit normal flow/job cancellation events. Bind implementations as
* singletons so the service manager can close the same instance used by KILL processing.</p>
*/
@FunctionalInterface
public interface ForceKillHandler extends Closeable {
enum Result {
/** All identified stop requests were acknowledged; this does not establish physical process termination. */
ACKNOWLEDGED,
/** No execution could be resolved, or at least one stop was not acknowledged. */
NOT_ACKNOWLEDGED,
/** An active DAG was observed; stop retained processing and let normal DAG cancellation take over. */
ACTIVE_DAG_PRESENT
}

@FunctionalInterface
interface ActiveDagCheck {
boolean isActive() throws IOException;
}

/**
* Attempt bounded, backend-specific stopping using defensive copies of the retained job states.
*
* @param action the exact flow execution and optional job requested for cancellation
* @param jobStates scoped job states and, if retained, the flow-summary state; the list is unmodifiable
* @param activeDagCheck must be checked before each backend stop; return {@link Result#ACTIVE_DAG_PRESENT} if true.
* This reduces, but does not atomically exclude, concurrent resume.
* @throws IOException when the operation cannot be completed; failures must not be reported as acknowledgements
*/
Result forceKill(DagActionStore.DagAction action, List<State> jobStates, ActiveDagCheck activeDagCheck)
throws IOException;

/** Stop accepting work and release local resources without waiting indefinitely for remote calls. Must be idempotent. */
@Override
default void close() {
}
}
Loading
Loading