Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions base/cvd/cuttlefish/host/commands/cvd/cli/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ cf_cc_library(
"//cuttlefish/host/commands/cvd/cli/commands:load_configs.h",
],
deps = [
"//cuttlefish/common/libs/fs",
"//cuttlefish/common/libs/utils:contains",
"//cuttlefish/common/libs/utils:environment",
"//cuttlefish/common/libs/utils:files",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include "fmt/core.h"
#include "fmt/format.h"

#include "cuttlefish/common/libs/fs/shared_fd.h"
#include "cuttlefish/common/libs/utils/files.h"
#include "cuttlefish/flag_parser/flag.h"
#include "cuttlefish/host/commands/cvd/cli/command_request.h"
Expand Down Expand Up @@ -93,9 +94,10 @@ Result<CommandRequest> BuildStartCommand(const CommandRequest& request,
.Build());
}

Result<LocalInstanceGroup> CreateGroup(
InstanceManager& instance_manager, const std::string& base_dir,
const EnvironmentSpecification& env_spec) {
Result<LocalInstanceGroup> CreateGroup(InstanceManager& instance_manager,
const std::string& base_dir,
const EnvironmentSpecification& env_spec,
SharedFD* out_in_progress_lock) {
InstanceGroupParams group_params{
.group_name = env_spec.common().group_name(),
};
Expand All @@ -106,7 +108,8 @@ Result<LocalInstanceGroup> CreateGroup(
}
return CF_EXPECT(instance_manager.CreateInstanceGroup(
std::move(group_params),
CF_EXPECT(GetGroupCreationDirectories(base_dir, env_spec))));
CF_EXPECT(GetGroupCreationDirectories(base_dir, env_spec)),
out_in_progress_lock));
}

} // namespace
Expand Down Expand Up @@ -167,19 +170,26 @@ Result<void> LoadConfigsCommand::Handle(const CommandRequest& request) {
});
auto listener_handle = CF_EXPECT(std::move(push_result));

SharedFD in_progress_lock;
group_creation_mtx.lock();
// Don't use CF_EXPECT here or the mutex will be left locked.
auto group_res = CreateGroup(instance_manager_, flags_.base_dir, env_spec);
auto group_res = CreateGroup(instance_manager_, flags_.base_dir, env_spec,
&in_progress_lock);
if (group_res.has_value()) {
// Have to initialize the group_name variable before releasing the mutex.
group_name = (*group_res).GroupName();
}
group_creation_mtx.unlock();
auto group = CF_EXPECT(std::move(group_res));

auto cvd_flags = CF_EXPECT(ParseCvdConfigs(env_spec, group));
Result<CvdFlags> cvd_flags_res = ParseCvdConfigs(env_spec, group);
if (!cvd_flags_res.has_value()) {
group.SetAllStates(cvd::INSTANCE_STATE_PREPARE_FAILED);
(void)instance_manager_.UpdateInstanceGroup(group);
}
CvdFlags cvd_flags = CF_EXPECT(std::move(cvd_flags_res));

auto res = LoadGroup(request, group, std::move(cvd_flags));
auto res = LoadGroup(request, group, std::move(cvd_flags), in_progress_lock);
if (!res.has_value()) {
auto first_instance_state = group.Instances()[0].State();
// The failure could have occurred during prepare(fetch) or start
Expand All @@ -197,7 +207,8 @@ Result<void> LoadConfigsCommand::Handle(const CommandRequest& request) {

Result<void> LoadConfigsCommand::LoadGroup(const CommandRequest& request,
LocalInstanceGroup& group,
CvdFlags cvd_flags) {
CvdFlags cvd_flags,
SharedFD& in_progress_lock) {
auto mkdir_res =
EnsureDirectoryExists(group.HomeDir(), 0775, /* group_name */ "");
if (!mkdir_res.has_value()) {
Expand Down Expand Up @@ -229,7 +240,8 @@ Result<void> LoadConfigsCommand::LoadGroup(const CommandRequest& request,
CommandRequest start_cmd =
CF_EXPECT(BuildStartCommand(request, cvd_flags, group));
std::unique_ptr<CvdCommandHandler> start_handler =
std::make_unique<CvdStartCommandHandler>(instance_manager_);
std::make_unique<CvdStartCommandHandler>(instance_manager_,
std::move(in_progress_lock));
CF_EXPECT(start_handler->Handle(start_cmd));
return {};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include <string>

#include "cuttlefish/common/libs/fs/shared_fd.h"
#include "cuttlefish/flag_parser/flag.h"
#include "cuttlefish/host/commands/cvd/cli/command_request.h"
#include "cuttlefish/host/commands/cvd/cli/commands/command_handler.h"
Expand Down Expand Up @@ -48,7 +49,8 @@ class LoadConfigsCommand : public CvdCommandHandler {

private:
Result<void> LoadGroup(const CommandRequest& request,
LocalInstanceGroup& group, CvdFlags cvd_flags);
LocalInstanceGroup& group, CvdFlags cvd_flags,
SharedFD& in_progress_lock);

InstanceManager& instance_manager_;
LoadFlags flags_;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ Result<void> RemoveCvdCommandHandler::Handle(const CommandRequest& request) {
return CF_ERR(NoGroupMessage(request));
}
auto group = CF_EXPECT(selector::SelectGroup(instance_manager_, request));
group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(group));

Result<void> stop_res = StopGroup(group);
if (!stop_res.has_value()) {
Expand All @@ -82,7 +83,9 @@ Result<void> RemoveCvdCommandHandler::Handle(const CommandRequest& request) {
"kill any remaining device processes.";
}

CF_EXPECT(instance_manager_.RemoveInstanceGroup(group));
bool removed = CF_EXPECT(instance_manager_.RemoveInstanceGroup(group));
CF_EXPECT(removed == true,
"Instance group no longer exists or changed concurrently");

return {};
}
Expand Down
102 changes: 78 additions & 24 deletions base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include <iostream>
#include <iterator>
#include <memory>
#include <mutex>
#include <optional>
#include <set>
#include <string>
Expand Down Expand Up @@ -376,8 +377,9 @@ bool CanBypassToSingleInstance(const LocalInstance& instance,
} // namespace

CvdStartCommandHandler::CvdStartCommandHandler(
InstanceManager& instance_manager)
: instance_manager_(instance_manager) {
InstanceManager& instance_manager, SharedFD preacquired_in_progress_lock)
: instance_manager_(instance_manager),
preacquired_in_progress_lock_(std::move(preacquired_in_progress_lock)) {
own_flags_.daemon = true;
own_flags_.print_group_format = isatty(0) ? "human" : "json";
}
Expand All @@ -396,6 +398,13 @@ Result<void> CvdStartCommandHandler::Handle(const CommandRequest& request) {
request.Selectors().instance_names->size() == 1) {
auto [instance, group] =
CF_EXPECT(selector::SelectInstance(instance_manager_, request));
group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(group));
for (const LocalInstance& refreshed_inst : group.Instances()) {
if (refreshed_inst.Id() == instance.Id()) {
instance = refreshed_inst;
break;
}
}

if (CanBypassToSingleInstance(instance, group, subcmd_args)) {
CF_EXPECT(LaunchSingleInstance(instance, group, request));
Expand All @@ -413,9 +422,18 @@ Result<void> CvdStartCommandHandler::Handle(const CommandRequest& request) {
CF_EXPECT(selector::SelectGroup(instance_manager_, request),
"Failed to select group to start, did you mean 'cvd create'?");

SharedFD in_progress_lock = std::move(preacquired_in_progress_lock_);
if (!in_progress_lock->IsOpen()) {
in_progress_lock =
CF_EXPECT(InstanceManager::AcquireGroupInProgressLock(group));
}
group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(
group, /*caller_holds_in_progress_lock=*/true));

CF_EXPECT(!group.HasActiveInstances(),
"Selected instance group is already started, use `cvd create` to "
"create a new one.");
CF_EXPECT(instance_manager_.EnsureInstanceLocksInUse(group));

std::unordered_map<std::string, std::string> envs = request.Env();

Expand Down Expand Up @@ -443,8 +461,10 @@ Result<void> CvdStartCommandHandler::Handle(const CommandRequest& request) {
Command command = CF_EXPECT(ConstructCvdNonHelpCommand(
bin, group, subcmd_args, envs, request, memfd));

std::mutex group_mutex;
Result<std::unique_ptr<InterruptListenerHandle>> handle_res =
PushInterruptListener([this, &group, stop_monitor](int signal) {
PushInterruptListener([this, &group, &group_mutex,
stop_monitor](int signal) {
LOG(WARNING) << strsignal(signal) << " signal received, cleanning up";
stop_monitor();
Result<void> interrupt_res = subprocess_waiter_.Interrupt();
Expand All @@ -455,38 +475,45 @@ Result<void> CvdStartCommandHandler::Handle(const CommandRequest& request) {
"run `cvd reset` to ensure a clean state";
}

group.SetAllStates(cvd::INSTANCE_STATE_CANCELLED);
Result<void> update_res = instance_manager_.UpdateInstanceGroup(group);
if (!update_res.has_value()) {
LOG(ERROR) << "Failed to update group status: " << update_res.error();
{
std::lock_guard lock(group_mutex);
group.SetAllStates(cvd::INSTANCE_STATE_CANCELLED);
Result<void> update_res =
instance_manager_.UpdateInstanceGroup(group);
if (!update_res.has_value()) {
LOG(ERROR) << "Failed to update group status: "
<< update_res.error();
}
}
// It's technically possible for the group's state to be set to
// "running" before abort has a chance to run, but that can only happen
// if the instances are indeed running, so it's OK.
// "running" before abort has a chance to run, but that can only
// happen if the instances are indeed running, so it's OK.

std::abort();
});

std::unique_ptr<InterruptListenerHandle> listener_handle =
CF_EXPECT(std::move(handle_res));
group.SetAllStates(cvd::INSTANCE_STATE_STARTING);
group.SetStartTime(CvdServerClock::now());
CF_EXPECT(instance_manager_.UpdateInstanceGroup(group));
{
std::lock_guard lock(group_mutex);
group.SetAllStates(cvd::INSTANCE_STATE_STARTING);
group.SetStartTime(CvdServerClock::now());
CF_EXPECT(instance_manager_.UpdateInstanceGroup(group));
}

Result<void> monitor_res;
#ifdef __linux__
std::thread monitor_thread;

if (!own_flags_.daemon) {
const LocalInstance& instance = *group.Instances().begin();
const std::string assemble_log =
absl::StrCat(instance.AssemblyDirectory(), "/", kLogNameAssembleCvd);
const LocalInstance first_instance = *group.Instances().begin();
const std::string assemble_log = absl::StrCat(
first_instance.AssemblyDirectory(), "/", kLogNameAssembleCvd);
(void)RemoveFile(assemble_log);
for (const auto& log : CF_EXPECT(instance.LogsFilenames())) {
for (const auto& log : CF_EXPECT(first_instance.LogsFilenames())) {
(void)RemoveFile(log);
}
monitor_thread = std::thread([&group, stop_read, &monitor_res]() {
const LocalInstance& first_instance = *group.Instances().begin();
monitor_thread = std::thread([first_instance, stop_read, &monitor_res]() {
monitor_res = MonitorLogs(first_instance, stop_read);
});
}
Expand All @@ -505,8 +532,8 @@ Result<void> CvdStartCommandHandler::Handle(const CommandRequest& request) {
};
#endif

Result<void> launch_res =
LaunchDeviceInterruptible(std::move(command), group, envs, request);
Result<void> launch_res = LaunchDeviceInterruptible(
std::move(command), group, group_mutex, envs, request);

if (!launch_res.has_value()) {
if (!own_flags_.daemon) {
Expand All @@ -518,8 +545,12 @@ Result<void> CvdStartCommandHandler::Handle(const CommandRequest& request) {
return launch_res;
}

group.SetAllStates(cvd::INSTANCE_STATE_RUNNING);
CF_EXPECT(instance_manager_.UpdateInstanceGroup(group));
{
std::lock_guard lock(group_mutex);
group.SetAllStates(cvd::INSTANCE_STATE_RUNNING);
CF_EXPECT(instance_manager_.UpdateInstanceGroup(group));
}
in_progress_lock = SharedFD();

if (!own_flags_.daemon) {
listener_handle.reset();
Expand Down Expand Up @@ -607,7 +638,7 @@ Result<void> CvdStartCommandHandler::LaunchDevice(
}

Result<void> CvdStartCommandHandler::LaunchDeviceInterruptible(
Command command, LocalInstanceGroup& group,
Command command, LocalInstanceGroup& group, std::mutex& group_mutex,
const std::unordered_map<std::string, std::string>& envs,
const CommandRequest& request) {
// cvd_internal_start uses the config from the previous invocation to
Expand All @@ -624,6 +655,7 @@ Result<void> CvdStartCommandHandler::LaunchDeviceInterruptible(
const Result<void> start_res =
LaunchDevice(std::move(command), group, group.Instances());
if (!start_res.has_value()) {
std::lock_guard lock(group_mutex);
group.SetAllStates(cvd::INSTANCE_STATE_BOOT_FAILED);
CF_EXPECT(instance_manager_.UpdateInstanceGroup(group));
return start_res;
Expand All @@ -635,6 +667,28 @@ Result<void> CvdStartCommandHandler::LaunchDeviceInterruptible(
Result<void> CvdStartCommandHandler::LaunchSingleInstance(
LocalInstance& instance, LocalInstanceGroup& group,
const CommandRequest& request) {
SharedFD in_progress_lock = std::move(preacquired_in_progress_lock_);
if (!in_progress_lock->IsOpen()) {
in_progress_lock =
CF_EXPECT(InstanceManager::AcquireGroupInProgressLock(group));
}
group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(
group, /*caller_holds_in_progress_lock=*/true));
bool found_instance = false;
for (const LocalInstance& refreshed_inst : group.Instances()) {
if (refreshed_inst.Id() == instance.Id()) {
instance = refreshed_inst;
found_instance = true;
break;
}
}
CF_EXPECT(found_instance == true,
"Selected instance no longer exists in group");
CF_EXPECT(
CanBypassToSingleInstance(instance, group, request.SubcommandArguments()),
"Selected instance cannot be started individually in its current state");
CF_EXPECT(instance_manager_.EnsureInstanceLocksInUse(group));

const std::string bin_path = group.HostArtifactsPath() + "/bin/run_cvd";
std::unordered_map<std::string, std::string> run_cvd_envs = request.Env();
run_cvd_envs[kCuttlefishInstanceEnvVarName] = std::to_string(instance.Id());
Expand Down Expand Up @@ -679,7 +733,6 @@ Result<void> CvdStartCommandHandler::LaunchSingleInstance(
};

set_instance_state(cvd::INSTANCE_STATE_STARTING);
group.SetStartTime(CvdServerClock::now());
CF_EXPECT(instance_manager_.UpdateInstanceGroup(group));

const Result<void> start_res =
Expand All @@ -692,6 +745,7 @@ Result<void> CvdStartCommandHandler::LaunchSingleInstance(

set_instance_state(cvd::INSTANCE_STATE_RUNNING);
CF_EXPECT(instance_manager_.UpdateInstanceGroup(group));
in_progress_lock = SharedFD();

const Json::Value group_json = CF_EXPECT(group.FetchStatus());
std::cout << group_json.toStyledString();
Expand Down
9 changes: 7 additions & 2 deletions base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,13 @@

#pragma once

#include <mutex>
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>

#include "cuttlefish/common/libs/fs/shared_fd.h"
#include "cuttlefish/flag_parser/flag.h"
#include "cuttlefish/host/commands/cvd/cli/commands/command_handler.h"
#include "cuttlefish/host/commands/cvd/instances/instance_manager.h"
Expand All @@ -32,7 +35,8 @@ namespace cuttlefish {

class CvdStartCommandHandler : public CvdCommandHandler {
public:
CvdStartCommandHandler(InstanceManager& instance_manager);
CvdStartCommandHandler(InstanceManager& instance_manager,
SharedFD preacquired_in_progress_lock = {});

Result<void> Handle(const CommandRequest& request) override;
std::vector<std::string> CmdList() const override;
Expand All @@ -54,14 +58,15 @@ class CvdStartCommandHandler : public CvdCommandHandler {
const std::vector<LocalInstance>& instances);

Result<void> LaunchDeviceInterruptible(
Command command, LocalInstanceGroup& group,
Command command, LocalInstanceGroup& group, std::mutex& group_mutex,
const std::unordered_map<std::string, std::string>& envs,
const CommandRequest& request);

// Flags handled by `cvd start` itself, not cvd_internal_start.
std::vector<Flag> BuildOwnFlags();

InstanceManager& instance_manager_;
SharedFD preacquired_in_progress_lock_;
SubprocessWaiter subprocess_waiter_;
struct {
std::vector<std::string> host_substitutions;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ Result<void> CvdStopCommandHandler::Handle(const CommandRequest& request) {
}

auto group = CF_EXPECT(selector::SelectGroup(instance_manager_, request));
group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(group));
CF_EXPECT(group.HasActiveInstances(), "Selected group is not running");

CF_EXPECT(ConsumeFlags(CF_EXPECT(Flags(request)), cmd_args,
Expand Down
Loading
Loading