diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/cli/BUILD.bazel index 389a8ab21be..f76e012ca72 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/BUILD.bazel @@ -119,6 +119,7 @@ cf_cc_library( "//cuttlefish/host/commands/cvd/cli/commands:load_configs.h", ], deps = [ + "//cuttlefish/common/libs/fs", "//cuttlefish/common/libs/utils:contains", "//cuttlefish/common/libs/utils:environment", "//cuttlefish/common/libs/utils:files", diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp index 05821dc3bca..42746a8ead4 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp @@ -26,6 +26,7 @@ #include "fmt/core.h" #include "fmt/format.h" +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/common/libs/utils/files.h" #include "cuttlefish/flag_parser/flag.h" #include "cuttlefish/host/commands/cvd/cli/command_request.h" @@ -93,9 +94,10 @@ Result BuildStartCommand(const CommandRequest& request, .Build()); } -Result CreateGroup( - InstanceManager& instance_manager, const std::string& base_dir, - const EnvironmentSpecification& env_spec) { +Result CreateGroup(InstanceManager& instance_manager, + const std::string& base_dir, + const EnvironmentSpecification& env_spec, + SharedFD* out_in_progress_lock) { InstanceGroupParams group_params{ .group_name = env_spec.common().group_name(), }; @@ -106,7 +108,8 @@ Result CreateGroup( } return CF_EXPECT(instance_manager.CreateInstanceGroup( std::move(group_params), - CF_EXPECT(GetGroupCreationDirectories(base_dir, env_spec)))); + CF_EXPECT(GetGroupCreationDirectories(base_dir, env_spec)), + out_in_progress_lock)); } } // namespace @@ -167,9 +170,11 @@ Result LoadConfigsCommand::Handle(const CommandRequest& request) { }); auto listener_handle = CF_EXPECT(std::move(push_result)); + SharedFD in_progress_lock; group_creation_mtx.lock(); // Don't use CF_EXPECT here or the mutex will be left locked. - auto group_res = CreateGroup(instance_manager_, flags_.base_dir, env_spec); + auto group_res = CreateGroup(instance_manager_, flags_.base_dir, env_spec, + &in_progress_lock); if (group_res.has_value()) { // Have to initialize the group_name variable before releasing the mutex. group_name = (*group_res).GroupName(); @@ -177,9 +182,14 @@ Result LoadConfigsCommand::Handle(const CommandRequest& request) { group_creation_mtx.unlock(); auto group = CF_EXPECT(std::move(group_res)); - auto cvd_flags = CF_EXPECT(ParseCvdConfigs(env_spec, group)); + Result cvd_flags_res = ParseCvdConfigs(env_spec, group); + if (!cvd_flags_res.has_value()) { + group.SetAllStates(cvd::INSTANCE_STATE_PREPARE_FAILED); + (void)instance_manager_.UpdateInstanceGroup(group); + } + CvdFlags cvd_flags = CF_EXPECT(std::move(cvd_flags_res)); - auto res = LoadGroup(request, group, std::move(cvd_flags)); + auto res = LoadGroup(request, group, std::move(cvd_flags), in_progress_lock); if (!res.has_value()) { auto first_instance_state = group.Instances()[0].State(); // The failure could have occurred during prepare(fetch) or start @@ -197,7 +207,8 @@ Result LoadConfigsCommand::Handle(const CommandRequest& request) { Result LoadConfigsCommand::LoadGroup(const CommandRequest& request, LocalInstanceGroup& group, - CvdFlags cvd_flags) { + CvdFlags cvd_flags, + SharedFD& in_progress_lock) { auto mkdir_res = EnsureDirectoryExists(group.HomeDir(), 0775, /* group_name */ ""); if (!mkdir_res.has_value()) { @@ -229,7 +240,8 @@ Result LoadConfigsCommand::LoadGroup(const CommandRequest& request, CommandRequest start_cmd = CF_EXPECT(BuildStartCommand(request, cvd_flags, group)); std::unique_ptr start_handler = - std::make_unique(instance_manager_); + std::make_unique(instance_manager_, + std::move(in_progress_lock)); CF_EXPECT(start_handler->Handle(start_cmd)); return {}; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.h b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.h index 24d1806e1db..16bfb650af5 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.h +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.h @@ -17,6 +17,7 @@ #include +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/flag_parser/flag.h" #include "cuttlefish/host/commands/cvd/cli/command_request.h" #include "cuttlefish/host/commands/cvd/cli/commands/command_handler.h" @@ -48,7 +49,8 @@ class LoadConfigsCommand : public CvdCommandHandler { private: Result LoadGroup(const CommandRequest& request, - LocalInstanceGroup& group, CvdFlags cvd_flags); + LocalInstanceGroup& group, CvdFlags cvd_flags, + SharedFD& in_progress_lock); InstanceManager& instance_manager_; LoadFlags flags_; diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/remove.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/remove.cpp index 747c7a415e0..60ad8abec30 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/remove.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/remove.cpp @@ -74,6 +74,7 @@ Result RemoveCvdCommandHandler::Handle(const CommandRequest& request) { return CF_ERR(NoGroupMessage(request)); } auto group = CF_EXPECT(selector::SelectGroup(instance_manager_, request)); + group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(group)); Result stop_res = StopGroup(group); if (!stop_res.has_value()) { @@ -82,7 +83,9 @@ Result RemoveCvdCommandHandler::Handle(const CommandRequest& request) { "kill any remaining device processes."; } - CF_EXPECT(instance_manager_.RemoveInstanceGroup(group)); + bool removed = CF_EXPECT(instance_manager_.RemoveInstanceGroup(group)); + CF_EXPECT(removed == true, + "Instance group no longer exists or changed concurrently"); return {}; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.cpp index 4aea9fc938b..441106c191c 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -376,8 +377,9 @@ bool CanBypassToSingleInstance(const LocalInstance& instance, } // namespace CvdStartCommandHandler::CvdStartCommandHandler( - InstanceManager& instance_manager) - : instance_manager_(instance_manager) { + InstanceManager& instance_manager, SharedFD preacquired_in_progress_lock) + : instance_manager_(instance_manager), + preacquired_in_progress_lock_(std::move(preacquired_in_progress_lock)) { own_flags_.daemon = true; own_flags_.print_group_format = isatty(0) ? "human" : "json"; } @@ -396,6 +398,13 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { request.Selectors().instance_names->size() == 1) { auto [instance, group] = CF_EXPECT(selector::SelectInstance(instance_manager_, request)); + group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(group)); + for (const LocalInstance& refreshed_inst : group.Instances()) { + if (refreshed_inst.Id() == instance.Id()) { + instance = refreshed_inst; + break; + } + } if (CanBypassToSingleInstance(instance, group, subcmd_args)) { CF_EXPECT(LaunchSingleInstance(instance, group, request)); @@ -413,9 +422,18 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { CF_EXPECT(selector::SelectGroup(instance_manager_, request), "Failed to select group to start, did you mean 'cvd create'?"); + SharedFD in_progress_lock = std::move(preacquired_in_progress_lock_); + if (!in_progress_lock->IsOpen()) { + in_progress_lock = + CF_EXPECT(InstanceManager::AcquireGroupInProgressLock(group)); + } + group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState( + group, /*caller_holds_in_progress_lock=*/true)); + CF_EXPECT(!group.HasActiveInstances(), "Selected instance group is already started, use `cvd create` to " "create a new one."); + CF_EXPECT(instance_manager_.EnsureInstanceLocksInUse(group)); std::unordered_map envs = request.Env(); @@ -443,8 +461,10 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { Command command = CF_EXPECT(ConstructCvdNonHelpCommand( bin, group, subcmd_args, envs, request, memfd)); + std::mutex group_mutex; Result> handle_res = - PushInterruptListener([this, &group, stop_monitor](int signal) { + PushInterruptListener([this, &group, &group_mutex, + stop_monitor](int signal) { LOG(WARNING) << strsignal(signal) << " signal received, cleanning up"; stop_monitor(); Result interrupt_res = subprocess_waiter_.Interrupt(); @@ -455,38 +475,45 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { "run `cvd reset` to ensure a clean state"; } - group.SetAllStates(cvd::INSTANCE_STATE_CANCELLED); - Result update_res = instance_manager_.UpdateInstanceGroup(group); - if (!update_res.has_value()) { - LOG(ERROR) << "Failed to update group status: " << update_res.error(); + { + std::lock_guard lock(group_mutex); + group.SetAllStates(cvd::INSTANCE_STATE_CANCELLED); + Result update_res = + instance_manager_.UpdateInstanceGroup(group); + if (!update_res.has_value()) { + LOG(ERROR) << "Failed to update group status: " + << update_res.error(); + } } // It's technically possible for the group's state to be set to - // "running" before abort has a chance to run, but that can only happen - // if the instances are indeed running, so it's OK. + // "running" before abort has a chance to run, but that can only + // happen if the instances are indeed running, so it's OK. std::abort(); }); std::unique_ptr listener_handle = CF_EXPECT(std::move(handle_res)); - group.SetAllStates(cvd::INSTANCE_STATE_STARTING); - group.SetStartTime(CvdServerClock::now()); - CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); + { + std::lock_guard lock(group_mutex); + group.SetAllStates(cvd::INSTANCE_STATE_STARTING); + group.SetStartTime(CvdServerClock::now()); + CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); + } Result monitor_res; #ifdef __linux__ std::thread monitor_thread; if (!own_flags_.daemon) { - const LocalInstance& instance = *group.Instances().begin(); - const std::string assemble_log = - absl::StrCat(instance.AssemblyDirectory(), "/", kLogNameAssembleCvd); + const LocalInstance first_instance = *group.Instances().begin(); + const std::string assemble_log = absl::StrCat( + first_instance.AssemblyDirectory(), "/", kLogNameAssembleCvd); (void)RemoveFile(assemble_log); - for (const auto& log : CF_EXPECT(instance.LogsFilenames())) { + for (const auto& log : CF_EXPECT(first_instance.LogsFilenames())) { (void)RemoveFile(log); } - monitor_thread = std::thread([&group, stop_read, &monitor_res]() { - const LocalInstance& first_instance = *group.Instances().begin(); + monitor_thread = std::thread([first_instance, stop_read, &monitor_res]() { monitor_res = MonitorLogs(first_instance, stop_read); }); } @@ -505,8 +532,8 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { }; #endif - Result launch_res = - LaunchDeviceInterruptible(std::move(command), group, envs, request); + Result launch_res = LaunchDeviceInterruptible( + std::move(command), group, group_mutex, envs, request); if (!launch_res.has_value()) { if (!own_flags_.daemon) { @@ -518,8 +545,12 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { return launch_res; } - group.SetAllStates(cvd::INSTANCE_STATE_RUNNING); - CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); + { + std::lock_guard lock(group_mutex); + group.SetAllStates(cvd::INSTANCE_STATE_RUNNING); + CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); + } + in_progress_lock = SharedFD(); if (!own_flags_.daemon) { listener_handle.reset(); @@ -607,7 +638,7 @@ Result CvdStartCommandHandler::LaunchDevice( } Result CvdStartCommandHandler::LaunchDeviceInterruptible( - Command command, LocalInstanceGroup& group, + Command command, LocalInstanceGroup& group, std::mutex& group_mutex, const std::unordered_map& envs, const CommandRequest& request) { // cvd_internal_start uses the config from the previous invocation to @@ -624,6 +655,7 @@ Result CvdStartCommandHandler::LaunchDeviceInterruptible( const Result start_res = LaunchDevice(std::move(command), group, group.Instances()); if (!start_res.has_value()) { + std::lock_guard lock(group_mutex); group.SetAllStates(cvd::INSTANCE_STATE_BOOT_FAILED); CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); return start_res; @@ -635,6 +667,28 @@ Result CvdStartCommandHandler::LaunchDeviceInterruptible( Result CvdStartCommandHandler::LaunchSingleInstance( LocalInstance& instance, LocalInstanceGroup& group, const CommandRequest& request) { + SharedFD in_progress_lock = std::move(preacquired_in_progress_lock_); + if (!in_progress_lock->IsOpen()) { + in_progress_lock = + CF_EXPECT(InstanceManager::AcquireGroupInProgressLock(group)); + } + group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState( + group, /*caller_holds_in_progress_lock=*/true)); + bool found_instance = false; + for (const LocalInstance& refreshed_inst : group.Instances()) { + if (refreshed_inst.Id() == instance.Id()) { + instance = refreshed_inst; + found_instance = true; + break; + } + } + CF_EXPECT(found_instance == true, + "Selected instance no longer exists in group"); + CF_EXPECT( + CanBypassToSingleInstance(instance, group, request.SubcommandArguments()), + "Selected instance cannot be started individually in its current state"); + CF_EXPECT(instance_manager_.EnsureInstanceLocksInUse(group)); + const std::string bin_path = group.HostArtifactsPath() + "/bin/run_cvd"; std::unordered_map run_cvd_envs = request.Env(); run_cvd_envs[kCuttlefishInstanceEnvVarName] = std::to_string(instance.Id()); @@ -679,7 +733,6 @@ Result CvdStartCommandHandler::LaunchSingleInstance( }; set_instance_state(cvd::INSTANCE_STATE_STARTING); - group.SetStartTime(CvdServerClock::now()); CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); const Result start_res = @@ -692,6 +745,7 @@ Result CvdStartCommandHandler::LaunchSingleInstance( set_instance_state(cvd::INSTANCE_STATE_RUNNING); CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); + in_progress_lock = SharedFD(); const Json::Value group_json = CF_EXPECT(group.FetchStatus()); std::cout << group_json.toStyledString(); diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.h b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.h index b8f2b229aa0..103f5ab3305 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.h +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.h @@ -16,10 +16,13 @@ #pragma once +#include #include #include +#include #include +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/flag_parser/flag.h" #include "cuttlefish/host/commands/cvd/cli/commands/command_handler.h" #include "cuttlefish/host/commands/cvd/instances/instance_manager.h" @@ -32,7 +35,8 @@ namespace cuttlefish { class CvdStartCommandHandler : public CvdCommandHandler { public: - CvdStartCommandHandler(InstanceManager& instance_manager); + CvdStartCommandHandler(InstanceManager& instance_manager, + SharedFD preacquired_in_progress_lock = {}); Result Handle(const CommandRequest& request) override; std::vector CmdList() const override; @@ -54,7 +58,7 @@ class CvdStartCommandHandler : public CvdCommandHandler { const std::vector& instances); Result LaunchDeviceInterruptible( - Command command, LocalInstanceGroup& group, + Command command, LocalInstanceGroup& group, std::mutex& group_mutex, const std::unordered_map& envs, const CommandRequest& request); @@ -62,6 +66,7 @@ class CvdStartCommandHandler : public CvdCommandHandler { std::vector BuildOwnFlags(); InstanceManager& instance_manager_; + SharedFD preacquired_in_progress_lock_; SubprocessWaiter subprocess_waiter_; struct { std::vector host_substitutions; diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/stop.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/stop.cpp index a8603c7521f..2cd844df4d7 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/stop.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/stop.cpp @@ -54,6 +54,7 @@ Result CvdStopCommandHandler::Handle(const CommandRequest& request) { } auto group = CF_EXPECT(selector::SelectGroup(instance_manager_, request)); + group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(group)); CF_EXPECT(group.HasActiveInstances(), "Selected group is not running"); CF_EXPECT(ConsumeFlags(CF_EXPECT(Flags(request)), cmd_args, diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/instances/BUILD.bazel index 3f2384370c4..7c3d2ea587e 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/BUILD.bazel @@ -47,6 +47,19 @@ cf_cc_library( ], ) +cf_cc_test( + name = "data_viewer_test", + srcs = ["data_viewer_test.cpp"], + deps = [ + ":cvd_persistent_data", + ":data_viewer", + "//cuttlefish/common/libs/fs:fd", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "@abseil-cpp//absl/strings", + ], +) + cf_cc_library( name = "device_name", srcs = ["device_name.cpp"], @@ -138,6 +151,7 @@ cf_cc_library( "//cuttlefish/host/libs/log_names", "//cuttlefish/host/libs/screen_recording_controls", "//cuttlefish/io:write_exact", + "//cuttlefish/posix:realpath", "//cuttlefish/posix:symlink", "//cuttlefish/process:command", "//cuttlefish/process:managed_stdio", @@ -163,12 +177,14 @@ cf_cc_library( ":cvd_persistent_data", ":instances", "//cuttlefish/common/libs/fs", + "//cuttlefish/common/libs/fs:fd", "//cuttlefish/common/libs/utils:contains", "//cuttlefish/common/libs/utils:files", "//cuttlefish/common/libs/utils:json", "//cuttlefish/common/libs/utils:signals", "//cuttlefish/common/libs/utils:users", "//cuttlefish/files:directory_exists", + "//cuttlefish/files:file_exists", "//cuttlefish/files:recursively_remove_directory", "//cuttlefish/host/commands/cvd/cli:command_request", "//cuttlefish/host/commands/cvd/cli:utils", @@ -186,6 +202,7 @@ cf_cc_library( "//cuttlefish/host/libs/config:cuttlefish_config", "//cuttlefish/host/libs/screen_recording_controls", "//cuttlefish/metrics:device_metrics_orchestration", + "//cuttlefish/posix:realpath", "//cuttlefish/posix:remove", "//cuttlefish/posix:symlink", "//cuttlefish/process:command", @@ -215,7 +232,20 @@ cf_cc_test( cf_cc_test( name = "local_instance_test", srcs = ["local_instance_test.cpp"], - deps = ["//cuttlefish/host/commands/cvd/instances"], + deps = [ + ":cvd_persistent_data", + ":data_viewer", + "//cuttlefish/common/libs/fs:fd", + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/files:directory_exists", + "//cuttlefish/files:recursively_remove_directory", + "//cuttlefish/files:write_new_file", + "//cuttlefish/host/commands/cvd/instances", + "//cuttlefish/metrics:device_metrics_orchestration", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "@abseil-cpp//absl/strings", + ], ) cf_cc_library( diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.cpp index e7dcbcff133..b452d911e29 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.cpp @@ -16,6 +16,10 @@ #include "cuttlefish/host/commands/cvd/instances/data_viewer.h" +#include +#include +#include + #include #include @@ -23,13 +27,15 @@ #include "cuttlefish/common/libs/fs/fd.h" #include "cuttlefish/common/libs/fs/shared_fd.h" +#include "cuttlefish/common/libs/utils/signals.h" #include "cuttlefish/io/string.h" #include "cuttlefish/io/write_exact.h" namespace cuttlefish { Result DataViewer::LockBackingFile(int op) const { - Fd fd = CF_EXPECT(Fd::Open(backing_file_, O_CREAT | O_RDWR, 0640)); + Fd fd = + CF_EXPECT(Fd::Open(backing_file_, O_CREAT | O_RDWR | O_CLOEXEC, 0640)); CF_EXPECTF(fd.Flock(op), "Failed to acquire lock for instance database backing file: {}", fd.StrError()); @@ -48,6 +54,15 @@ Result DataViewer::StoreData(SharedFD fd, const cvd::PersistentData& data) { std::string str; CF_EXPECT(data.SerializeToString(&str), "Failed to serialize data"); + // Block signals while writing to the instance database file. This reduces + // the chances of corrupting the file. + sigset_t all_signals; + sigfillset(&all_signals); + SignalMasker blocker(all_signals); + // Overwrite the file contents, don't append. + CF_EXPECTF(fd->Truncate(0), "Failed to truncate fd: {}", fd->StrError()); + CF_EXPECTF(fd->LSeek(0, SEEK_SET) >= 0, "Failed to seek to 0: {}", + fd->StrError()); CF_EXPECT(WriteExact(*fd, str), "Failed to write to backing file"); return {}; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.h b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.h index 7982a4a02dc..06c36e8dd07 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/common/libs/utils/signals.h" @@ -69,26 +70,35 @@ class DataViewer { Result WithExclusiveLock( std::function(cvd::PersistentData&)> task) { DeadlockProtector dp(*this); - auto fd = CF_EXPECT(LockBackingFile(LOCK_SH)); + auto fd = CF_EXPECT(LockBackingFile(LOCK_EX)); auto data = CF_EXPECT(LoadData(fd)); auto res = task(data); if (!res.has_value()) { // Don't update if there is an error return res; } - // Block signals while writing to the instance database file. This reduces - // the chances of corrupting the file. - sigset_t all_signals; - sigfillset(&all_signals); - SignalMasker blocker(all_signals); - // Overwrite the file contents, don't append - CF_EXPECTF(fd->Truncate(0), "Failed to truncate fd: {}", fd->StrError()); - CF_EXPECTF(fd->LSeek(0, SEEK_SET) >= 0, "Failed to seek to 0: {}", - fd->StrError()); CF_EXPECT(StoreData(fd, data)); return res; } + /** + * Like `WithExclusiveLock`, except the backing file is only truncated and + * rewritten when the second element of the returned pair is `true` + * (indicating `data` was modified). + */ + template + Result WithConditionalExclusiveLock( + std::function>(cvd::PersistentData&)> task) { + DeadlockProtector dp(*this); + auto fd = CF_EXPECT(LockBackingFile(LOCK_EX)); + auto data = CF_EXPECT(LoadData(fd)); + auto [value, modified] = CF_EXPECT(task(data)); + if (modified) { + CF_EXPECT(StoreData(fd, data)); + } + return std::move(value); + } + private: // Opens and locks the backing file. The lock will be dropped when the file // descriptor closes. diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer_test.cpp new file mode 100644 index 00000000000..988284fa539 --- /dev/null +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer_test.cpp @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/commands/cvd/instances/data_viewer.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "gmock/gmock-matchers.h" +#include "gtest/gtest.h" + +#include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/host/commands/cvd/instances/cvd_persistent_data.pb.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +class TempDbFile { + public: + TempDbFile() { + const char* tmpdir = getenv("TEST_TMPDIR"); + std::string base = + (tmpdir != nullptr && tmpdir[0] != '\0') ? tmpdir : "/tmp"; + std::string tmpl = absl::StrCat(base, "/cf_data_viewer_test.XXXXXX"); + int fd = mkstemp(tmpl.data()); + if (fd >= 0) { + close(fd); + path_ = tmpl; + } + } + + ~TempDbFile() { + if (!path_.empty()) { + unlink(path_.c_str()); + } + } + + const std::string& Path() const { return path_; } + + private: + std::string path_; +}; + +TEST(DataViewerTest, WithExclusiveLockAcquiresExclusiveFlock) { + TempDbFile db_file; + ASSERT_FALSE(db_file.Path().empty()); + DataViewer viewer(db_file.Path()); + + Result res = viewer.WithExclusiveLock( + [&db_file](cvd::PersistentData& data) -> Result { + // An independent file description must fail `LOCK_SH | LOCK_NB` while + // `WithExclusiveLock` holds `LOCK_EX`. + Fd probe = CF_EXPECT(Fd::Open(db_file.Path(), O_RDONLY | O_CLOEXEC)); + Result probe_lock = probe.Flock(LOCK_SH | LOCK_NB); + CF_EXPECT(!probe_lock.has_value(), + "Expected LOCK_SH | LOCK_NB to block against LOCK_EX"); + CF_EXPECT(*probe_lock.error() == EWOULDBLOCK || + *probe_lock.error() == EAGAIN); + data.add_instance_groups()->set_name("grp_1"); + return {}; + }); + ASSERT_THAT(res, IsOk()); + + Result count = viewer.WithSharedLock( + [](const cvd::PersistentData& data) -> Result { + return data.instance_groups_size(); + }); + ASSERT_THAT(count, IsOkAndValue(1)); +} + +TEST(DataViewerTest, ConcurrentWritersPreserveAllUpdates) { + TempDbFile db_file; + ASSERT_FALSE(db_file.Path().empty()); + + constexpr int kNumThreads = 8; + constexpr int kIterationsPerThread = 25; + std::vector threads; + threads.reserve(kNumThreads); + for (int t = 0; t < kNumThreads; ++t) { + threads.emplace_back([&db_file, t]() { + DataViewer viewer(db_file.Path()); + for (int i = 0; i < kIterationsPerThread; ++i) { + Result res = viewer.WithExclusiveLock( + [t, i](cvd::PersistentData& data) -> Result { + auto* grp = data.add_instance_groups(); + grp->set_name(absl::StrCat("grp_", t, "_", i)); + grp->set_home_directory(absl::StrCat("/tmp/home_", t, "_", i)); + return {}; + }); + EXPECT_THAT(res, IsOk()); + } + }); + } + for (std::thread& th : threads) { + th.join(); + } + + DataViewer reader(db_file.Path()); + Result total = reader.WithSharedLock( + [](const cvd::PersistentData& data) -> Result { + return data.instance_groups_size(); + }); + ASSERT_THAT(total, IsOkAndValue(kNumThreads * kIterationsPerThread)); +} + +TEST(DataViewerTest, WithConditionalExclusiveLockSkipsRewriteWhenUnmodified) { + TempDbFile db_file; + ASSERT_FALSE(db_file.Path().empty()); + DataViewer viewer(db_file.Path()); + + ASSERT_THAT(viewer.WithExclusiveLock( + [](cvd::PersistentData& data) -> Result { + data.add_instance_groups()->set_name("grp_initial"); + return {}; + }), + IsOk()); + + // Make the backing file's mtime in the past and confirm an unmodified + // `WithConditionalExclusiveLock` does not truncate or rewrite it. + struct timespec past[2] = {{.tv_sec = 1000000, .tv_nsec = 0}, + {.tv_sec = 1000000, .tv_nsec = 0}}; + ASSERT_EQ(utimensat(AT_FDCWD, db_file.Path().c_str(), past, 0), 0); + + Result noop = viewer.WithConditionalExclusiveLock( + [](cvd::PersistentData& data) -> Result> { + data.add_instance_groups()->set_name("unpersisted"); + return std::make_pair(42, false); + }); + ASSERT_THAT(noop, IsOkAndValue(42)); + + struct stat st{}; + ASSERT_EQ(stat(db_file.Path().c_str(), &st), 0); + EXPECT_EQ(st.st_mtim.tv_sec, 1000000); + + Result groups_after_noop = viewer.WithSharedLock( + [](const cvd::PersistentData& data) -> Result { + return data.instance_groups_size(); + }); + ASSERT_THAT(groups_after_noop, IsOkAndValue(1)); + + Result modified = viewer.WithConditionalExclusiveLock( + [](cvd::PersistentData& data) -> Result> { + data.add_instance_groups()->set_name("grp_second"); + return std::make_pair(99, true); + }); + ASSERT_THAT(modified, IsOkAndValue(99)); + + Result groups_after_mod = viewer.WithSharedLock( + [](const cvd::PersistentData& data) -> Result { + return data.instance_groups_size(); + }); + ASSERT_THAT(groups_after_mod, IsOkAndValue(2)); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.cpp index 5da732b86e7..6312d5ed85d 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.cpp @@ -16,6 +16,7 @@ #include "cuttlefish/host/commands/cvd/instances/instance_database.h" +#include #include #include #include @@ -33,6 +34,7 @@ #include "cuttlefish/host/commands/cvd/instances/local_instance.h" #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" #include "cuttlefish/host/libs/config/config_constants.h" +#include "cuttlefish/posix/realpath.h" namespace cuttlefish { @@ -127,7 +129,12 @@ Result InstanceDatabase::AddInstanceGroup(LocalInstanceGroup group) { CF_EXPECTF(IsValidInstanceName(instance_proto.name()), "instance_name \"{}\" is invalid", instance_proto.name()); } - auto add_res = viewer_.WithExclusiveLock([&group]( + std::optional new_real_home; + if (Result resolved = RealPath(group.HomeDir()); + resolved.has_value()) { + new_real_home = std::move(*resolved); + } + auto add_res = viewer_.WithExclusiveLock([&group, &new_real_home]( cvd::PersistentData& data) -> Result { if (group.group_proto_->name().empty()) { @@ -140,8 +147,17 @@ Result InstanceDatabase::AddInstanceGroup(LocalInstanceGroup group) { for (const auto& existing_group : data.instance_groups()) { CF_EXPECT_NE(existing_group.home_directory(), group.group_proto_->home_directory(), - "An instance group already exists with HOME directory: {}" + "An instance group already exists with HOME directory: " << group.group_proto_->home_directory()); + if (new_real_home.has_value()) { + Result existing_real_home = + RealPath(existing_group.home_directory()); + if (existing_real_home.has_value()) { + CF_EXPECT_NE(*existing_real_home, *new_real_home, + "An instance group already exists with HOME directory: " + << *new_real_home); + } + } } std::unordered_map ids_to_name_map; for (const auto& group_proto : data.instance_groups()) { @@ -186,19 +202,118 @@ Result InstanceDatabase::UpdateInstanceGroup( return {}; } -Result InstanceDatabase::RemoveInstanceGroup( - const std::string& group_name) { - return viewer_.WithExclusiveLock([&group_name]( - cvd::PersistentData& data) { - auto mutable_groups = data.mutable_instance_groups(); - for (auto it = mutable_groups->begin(); it != mutable_groups->end(); ++it) { - if (it->name() == group_name) { - mutable_groups->erase(it); - return true; - } - } +Result InstanceDatabase::BatchCompareAndUpdateInstanceStates( + const std::vector& updates) { + if (updates.empty()) { + return false; + } + return viewer_.WithConditionalExclusiveLock( + [&updates](cvd::PersistentData& data) -> Result> { + bool any_updated = false; + for (const GroupStateUpdate& update : updates) { + if (update.updated_states.empty()) { + continue; + } + const cvd::InstanceGroup& expected = update.snapshot.Proto(); + for (cvd::InstanceGroup& group_proto : + *data.mutable_instance_groups()) { + if (group_proto.name() != expected.name() || + group_proto.home_directory() != expected.home_directory() || + group_proto.start_time_sec() != expected.start_time_sec() || + group_proto.instances_size() != expected.instances_size()) { + continue; + } + bool states_match = true; + for (int i = 0; i < group_proto.instances_size(); ++i) { + const cvd::Instance& inst = group_proto.instances(i); + const cvd::Instance& expected_inst = expected.instances(i); + if (inst.id() != expected_inst.id() || + inst.state() != expected_inst.state()) { + states_match = false; + break; + } + } + if (!states_match) { + break; + } + if (update.precondition && !update.precondition()) { + break; + } + for (int i = 0; i < group_proto.instances_size(); ++i) { + cvd::Instance* inst = group_proto.mutable_instances(i); + auto state_it = update.updated_states.find(inst->id()); + if (state_it != update.updated_states.end() && + inst->state() != state_it->second) { + inst->set_state(state_it->second); + any_updated = true; + } + } + break; + } + } + return std::make_pair(any_updated, any_updated); + }); +} + +Result InstanceDatabase::CompareAndUpdateInstanceStates( + const LocalInstanceGroup& snapshot, + const std::unordered_map& updated_states, + std::function precondition) { + if (updated_states.empty()) { return false; + } + std::vector batch; + batch.push_back(GroupStateUpdate{ + .snapshot = snapshot, + .updated_states = updated_states, + .precondition = std::move(precondition), }); + return BatchCompareAndUpdateInstanceStates(batch); +} + +Result InstanceDatabase::RemoveInstanceGroup( + const std::string& group_name) { + return viewer_.WithConditionalExclusiveLock( + [&group_name]( + cvd::PersistentData& data) -> Result> { + auto mutable_groups = data.mutable_instance_groups(); + for (auto it = mutable_groups->begin(); it != mutable_groups->end(); + ++it) { + if (it->name() == group_name) { + mutable_groups->erase(it); + return std::make_pair(true, true); + } + } + return std::make_pair(false, false); + }); +} + +Result InstanceDatabase::RemoveInstanceGroup( + const LocalInstanceGroup& group, std::function precondition) { + return viewer_.WithConditionalExclusiveLock( + [&group, &precondition]( + cvd::PersistentData& data) -> Result> { + auto mutable_groups = data.mutable_instance_groups(); + for (auto it = mutable_groups->begin(); it != mutable_groups->end(); + ++it) { + if (it->name() != group.GroupName() || + it->home_directory() != group.HomeDir() || + it->start_time_sec() != group.Proto().start_time_sec()) { + continue; + } + LocalInstanceGroup existing = + CF_EXPECT(LocalInstanceGroup::Create(*it)); + CF_EXPECT(!existing.HasActiveInstances(), + "Group still contains active instances"); + if (precondition) { + CF_EXPECT(precondition() == true, + "Group transition is currently in progress"); + } + mutable_groups->erase(it); + return std::make_pair(true, true); + } + return std::make_pair(false, false); + }); } Result> InstanceDatabase::FindGroups( diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.h b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.h index 56d8dada844..fb88cfda816 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.h @@ -18,8 +18,10 @@ #include +#include #include #include +#include #include #include #include @@ -60,9 +62,33 @@ class InstanceDatabase { */ Result AddInstanceGroup(LocalInstanceGroup group); Result UpdateInstanceGroup(const LocalInstanceGroup& group); + struct GroupStateUpdate { + LocalInstanceGroup snapshot; + std::unordered_map updated_states; + std::function precondition; + }; + + // Atomically updates instance states across one or more groups in a single + // exclusive database lock acquisition. Each entry in `updates` is applied iff + // the stored group still matches `snapshot` and satisfies `precondition`. + Result BatchCompareAndUpdateInstanceStates( + const std::vector& updates); + + // Convenience wrapper around `BatchCompareAndUpdateInstanceStates` for a + // single group. + Result CompareAndUpdateInstanceStates( + const LocalInstanceGroup& snapshot, + const std::unordered_map& updated_states, + std::function precondition = nullptr); Result> InstanceGroups() const; Result RemoveInstanceGroup(const std::string& group_name); + // Removes `group` if it still matches `group` in the database, has no active + // instances, and satisfies `precondition` (if provided) under the exclusive + // database lock. + Result RemoveInstanceGroup( + const LocalInstanceGroup& group, + std::function precondition = nullptr); /** * Empties the database and returns the recently deleted instance groups. */ diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database_test.cpp index 2fadfd891ce..0ea57228fa8 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database_test.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database_test.cpp @@ -406,5 +406,42 @@ TEST_F(CvdInstanceDatabaseTest, UpdateInstances) { EXPECT_EQ(find_res->Instances()[1].State(), cvd::INSTANCE_STATE_STARTING); } +TEST_F(CvdInstanceDatabaseTest, CompareAndUpdateInstanceStatesUsesSnapshotCas) { + if (!SetUpOk()) { + GTEST_SKIP() << Error().msg; + } + auto& db = GetDb(); + + Result grp_res = + LocalInstanceGroup::Builder("cas_grp").AddInstance(1, "ins1").Build(); + ASSERT_THAT(grp_res, IsOk()); + ASSERT_THAT(db.AddInstanceGroup(*grp_res), IsOk()); + + Result stale_snapshot = + db.FindGroup({.group_name = "cas_grp"}); + ASSERT_THAT(stale_snapshot, IsOk()); + + // Advance state in DB to RUNNING via an independent lookup + // (`LocalInstanceGroup` shares `group_proto_` across shallow copies), then + // verify CAS against the stale PREPARING snapshot is a no-op while CAS + // against RUNNING succeeds. + Result running = db.FindGroup({.group_name = "cas_grp"}); + ASSERT_THAT(running, IsOk()); + running->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(db.UpdateInstanceGroup(*running), IsOk()); + + ASSERT_THAT(db.CompareAndUpdateInstanceStates( + *stale_snapshot, {{1, cvd::INSTANCE_STATE_STOPPED}}), + IsOkAndValue(false)); + ASSERT_THAT(db.CompareAndUpdateInstanceStates( + *running, {{1, cvd::INSTANCE_STATE_STOPPED}}), + IsOkAndValue(true)); + + Result final_grp = + db.FindGroup({.group_name = "cas_grp"}); + ASSERT_THAT(final_grp, IsOk()); + EXPECT_EQ(final_grp->Instances()[0].State(), cvd::INSTANCE_STATE_STOPPED); +} + } // namespace selector } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.cpp index 752d6a81220..a24584599f7 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.cpp @@ -16,54 +16,278 @@ #include "cuttlefish/host/commands/cvd/instances/instance_manager.h" +#include +#include +#include +#include +#include + #include +#include #include #include #include #include +#include +#include +#include #include #include #include "absl/log/log.h" #include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" #include "android-base/file.h" #include "fmt/format.h" +#include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/common/libs/utils/files.h" #include "cuttlefish/files/directory_exists.h" +#include "cuttlefish/files/file_exists.h" #include "cuttlefish/files/recursively_remove_directory.h" #include "cuttlefish/host/commands/cvd/cli/commands/host_tool_target.h" +#include "cuttlefish/host/commands/cvd/instances/cvd_persistent_data.pb.h" #include "cuttlefish/host/commands/cvd/instances/local_instance.h" #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" #include "cuttlefish/host/commands/cvd/instances/lock/instance_lock.h" #include "cuttlefish/host/commands/cvd/instances/lock/lock_file.h" #include "cuttlefish/host/commands/cvd/instances/stop.h" #include "cuttlefish/host/commands/cvd/utils/common.h" +#include "cuttlefish/host/libs/config/config_constants.h" #include "cuttlefish/host/libs/config/config_utils.h" #include "cuttlefish/metrics/device_metrics_orchestration.h" +#include "cuttlefish/posix/realpath.h" #include "cuttlefish/posix/remove.h" #include "cuttlefish/posix/symlink.h" +#include "cuttlefish/process/proc_file_utils.h" #include "cuttlefish/result/result.h" namespace cuttlefish { namespace { -Result RemoveGroupDirectory(const LocalInstanceGroup& group) { - std::string per_user_dir = PerUserDir(); - if (!absl::StartsWith(group.HomeDir(), per_user_dir)) { +constexpr absl::string_view kGroupInProgressLockName = "in_progress.lock"; + +struct ActiveRunCvdProc { + std::string home; + std::optional real_home; + std::optional instance_id; +}; + +Result> CollectActiveRunCvdProcs() { + std::vector procs; + std::vector pids = + CF_EXPECT(CollectPidsByExecName("run_cvd", getuid())); + for (pid_t pid : pids) { + Result> envs = GetEnvs(pid); + if (!envs.has_value()) { + continue; + } + auto home_it = envs->find("HOME"); + if (home_it == envs->end() || home_it->second.empty()) { + continue; + } + ActiveRunCvdProc proc{ + .home = home_it->second, + }; + if (Result real_home = RealPath(proc.home); + real_home.has_value()) { + proc.real_home = std::move(*real_home); + } + if (auto id_it = envs->find(kCuttlefishInstanceEnvVarName); + id_it != envs->end()) { + unsigned parsed_id = 0; + if (absl::SimpleAtoi(id_it->second, &parsed_id)) { + proc.instance_id = parsed_id; + } + } + procs.push_back(std::move(proc)); + } + return procs; +} + +bool MatchesActiveRunCvd(const LocalInstanceGroup& group, + const LocalInstance& instance, + const std::vector& procs) { + std::optional real_group_home; + if (Result real_home = RealPath(group.HomeDir()); + real_home.has_value()) { + real_group_home = std::move(*real_home); + } + for (const ActiveRunCvdProc& proc : procs) { + bool home_matches = + (proc.home == group.HomeDir()) || + (real_group_home.has_value() && proc.real_home.has_value() && + *real_group_home == *proc.real_home); + if (!home_matches) { + continue; + } + if (!proc.instance_id.has_value() || *proc.instance_id == instance.Id()) { + return true; + } + } + return false; +} + +bool IsGroupBackingDirsIntact(const LocalInstanceGroup& group) { + if (!DirectoryExists(group.HomeDir())) { + return false; + } + if (!HostToolTarget(group.HostArtifactsPath()) + .GetStartBinName() + .has_value()) { + return false; + } + for (absl::string_view product_dir : + absl::StrSplit(group.ProductOutPath(), ',')) { + if (product_dir.empty() || !DirectoryExists(std::string(product_dir))) { + return false; + } + } + return true; +} + +Result> ProbeGroupStateUpdates( + const LocalInstanceGroup& group, + std::optional>& cached_procs, + bool caller_holds_in_progress_lock) { + std::unordered_map updated_states; + if (!caller_holds_in_progress_lock && + InstanceManager::IsGroupTransitionInProgress(group)) { + return updated_states; + } + + for (const LocalInstance& instance : group.Instances()) { + const cvd::InstanceState state = instance.State(); + const bool check_dead = instance.IsActive(); + const bool check_alive = state == cvd::INSTANCE_STATE_BOOT_FAILED || + state == cvd::INSTANCE_STATE_CANCELLED || + state == cvd::INSTANCE_STATE_STOPPED; + if (!check_dead && !check_alive) { + continue; + } + const auto probe_start = std::chrono::steady_clock::now(); + bool is_alive = + instance.CanConnectToLauncherMonitor(std::chrono::seconds(1)); + if (std::chrono::steady_clock::now() - probe_start > + std::chrono::milliseconds(10)) { + cached_procs.reset(); + } + if (!is_alive && state != cvd::INSTANCE_STATE_STOPPED) { + if (!cached_procs.has_value()) { + cached_procs = CF_EXPECT(CollectActiveRunCvdProcs()); + } + is_alive = MatchesActiveRunCvd(group, instance, *cached_procs); + } + if (check_dead && !is_alive) { + updated_states[instance.Id()] = cvd::INSTANCE_STATE_STOPPED; + } else if (check_alive && is_alive) { + updated_states[instance.Id()] = cvd::INSTANCE_STATE_RUNNING; + } + } + return updated_states; +} + +Result RefreshGroupWithProcs( + LocalInstanceGroup& group, + std::optional>& cached_procs, + InstanceDatabase& instance_db, bool caller_holds_in_progress_lock) { + if (caller_holds_in_progress_lock) { + std::vector reloaded = + CF_EXPECT(instance_db.FindGroups({.group_name = group.GroupName()})); + CF_EXPECTF( + reloaded.size() == 1 && reloaded.front().HomeDir() == group.HomeDir(), + "Instance group '{}' no longer exists in database", group.GroupName()); + group = std::move(reloaded.front()); + } + + std::unordered_map updated_states = + CF_EXPECT(ProbeGroupStateUpdates(group, cached_procs, + caller_holds_in_progress_lock)); + if (updated_states.empty()) { + return {}; + } + + std::function precondition; + if (!caller_holds_in_progress_lock) { + precondition = [&group]() { + return !InstanceManager::IsGroupTransitionInProgress(group); + }; + } + bool updated = CF_EXPECT(instance_db.CompareAndUpdateInstanceStates( + group, updated_states, std::move(precondition))); + if (updated) { + for (LocalInstance& instance : group.Instances()) { + if (auto it = updated_states.find(instance.Id()); + it != updated_states.end()) { + instance.SetState(it->second); + } + } + } else { + Result> reloaded = + instance_db.FindGroups({.group_name = group.GroupName()}); + if (reloaded.has_value() && reloaded->size() == 1 && + reloaded->front().HomeDir() == group.HomeDir()) { + group = std::move(reloaded->front()); + } + } + return {}; +} + +Result RemoveGroupDirectory(const LocalInstanceGroup& group, + const std::string& base_root_dir) { + std::string prefix = absl::EndsWith(base_root_dir, "/") + ? base_root_dir + : absl::StrCat(base_root_dir, "/"); + if (!absl::StartsWith(group.HomeDir(), prefix)) { LOG(WARNING) << "Instance group home directory not under user specific directory(" - << per_user_dir << "), artifacts not deleted"; + << base_root_dir << "), artifacts not deleted"; return {}; } std::string group_directory = CF_EXPECT(GroupDirFromHome(group.HomeDir())); - if (DirectoryExists(group_directory)) { + if (DirectoryExists(group_directory, /*follow_symlinks=*/false)) { CF_EXPECT(RecursivelyRemoveDirectory(group_directory), "Failed to remove group directory"); + } else if (FileExists(group_directory, /*follow_symlinks=*/false)) { + CF_EXPECT(RemoveFile(group_directory), + "Failed to remove group directory symlink"); } return {}; } +enum class ClaimBaseDirResult { + kClaimed, + kAlreadyExists, +}; + +Result TryClaimGroupBaseDir( + const std::optional& custom_base_dir, + const std::string& base_dir, const std::string& base_root_dir) { + CF_EXPECT(EnsureDirectoryExists(base_root_dir)); + if (custom_base_dir.has_value()) { + CF_EXPECT(DirectoryExists(*custom_base_dir)); + if (symlink(custom_base_dir->c_str(), base_dir.c_str()) == 0) { + return ClaimBaseDirResult::kClaimed; + } + if (errno == EEXIST) { + return ClaimBaseDirResult::kAlreadyExists; + } + return CF_ERRNO("Failed to symlink '" << *custom_base_dir << "' to '" + << base_dir << "'"); + } + if (mkdir(base_dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == 0) { + return ClaimBaseDirResult::kClaimed; + } + if (errno == EEXIST) { + return ClaimBaseDirResult::kAlreadyExists; + } + return CF_ERRNO("Failed to create group base directory '" << base_dir << "'"); +} + Result LinkOrMakeDir(std::optional target, const std::string& path) { if (target.has_value()) { @@ -76,11 +300,9 @@ Result LinkOrMakeDir(std::optional target, return {}; } -Result CreateOrLinkGroupDirectories( +Result PopulateGroupSubdirectories( const LocalInstanceGroup& group, InstanceManager::GroupDirectories directories) { - CF_EXPECT( - LinkOrMakeDir(std::move(directories.base_directory), group.BaseDir())); CF_EXPECT(LinkOrMakeDir(std::move(directories.home), group.HomeDir())); CF_EXPECT(EnsureDirectoryExists(group.ArtifactsDir())); CF_EXPECT(LinkOrMakeDir(std::move(directories.host_artifacts_path), @@ -100,7 +322,64 @@ Result StopBin(const std::string& host_android_out) { InstanceManager::InstanceManager(InstanceLockFileManager& lock_manager, InstanceDatabase& instance_db) - : lock_manager_(lock_manager), instance_db_(instance_db) {} + : InstanceManager(lock_manager, instance_db, PerUserDir()) {} + +InstanceManager::InstanceManager(InstanceLockFileManager& lock_manager, + InstanceDatabase& instance_db, + std::string base_root_dir) + : lock_manager_(lock_manager), + instance_db_(instance_db), + base_root_dir_(std::move(base_root_dir)) {} + +Result InstanceManager::AcquireGroupInProgressLock( + const LocalInstanceGroup& group) { + CF_EXPECTF(DirectoryExists(group.BaseDir()), + "Instance group base directory '{}' does not exist", + group.BaseDir()); + std::string lock_path = + fmt::format("{}/{}", group.BaseDir(), kGroupInProgressLockName); + SharedFD fd = + CF_EXPECT(Fd::Open(lock_path, O_CREAT | O_RDWR | O_CLOEXEC, 0600)); + constexpr int kMaxLockRetries = 50; + Result flock_res = fd->Flock(LOCK_EX | LOCK_NB); + for (int retry = 0; + retry < kMaxLockRetries && !flock_res.has_value() && + (*flock_res.error() == EWOULDBLOCK || *flock_res.error() == EAGAIN); + ++retry) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + flock_res = fd->Flock(LOCK_EX | LOCK_NB); + } + if (!flock_res.has_value() && + (*flock_res.error() == EWOULDBLOCK || *flock_res.error() == EAGAIN)) { + return CF_ERR( + "Selected instance group is already starting, use `cvd create` to " + "create a new one."); + } + CF_EXPECT(std::move(flock_res)); + return fd; +} + +bool InstanceManager::IsGroupTransitionInProgress( + const LocalInstanceGroup& group) { + std::string lock_path = + fmt::format("{}/{}", group.BaseDir(), kGroupInProgressLockName); + if (!FileExists(lock_path)) { + return false; + } + Result fd = Fd::Open(lock_path, O_RDONLY | O_CLOEXEC); + if (!fd.has_value()) { + return false; + } + Result flock_res = fd->Flock(LOCK_SH | LOCK_NB); + if (!flock_res.has_value() && + (*flock_res.error() == EWOULDBLOCK || *flock_res.error() == EAGAIN)) { + return true; + } + if (flock_res.has_value()) { + (void)fd->Flock(LOCK_UN | LOCK_NB); + } + return false; +} Result> InstanceManager::FindInstanceWithGroup( @@ -121,36 +400,41 @@ Result> InstanceManager::AllocateAndLockInstanceIds( std::vector instances) { std::set requested; - std::vector requested_lock_files; - // Acquire requested locks first for (const auto& instance : instances) { if (instance.instance_id.has_value()) { unsigned id = instance.instance_id.value(); auto [_, inserted] = requested.insert(id); - // This check avoids a possible deadlock when trying to acquire the lock a - // second time CF_EXPECTF(std::move(inserted), "Requested instance ids must be distinct, but {} is repeated", id); - requested_lock_files.emplace_back( - CF_EXPECT(lock_manager_.AcquireLock(id))); } } + // Acquire requested locks in ascending ID order to avoid ABBA deadlocks. + std::unordered_map> + requested_lock_files; + requested_lock_files.reserve(requested.size()); + for (unsigned id : requested) { + InstanceLockFile lock = CF_EXPECT(lock_manager_.AcquireCurrentUserLock(id)); + bool was_unused = (CF_EXPECT(lock.Status()) == InUseState::kNotInUse); + requested_lock_files.emplace(id, + std::make_pair(std::move(lock), was_unused)); + } std::vector ret; ret.reserve(instances.size()); - auto requested_it = requested_lock_files.begin(); for (auto& instance : instances) { if (instance.instance_id.has_value()) { - CF_EXPECT(requested_it != requested_lock_files.end()); + auto lock_it = requested_lock_files.find(*instance.instance_id); + CF_EXPECT(lock_it != requested_lock_files.end()); ret.emplace_back(InternalInstanceDesc{ - .lock_file = std::move(*requested_it), + .lock_file = std::move(lock_it->second.first), .name = std::move(instance.per_instance_name), + .release_on_error = lock_it->second.second, }); - ++requested_it; } else { ret.emplace_back(InternalInstanceDesc{ .lock_file = CF_EXPECT(lock_manager_.AcquireUnusedLock()), .name = std::move(instance.per_instance_name), + .release_on_error = true, }); } } @@ -158,55 +442,210 @@ InstanceManager::AllocateAndLockInstanceIds( } Result InstanceManager::CreateInstanceGroup( - InstanceGroupParams group_params, GroupDirectories directories) { + InstanceGroupParams group_params, GroupDirectories directories, + SharedFD* out_in_progress_lock) { CF_EXPECT_EQ( group_params.instances.size(), directories.product_out_paths.size(), "Number of product directories doesn't match number of instances"); std::vector instance_descs = CF_EXPECT(AllocateAndLockInstanceIds(std::move(group_params.instances))); + auto release_unused_locks = [this, &instance_descs]() { + for (InternalInstanceDesc& desc : instance_descs) { + if (desc.release_on_error) { + (void)lock_manager_.ReleaseHeldLock(desc.lock_file); + } + } + }; + + constexpr int kMaxBaseDirAttempts = 1024; + std::optional built_group; + for (int attempt = 0; attempt < kMaxBaseDirAttempts; ++attempt) { + LocalInstanceGroup::Builder group_builder(group_params.group_name, + base_root_dir_); + for (const auto& instance_desc : instance_descs) { + if (instance_desc.name.has_value()) { + group_builder.AddInstance(instance_desc.lock_file.Instance(), + instance_desc.name.value()); + } else { + group_builder.AddInstance(instance_desc.lock_file.Instance()); + } + } + Result build_res = group_builder.Build(); + if (!build_res.has_value()) { + release_unused_locks(); + } + LocalInstanceGroup candidate = CF_EXPECT(std::move(build_res)); - LocalInstanceGroup::Builder group_builder(std::move(group_params.group_name)); - for (const auto& instance_desc : instance_descs) { - if (instance_desc.name.has_value()) { - group_builder.AddInstance(instance_desc.lock_file.Instance(), - instance_desc.name.value()); - } else { - group_builder.AddInstance(instance_desc.lock_file.Instance()); + Result claim_res = TryClaimGroupBaseDir( + directories.base_directory, candidate.BaseDir(), base_root_dir_); + if (!claim_res.has_value()) { + release_unused_locks(); } + if (CF_EXPECT(std::move(claim_res)) == ClaimBaseDirResult::kClaimed) { + built_group.emplace(std::move(candidate)); + break; + } + } + if (!built_group.has_value()) { + release_unused_locks(); + return CF_ERR("Failed to allocate unique group base directory"); } - LocalInstanceGroup group = CF_EXPECT(group_builder.Build()); + LocalInstanceGroup group = std::move(*built_group); + + auto cleanup_group_and_locks = [this, &group, &release_unused_locks]() { + (void)RemoveGroupDirectory(group, base_root_dir_); + release_unused_locks(); + }; // The base and other directories are always set to the default location, if // the user provides custom directories the ones in the default locations // become symbolic links to those. - CF_EXPECT(CreateOrLinkGroupDirectories(group, std::move(directories))); + Result dirs_res = + PopulateGroupSubdirectories(group, std::move(directories)); + if (!dirs_res.has_value()) { + cleanup_group_and_locks(); + } + CF_EXPECT(std::move(dirs_res)); - CF_EXPECT(instance_db_.AddInstanceGroup(group)); + SharedFD in_progress_lock; + if (out_in_progress_lock != nullptr) { + Result lock_res = AcquireGroupInProgressLock(group); + if (!lock_res.has_value()) { + cleanup_group_and_locks(); + } + in_progress_lock = CF_EXPECT(std::move(lock_res)); + } + + Result add_res = instance_db_.AddInstanceGroup(group); + if (!add_res.has_value()) { + in_progress_lock = SharedFD(); + cleanup_group_and_locks(); + } + CF_EXPECT(std::move(add_res)); for (auto& instance_desc : instance_descs) { CF_EXPECT(instance_desc.lock_file.Status(InUseState::kInUse)); } + if (out_in_progress_lock != nullptr) { + *out_in_progress_lock = std::move(in_progress_lock); + } GatherVmInstantiationMetrics(group); return group; } +Result InstanceManager::EnsureInstanceLocksInUse( + const LocalInstanceGroup& group) { + std::set sorted_ids; + for (const LocalInstance& instance : group.Instances()) { + if (instance.Id() != 0) { + sorted_ids.insert(instance.Id()); + } + } + std::vector locks; + locks.reserve(sorted_ids.size()); + for (unsigned id : sorted_ids) { + locks.push_back(CF_EXPECT(lock_manager_.AcquireCurrentUserLock(id))); + } + for (InstanceLockFile& lock : locks) { + CF_EXPECT(lock.Status(InUseState::kInUse)); + } + return {}; +} + Result InstanceManager::RemoveInstanceGroup(LocalInstanceGroup group) { CF_EXPECT(!group.HasActiveInstances(), "Group still contains active instances"); - for (auto& instance : group.Instances()) { - if (instance.Id() == 0) { - continue; + CF_EXPECT(!IsGroupTransitionInProgress(group), + "Group transition is currently in progress"); + std::set sorted_ids; + for (const LocalInstance& instance : group.Instances()) { + if (instance.Id() != 0) { + sorted_ids.insert(instance.Id()); + } + } + std::vector held_locks; + held_locks.reserve(sorted_ids.size()); + for (unsigned id : sorted_ids) { + std::optional lock_opt = + CF_EXPECT(lock_manager_.AcquireOwnedLockIfPresent(id)); + if (lock_opt.has_value()) { + held_locks.emplace_back(std::move(*lock_opt)); } - if (auto res = lock_manager_.RemoveLockFile(instance.Id()); + } + + bool removed = CF_EXPECT(instance_db_.RemoveInstanceGroup( + group, [&group]() { return !IsGroupTransitionInProgress(group); })); + if (!removed) { + return false; + } + + for (InstanceLockFile& lock : held_locks) { + if (Result res = lock_manager_.ReleaseHeldLock(lock); !res.has_value()) { - LOG(ERROR) << "Failed to remove instance id lock: " << res.error(); + LOG(ERROR) << "Failed to release instance id lock " << lock.Instance() + << ": " << res.error(); + } + } + CF_EXPECT(RemoveGroupDirectory(group, base_root_dir_)); + return true; +} + +Result InstanceManager::RefreshInstanceGroupState( + LocalInstanceGroup group, bool caller_holds_in_progress_lock) { + std::optional> cached_procs; + CF_EXPECT(RefreshGroupWithProcs(group, cached_procs, instance_db_, + caller_holds_in_progress_lock)); + return group; +} + +Result InstanceManager::RefreshInstanceGroups() { + std::vector groups = + CF_EXPECT(instance_db_.FindGroups({})); + std::optional> cached_procs; + std::vector batch_updates; + for (const LocalInstanceGroup& group : groups) { + std::unordered_map updates = CF_EXPECT( + ProbeGroupStateUpdates(group, cached_procs, + /*caller_holds_in_progress_lock=*/false)); + if (!updates.empty()) { + batch_updates.push_back(InstanceDatabase::GroupStateUpdate{ + .snapshot = group, + .updated_states = std::move(updates), + .precondition = + [group]() { + return !InstanceManager::IsGroupTransitionInProgress(group); + }, + }); } } - CF_EXPECT(RemoveGroupDirectory(group)); + if (!batch_updates.empty()) { + (void)CF_EXPECT( + instance_db_.BatchCompareAndUpdateInstanceStates(batch_updates)); + } + return {}; +} - return CF_EXPECT(instance_db_.RemoveInstanceGroup(group.GroupName())); +Result InstanceManager::PruneBrokenInstanceGroups() { + CF_EXPECT(RefreshInstanceGroups()); + std::vector groups = + CF_EXPECT(instance_db_.FindGroups({})); + for (LocalInstanceGroup& group : groups) { + if (group.HasActiveInstances()) { + continue; + } + if (!IsGroupBackingDirsIntact(group)) { + LOG(INFO) << "Pruning inactive instance group '" << group.GroupName() + << "' with missing backing directories"; + Result remove_res = RemoveInstanceGroup(std::move(group)); + if (!remove_res.has_value()) { + LOG(WARNING) << "Failed to prune broken instance group: " + << remove_res.error(); + } + } + } + return {}; } Result InstanceManager::UpdateInstanceGroup( @@ -263,6 +702,11 @@ Result InstanceManager::StopInstanceGroup( } Result InstanceManager::Clear() { + if (Result refresh_res = RefreshInstanceGroups(); + !refresh_res.has_value()) { + LOG(WARNING) << "Failed to refresh instance groups before clear: " + << refresh_res.error(); + } const std::string config_json_name = android::base::Basename(GetGlobalConfigFileLink()); auto instance_groups = @@ -280,9 +724,9 @@ Result InstanceManager::Clear() { if (instance.Id() <= 0) { continue; } - auto res = lock_manager_.RemoveLockFile(instance.Id()); + auto res = lock_manager_.ReleaseInstanceLock(instance.Id()); if (!res.has_value()) { - LOG(ERROR) << "Failed to remove lock file for instance: " + LOG(ERROR) << "Failed to release lock file for instance: " << res.error(); } } @@ -295,7 +739,7 @@ Result InstanceManager::Clear() { LOG(ERROR) << res.error(); } // TODO: b/471069557 - diagnose unused - Result unused = RemoveGroupDirectory(group); + Result unused = RemoveGroupDirectory(group, base_root_dir_); } std::cerr << "Stopped all known instances\n"; return {}; diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.h b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.h index 8e68c678bc4..c6358f79378 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.h @@ -24,6 +24,7 @@ #include #include +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/host/commands/cvd/instances/instance_database.h" #include "cuttlefish/host/commands/cvd/instances/local_instance.h" #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" @@ -56,12 +57,39 @@ class InstanceManager { std::vector> product_out_paths; }; InstanceManager(InstanceLockFileManager&, InstanceDatabase& instance_db); + InstanceManager(InstanceLockFileManager&, InstanceDatabase& instance_db, + std::string base_root_dir); + + // Acquires an exclusive non-blocking flock on `/in_progress.lock` + // to guard an in-progress prepare or boot transition against concurrent + // state refreshes and duplicate starts. + static Result AcquireGroupInProgressLock( + const LocalInstanceGroup& group); + // Returns true if another open file description currently holds an exclusive + // flock on `/in_progress.lock`. + static bool IsGroupTransitionInProgress(const LocalInstanceGroup& group); Result HasInstanceGroups() const; + // Creates and registers an instance group. If `out_in_progress_lock` is + // non-null, acquires `in_progress.lock` before inserting the group into + // `instance_db_` so no unlocked `PREPARING`/`STOPPED` window exists. Result CreateInstanceGroup( - InstanceGroupParams group_params, GroupDirectories group_directories); + InstanceGroupParams group_params, GroupDirectories group_directories, + SharedFD* out_in_progress_lock = nullptr); Result UpdateInstanceGroup(const LocalInstanceGroup& group); Result RemoveInstanceGroup(LocalInstanceGroup group); + // Verifies that all instance ID lock files for `group` are not owned by + // another user and marks them `InUseState::kInUse`. + Result EnsureInstanceLocksInUse(const LocalInstanceGroup& group); + // Reconciles `group`'s persisted states against live launcher monitor sockets + // and `run_cvd` processes. Set `caller_holds_in_progress_lock` to true when + // the caller already holds `AcquireGroupInProgressLock(group)`. + Result RefreshInstanceGroupState( + LocalInstanceGroup group, bool caller_holds_in_progress_lock = false); + Result RefreshInstanceGroups(); + // Refreshes all groups and removes inactive groups whose backing directories + // (`HomeDir`, `HostArtifactsPath`, or `ProductOutPath`) no longer exist. + Result PruneBrokenInstanceGroups(); // Stops and removes all known instance instance groups Result Clear(); @@ -91,6 +119,7 @@ class InstanceManager { struct InternalInstanceDesc { InstanceLockFile lock_file; std::optional name; + bool release_on_error = false; }; Result> AllocateAndLockInstanceIds( @@ -98,6 +127,7 @@ class InstanceManager { InstanceLockFileManager& lock_manager_; InstanceDatabase& instance_db_; + std::string base_root_dir_; }; } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.cpp index 359be9c31da..4b388854b8b 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -115,6 +116,11 @@ bool LocalInstance::IsActive() const { return false; } +bool LocalInstance::CanConnectToLauncherMonitor( + std::chrono::seconds timeout) const { + return GetLauncherMonitor(timeout).has_value(); +} + Result LocalInstance::FetchStatus(std::chrono::seconds timeout) { return CF_EXPECT(FetchInstanceStatus(*this, timeout)); } @@ -278,13 +284,18 @@ Result LocalInstance::GetLauncherMonitor( // under /tmp, and store this path in the config. Older instances just put // them in the instance directory. std::string uds_dir = InstanceDirectory(); - Json::Value config = CF_EXPECT(ReadJsonConfig()); - if (config.isMember("instances_uds_dir") && - config["instances_uds_dir"].isString()) { - uds_dir = - fmt::format("{}/cvd-{}", config["instances_uds_dir"].asString(), Id()); + if (Result config = ReadJsonConfig(); + config.has_value() && config->isMember("instances_uds_dir") && + (*config)["instances_uds_dir"].isString()) { + uds_dir = fmt::format("{}/cvd-{}", + (*config)["instances_uds_dir"].asString(), Id()); } std::string monitor_path = uds_dir + "/launcher_monitor.sock"; + CF_EXPECTF(FileExists(monitor_path), + "Launcher monitor socket ({}) does not exist", monitor_path); + CF_EXPECTF(monitor_path.size() < sizeof(sockaddr_un::sun_path), + "Launcher monitor socket path ({}) exceeds AF_UNIX limit", + monitor_path); SharedFD monitor = SharedFD::SocketLocalClient(monitor_path, false, SOCK_STREAM, timeout.count()); CF_EXPECTF(monitor->IsOpen(), diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.h b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.h index da97d991761..d5048f6a8b8 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.h @@ -60,6 +60,8 @@ class LocalInstance { std::string AssemblyDirectory() const; bool IsActive() const; + bool CanConnectToLauncherMonitor( + std::chrono::seconds timeout = std::chrono::seconds(1)) const; // Contacts run_cvd to query the instance status. Returns a JSON object with // a description of the instance properties. Waits for run_cvd to respond for // at most timeout seconds. diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.cpp index 9a8e17039e4..b40efc8526c 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.cpp @@ -17,6 +17,8 @@ #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" #include +#include +#include #include #include #include @@ -47,9 +49,17 @@ std::vector Filter( return ret; } -std::string DefaultBaseDir() { - auto time = std::chrono::system_clock::now().time_since_epoch().count(); - return fmt::format("{}/{}", PerUserDir(), time); +std::string DefaultBaseDir(const std::string& base_root_dir) { + static std::atomic last_time{0}; + const int64_t now = + std::chrono::system_clock::now().time_since_epoch().count(); + int64_t prev = last_time.load(std::memory_order_relaxed); + int64_t chosen = std::max(now, prev + 1); + while (!last_time.compare_exchange_weak(prev, chosen, + std::memory_order_relaxed)) { + chosen = std::max(now, prev + 1); + } + return fmt::format("{}/{}", base_root_dir, chosen); } std::string HomeDirFromBase(const std::string& base_dir) { @@ -94,7 +104,11 @@ Result LocalInstanceGroup::Create( } LocalInstanceGroup::Builder::Builder(std::string group_name) - : base_dir_(DefaultBaseDir()) { + : Builder(std::move(group_name), PerUserDir()) {} + +LocalInstanceGroup::Builder::Builder(std::string group_name, + const std::string& base_root_dir) + : base_dir_(DefaultBaseDir(base_root_dir)) { group_proto_.set_name(std::move(group_name)); group_proto_.set_home_directory(HomeDirFromBase(base_dir_)); group_proto_.set_host_artifacts_path(HostArtifactsDirFromBase(base_dir_)); diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.h b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.h index b18d05218cb..f30d7aa47dd 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.h @@ -33,7 +33,8 @@ class LocalInstanceGroup { public: class Builder { public: - Builder(std::string group_name); + explicit Builder(std::string group_name); + Builder(std::string group_name, const std::string& base_root_dir); Builder& AddInstance(unsigned id) &; Builder& AddInstance(unsigned id, std::string name) &; diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group_test.cpp index aba929d5239..6e021957351 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group_test.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group_test.cpp @@ -82,5 +82,35 @@ TEST_F(LocalInstanceGroupUnitTest, SearchById) { } } +TEST_F(LocalInstanceGroupUnitTest, + BuilderProducesDistinctPureDigitBaseDirBasenamesForPodcvd) { + Result g1 = + LocalInstanceGroup::Builder("grp1", "/var/tmp/cvd/0") + .AddInstance(1) + .Build(); + Result g2 = + LocalInstanceGroup::Builder("grp2", "/var/tmp/cvd/0") + .AddInstance(2) + .Build(); + ASSERT_THAT(g1, IsOk()); + ASSERT_THAT(g2, IsOk()); + EXPECT_NE(g1->BaseDir(), g2->BaseDir()); + + // `podcvd` translates `/var/tmp/cvd//` via + // `/var/tmp/cvd/[0-9]+/[0-9]+`, so the basename must consist strictly of + // ASCII digits with no underscores or non-numeric suffixes. + for (const LocalInstanceGroup& grp : {*g1, *g2}) { + const std::string base = grp.BaseDir(); + const std::string prefix = "/var/tmp/cvd/0/"; + ASSERT_EQ(base.rfind(prefix, 0), 0u); + const std::string leaf = base.substr(prefix.size()); + ASSERT_FALSE(leaf.empty()); + for (char ch : leaf) { + EXPECT_GE(ch, '0'); + EXPECT_LE(ch, '9'); + } + } +} + } // namespace selector } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_test.cpp index 42fdfa5976f..1b9501a2e83 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_test.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_test.cpp @@ -13,12 +13,83 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "cuttlefish/host/commands/cvd/instances/local_instance.h" + +#include +#include + +#include +#include + +#include "absl/strings/str_cat.h" +#include "gmock/gmock-matchers.h" #include "gtest/gtest.h" +#include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/files/directory_exists.h" +#include "cuttlefish/files/recursively_remove_directory.h" +#include "cuttlefish/files/write_new_file.h" +#include "cuttlefish/host/commands/cvd/instances/cvd_persistent_data.pb.h" +#include "cuttlefish/host/commands/cvd/instances/data_viewer.h" +#include "cuttlefish/host/commands/cvd/instances/instance_database.h" #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" +#include "cuttlefish/metrics/device_metrics_orchestration.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" namespace cuttlefish { namespace selector { +namespace { + +class ScopedShortTempDir { + public: + ScopedShortTempDir() { + std::string tmpl = "/tmp/cf_inst_test.XXXXXX"; + char* created = mkdtemp(tmpl.data()); + if (created != nullptr) { + root_ = created; + setenv("CUTTLEFISH_METRICS_TRANSMITTER_PATH", + absl::StrCat(root_, "/no_metrics_transmitter").c_str(), 1); + setenv("XDG_DATA_HOME", absl::StrCat(root_, "/xdg_data").c_str(), 1); + } + } + + ~ScopedShortTempDir() { + if (!root_.empty()) { + (void)RecursivelyRemoveDirectory(root_); + } + } + + const std::string& Root() const { return root_; } + + private: + std::string root_; +}; + +Result BuildGroupWithHome(const std::string& db_path, + const std::string& name, + const std::string& home_dir, + unsigned id) { + DataViewer viewer(db_path); + CF_EXPECT(viewer.WithExclusiveLock( + [&](cvd::PersistentData& data) -> Result { + cvd::InstanceGroup* grp = data.add_instance_groups(); + grp->set_name(name); + grp->set_home_directory(home_dir); + grp->set_host_artifacts_path(home_dir); + grp->set_product_out_path(home_dir); + cvd::Instance* inst = grp->add_instances(); + inst->set_id(id); + inst->set_name(absl::StrCat(id)); + inst->set_state(cvd::INSTANCE_STATE_RUNNING); + return {}; + })); + InstanceDatabase db(db_path); + return CF_EXPECT(db.FindGroup({.group_name = name})); +} + +} // namespace /** * Note that invalid inputs must be tested at the InstanceDatabase level @@ -27,15 +98,6 @@ TEST(LocalinstanceTest, Fields) { auto parent_group_res = LocalInstanceGroup::Builder("super").AddInstance(3, "phone").Build(); if (!parent_group_res.has_value()) { - /* - * Here's why we skip the test rather than see it as a failure. - * - * 1. The test is specifically designed for operations in - * LocalInstanceRecord. - * 2. Adding instance to a group is tested in another test suites designed - * for LocalInstanceGroup. It's a failure there but not here. - * - */ GTEST_SKIP() << "Failed to add instance group. Set up failed."; } const auto& parent_group = *parent_group_res; @@ -47,5 +109,73 @@ TEST(LocalinstanceTest, Fields) { EXPECT_EQ(parent_group.Proto().name(), "super"); } +TEST(LocalinstanceTest, CanConnectToLauncherMonitorHandlesMissingAndLongPaths) { + ScopedShortTempDir tmp; + ASSERT_FALSE(tmp.Root().empty()); + + std::string long_home = absl::StrCat( + tmp.Root(), + "/very_long_home_directory_exceeding_unix_domain_socket_sun_path_" + "limit_0123456789abcdef"); + std::string fallback_inst_dir = + absl::StrCat(long_home, "/cuttlefish/instances/cvd-7"); + ASSERT_THAT(EnsureDirectoryExists(fallback_inst_dir), IsOk()); + std::string long_sock = + absl::StrCat(fallback_inst_dir, "/launcher_monitor.sock"); + ASSERT_GE(long_sock.size(), sizeof(sockaddr_un::sun_path)); + ASSERT_THAT(WriteNewFile(long_sock, ""), IsOk()); + + Result group = BuildGroupWithHome( + absl::StrCat(tmp.Root(), "/db.binpb"), "grp_long", long_home, 7); + ASSERT_THAT(group, IsOk()); + EXPECT_FALSE(group->Instances()[0].CanConnectToLauncherMonitor( + std::chrono::seconds(1))); + + // Now point `.cuttlefish_config.json` at a short `instances_uds_dir` with a + // live listening AF_UNIX socket and verify `CanConnectToLauncherMonitor` + // succeeds. + std::string uds_root = absl::StrCat(tmp.Root(), "/uds"); + std::string cvd_uds = absl::StrCat(uds_root, "/cvd-7"); + ASSERT_THAT(EnsureDirectoryExists(cvd_uds), IsOk()); + ASSERT_THAT( + WriteNewFile(absl::StrCat(long_home, "/.cuttlefish_config.json"), + absl::StrCat("{\"instances_uds_dir\":\"", uds_root, "\"}")), + IsOk()); + std::string live_sock_path = absl::StrCat(cvd_uds, "/launcher_monitor.sock"); + Result server_sock = Fd::SocketLocalServer( + live_sock_path, /*is_abstract=*/false, SOCK_STREAM, 0600); + ASSERT_THAT(server_sock, IsOk()); + + EXPECT_TRUE(group->Instances()[0].CanConnectToLauncherMonitor( + std::chrono::seconds(1))); +} + +TEST(LocalinstanceTest, MetricsSetupRecreatesWipedDirectoryWithoutAborting) { + ScopedShortTempDir tmp; + ASSERT_FALSE(tmp.Root().empty()); + + std::string home = absl::StrCat(tmp.Root(), "/slot_home"); + ASSERT_THAT(EnsureDirectoryExists(home), IsOk()); + Result group = BuildGroupWithHome( + absl::StrCat(tmp.Root(), "/metrics_db.binpb"), "grp_metrics", home, 1); + ASSERT_THAT(group, IsOk()); + + // Even when `AreMetricsEnabled()` is false, `GatherVmInstantiationMetrics` + // creates `/metrics`. + GatherVmInstantiationMetrics(*group); + EXPECT_TRUE(DirectoryExists(group->MetricsDir())); + + // Simulate a caller wiping `$HOME/*` between runs: `GatherVmStartMetrics` + // recreates `/metrics` without crashing, and `GatherVmStopMetrics` + // does not abort if `/metrics` was deleted mid-run. + ASSERT_THAT(RecursivelyRemoveDirectory(group->MetricsDir()), IsOk()); + EXPECT_FALSE(DirectoryExists(group->MetricsDir())); + GatherVmStartMetrics(*group); + EXPECT_TRUE(DirectoryExists(group->MetricsDir())); + + ASSERT_THAT(RecursivelyRemoveDirectory(group->MetricsDir()), IsOk()); + GatherVmStopMetrics(*group); +} + } // namespace selector } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel index 875163b687e..7eabe3615a3 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel @@ -1,5 +1,5 @@ load("//:build_variables.bzl", "COPTS") -load("//cuttlefish/bazel:rules.bzl", "cf_cc_library") +load("//cuttlefish/bazel:rules.bzl", "cf_cc_library", "cf_cc_test") package( default_visibility = ["//:android_cuttlefish"], @@ -33,3 +33,16 @@ cf_cc_library( "@android_system_libbase//:file_logging", ], ) + +cf_cc_test( + name = "instance_lock_test", + srcs = ["instance_lock_test.cpp"], + deps = [ + ":lock", + "//cuttlefish/files:file_exists", + "//cuttlefish/files:recursively_remove_directory", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "@abseil-cpp//absl/strings", + ], +) diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp index 773e25c6a0d..8a0dce2e3cd 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp @@ -17,6 +17,7 @@ #include "cuttlefish/host/commands/cvd/instances/lock/instance_lock.h" #include +#include #include #include @@ -36,6 +37,10 @@ InstanceLockFile::InstanceLockFile(LockFile&& lock_file, unsigned InstanceLockFile::Instance() const { return instance_num_; } +const std::string& InstanceLockFile::LockFilePath() const { + return lock_file_.LockFilePath(); +} + Result InstanceLockFile::Status() const { auto in_use_state = CF_EXPECT(lock_file_.Status()); return in_use_state; @@ -46,6 +51,10 @@ Result InstanceLockFile::Status(InUseState state) { return {}; } +Result InstanceLockFile::OwnerUid() const { + return CF_EXPECT(lock_file_.OwnerUid()); +} + InstanceLockFileManager::InstanceLockFileManager( std::string instance_locks_path) : instance_locks_path_(std::move(instance_locks_path)) {}; @@ -59,18 +68,97 @@ Result InstanceLockFileManager::LockFilePath( return path.str(); } +Result> +InstanceLockFileManager::AcquireOwnedLockIfPresent(unsigned instance_num) { + const auto lock_file_path = CF_EXPECT(LockFilePath(instance_num)); + Result path_owner = FileOwner(lock_file_path); + if (!path_owner.has_value() || *path_owner != geteuid()) { + return std::nullopt; + } + InstanceLockFile lock = CF_EXPECT(AcquireLock(instance_num)); + uid_t fd_owner = CF_EXPECT(lock.OwnerUid()); + if (fd_owner != geteuid()) { + return std::nullopt; + } + return lock; +} + +Result InstanceLockFileManager::ReleaseHeldLock(InstanceLockFile& lock) { + uid_t fd_owner = CF_EXPECT(lock.OwnerUid()); + if (fd_owner != geteuid()) { + return {}; + } + CF_EXPECT(lock.Status(InUseState::kNotInUse)); + CF_EXPECT(RemoveFile(lock.LockFilePath()), StrError()); + return {}; +} + +Result InstanceLockFileManager::ReleaseInstanceLock( + unsigned instance_num) { + std::optional lock = + CF_EXPECT(AcquireOwnedLockIfPresent(instance_num)); + if (!lock.has_value()) { + return {}; + } + CF_EXPECT(ReleaseHeldLock(*lock)); + return {}; +} + Result InstanceLockFileManager::RemoveLockFile(unsigned instance_num) { const auto lock_file_path = CF_EXPECT(LockFilePath(instance_num)); CF_EXPECT(RemoveFile(lock_file_path), StrError()); return {}; } +Result InstanceLockFileManager::AcquireCurrentUserLock( + unsigned instance_num) { + constexpr int kMaxAttempts = 16; + for (int attempt = 0; attempt < kMaxAttempts; ++attempt) { + InstanceLockFile lock = CF_EXPECT(AcquireLock(instance_num)); + InUseState status = CF_EXPECT(lock.Status()); + uid_t owner = CF_EXPECT(lock.OwnerUid()); + if (owner == geteuid()) { + return lock; + } + if (status == InUseState::kInUse) { + return CF_ERRF( + "Instance ID {} is currently in use by another user (uid {})", + instance_num, owner); + } + // Stale `kNotInUse` file owned by another UID: unlink while holding + // `LOCK_EX` so the next iteration creates a fresh inode owned by geteuid(). + CF_EXPECTF(RemoveFile(lock.LockFilePath()), + "Failed to replace stale lock file '{}': {}", + lock.LockFilePath(), StrError()); + } + return CF_ERRF("Failed to acquire instance lock {} for current user", + instance_num); +} + Result InstanceLockFileManager::AcquireUnusedLock() { for (unsigned i = 1;; i++) { auto lock = CF_EXPECT(TryAcquireLock(i)); - if (lock && CF_EXPECT(lock->Status()) == InUseState::kNotInUse) { + if (!lock) { + continue; + } + if (CF_EXPECT(lock->Status()) != InUseState::kNotInUse) { + continue; + } + if (CF_EXPECT(lock->OwnerUid()) == geteuid()) { return *lock; } + // Replace stale `kNotInUse` inode owned by another UID so `st_uid` matches + // `geteuid()`. If unlinking fails (e.g. sticky custom dir), skip to i + 1. + if (!RemoveFile(lock->LockFilePath()).has_value()) { + continue; + } + lock.reset(); + auto fresh_lock = CF_EXPECT(TryAcquireLock(i)); + if (fresh_lock && + CF_EXPECT(fresh_lock->Status()) == InUseState::kNotInUse && + CF_EXPECT(fresh_lock->OwnerUid()) == geteuid()) { + return *fresh_lock; + } } } diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h index 721dd188050..05ab4b263cf 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h @@ -15,6 +15,8 @@ */ #pragma once +#include + #include #include @@ -28,8 +30,10 @@ class InstanceLockFile { public: unsigned Instance() const; + const std::string& LockFilePath() const; Result Status() const; Result Status(InUseState); + Result OwnerUid() const; private: InstanceLockFile(LockFile&& lock_file, unsigned instance_num); @@ -42,7 +46,24 @@ class InstanceLockFileManager { InstanceLockFileManager(std::string instance_locks_path); Result AcquireLock(unsigned instance_num); + // Acquires `instance_num` for the current user (`geteuid()`): fails if + // marked `kInUse` by another UID, and replaces any stale `kNotInUse` inode + // owned by another UID with a fresh inode owned by `geteuid()`. + Result AcquireCurrentUserLock(unsigned instance_num); Result AcquireUnusedLock(); + Result> TryAcquireLock(unsigned instance_num); + + // Acquires `instance_num` if its lock file exists and is owned by `geteuid()` + // so the caller can release it while holding the lock. + Result> AcquireOwnedLockIfPresent( + unsigned instance_num); + + // Resets and removes an acquired lock file while holding its exclusive flock. + Result ReleaseHeldLock(InstanceLockFile& lock); + + // Releases `instance_num` if its lock file exists and is owned by + // `geteuid()`. + Result ReleaseInstanceLock(unsigned instance_num); // TODO: This routine should be removed and replaced with allocd // The caller must check if the instance_num belongs to the user, before @@ -51,7 +72,6 @@ class InstanceLockFileManager { private: Result LockFilePath(unsigned instance_num); - Result> TryAcquireLock(unsigned instance_num); std::string instance_locks_path_; LockFileManager lock_file_manager_; diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock_test.cpp new file mode 100644 index 00000000000..0d18e9a30f5 --- /dev/null +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock_test.cpp @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/commands/cvd/instances/lock/instance_lock.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "gmock/gmock-matchers.h" +#include "gtest/gtest.h" + +#include "cuttlefish/files/file_exists.h" +#include "cuttlefish/files/recursively_remove_directory.h" +#include "cuttlefish/host/commands/cvd/instances/lock/lock_file.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +class TempLocksDir { + public: + TempLocksDir() { + const char* tmpdir = getenv("TEST_TMPDIR"); + std::string base = + (tmpdir != nullptr && tmpdir[0] != '\0') ? tmpdir : "/tmp"; + std::string tmpl = absl::StrCat(base, "/cf_lock_test.XXXXXX"); + char* created = mkdtemp(tmpl.data()); + if (created != nullptr) { + dir_ = absl::StrCat(created, "/"); + } + } + + ~TempLocksDir() { + if (!dir_.empty()) { + (void)RecursivelyRemoveDirectory(dir_); + } + } + + const std::string& Dir() const { return dir_; } + + private: + std::string dir_; +}; + +TEST(InstanceLockTest, AcquireCurrentUserAndReleaseHeldLock) { + TempLocksDir tmp; + ASSERT_FALSE(tmp.Dir().empty()); + InstanceLockFileManager mgr(tmp.Dir()); + + Result lock = mgr.AcquireCurrentUserLock(42); + ASSERT_THAT(lock, IsOk()); + EXPECT_EQ(lock->Instance(), 42u); + ASSERT_THAT(lock->OwnerUid(), IsOkAndValue(geteuid())); + ASSERT_THAT(lock->Status(), IsOkAndValue(InUseState::kNotInUse)); + ASSERT_THAT(lock->Status(InUseState::kInUse), IsOk()); + + // Non-blocking `TryAcquireLock` from another manager fails while held. + InstanceLockFileManager other_mgr(tmp.Dir()); + Result> contended = + other_mgr.TryAcquireLock(42); + ASSERT_THAT(contended, IsOk()); + EXPECT_FALSE(contended->has_value()); + + std::string lock_path = lock->LockFilePath(); + EXPECT_TRUE(FileExists(lock_path)); + ASSERT_THAT(mgr.ReleaseHeldLock(*lock), IsOk()); + EXPECT_FALSE(FileExists(lock_path)); +} + +TEST(InstanceLockTest, ConcurrentAcquireAndReleaseNeverSplitBrain) { + TempLocksDir tmp; + ASSERT_FALSE(tmp.Dir().empty()); + + constexpr int kNumThreads = 8; + constexpr int kIterations = 40; + std::atomic active_holders{0}; + std::atomic overlap_detected{false}; + + std::vector threads; + threads.reserve(kNumThreads); + for (int t = 0; t < kNumThreads; ++t) { + threads.emplace_back([&]() { + InstanceLockFileManager mgr(tmp.Dir()); + for (int i = 0; i < kIterations; ++i) { + Result lock = mgr.AcquireCurrentUserLock(7); + ASSERT_THAT(lock, IsOk()); + int prev = active_holders.fetch_add(1, std::memory_order_seq_cst); + if (prev != 0) { + overlap_detected.store(true, std::memory_order_seq_cst); + } + std::this_thread::yield(); + active_holders.fetch_sub(1, std::memory_order_seq_cst); + if ((i % 2) == 0) { + EXPECT_THAT(mgr.ReleaseHeldLock(*lock), IsOk()); + } + } + }); + } + for (std::thread& th : threads) { + th.join(); + } + + EXPECT_FALSE(overlap_detected.load()); +} + +TEST(InstanceLockTest, AcquireUnusedLockSkipsInUseIds) { + TempLocksDir tmp; + ASSERT_FALSE(tmp.Dir().empty()); + InstanceLockFileManager mgr(tmp.Dir()); + + { + // Drop flock at scope exit while leaving `1` marked `kInUse`. + Result lock1 = mgr.AcquireCurrentUserLock(1); + ASSERT_THAT(lock1, IsOk()); + ASSERT_THAT(lock1->Status(InUseState::kInUse), IsOk()); + } + + Result unused = mgr.AcquireUnusedLock(); + ASSERT_THAT(unused, IsOk()); + EXPECT_EQ(unused->Instance(), 2u); + ASSERT_THAT(mgr.ReleaseInstanceLock(1), IsOk()); + EXPECT_FALSE(FileExists(absl::StrCat(tmp.Dir(), "local-instance-1.lock"))); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp index 4066f6425f4..32bd1d68cbe 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp @@ -16,7 +16,11 @@ #include "cuttlefish/host/commands/cvd/instances/lock/lock_file.h" +#include +#include #include +#include +#include #include #include @@ -87,6 +91,23 @@ Result LockFile::Status(InUseState state) { return {}; } +Result LockFile::OwnerUid() const { + return CF_EXPECT(fd_->Fstat()).st_uid; +} + +static bool IsSameInode(const SharedFD& fd, const std::string& file_path) { + Result fd_stat = fd->Fstat(); + if (!fd_stat.has_value()) { + return false; + } + struct stat path_stat{}; + if (stat(file_path.c_str(), &path_stat) != 0) { + return false; + } + return fd_stat->st_dev == path_stat.st_dev && + fd_stat->st_ino == path_stat.st_ino; +} + bool LockFile::operator<(const LockFile& other) const { if (this == std::addressof(other)) { return false; @@ -103,35 +124,75 @@ bool LockFile::operator<(const LockFile& other) const { Result LockFileManager::OpenLockFile(const std::string& file_path) { auto parent_dir = android::base::Dirname(file_path); CF_EXPECT(EnsureDirectoryExists(parent_dir, 0777)); - int result = chmod(parent_dir.c_str(), 0777); - if (result) { - VLOG(0) << "failed: chmod 777 " << parent_dir; + if (Result dir_owner = FileOwner(parent_dir); + dir_owner.has_value() && *dir_owner == geteuid()) { + int result = chmod(parent_dir.c_str(), 0777); + if (result != 0) { + VLOG(0) << "failed: chmod 777 " << parent_dir; + } } - Fd fd = CF_EXPECT(Fd::Open(file_path.data(), O_CREAT | O_RDWR, 0666)); - result = chmod(file_path.c_str(), 0666); - if (result) { - VLOG(0) << "failed: chmod 666 " << file_path; + + // Open existing lock files WITHOUT `O_CREAT` first so Linux + // `fs.protected_regular=2` (`may_create_in_sticky`) does not reject opening a + // `0666` file owned by another UID inside the `0777` shared lock directory. + constexpr int kMaxOpenAttempts = 16; + for (int attempt = 0; attempt < kMaxOpenAttempts; ++attempt) { + Result existing_fd = + Fd::Open(file_path.data(), O_RDWR | O_CLOEXEC); + if (existing_fd.has_value()) { + return std::move(*existing_fd); + } + if (*existing_fd.error() != ENOENT) { + return CF_EXPECT(std::move(existing_fd)); + } + + Result created_fd = + Fd::Open(file_path.data(), O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0666); + if (created_fd.has_value()) { + if (!created_fd->Chmod(0666)) { + VLOG(0) << "failed: fchmod 666 " << file_path; + } + return std::move(*created_fd); + } + if (*created_fd.error() == EEXIST) { + continue; + } + return CF_EXPECT(std::move(created_fd)); } - return fd; + return CF_ERRF("Failed to open or create lock file '{}' after {} attempts", + file_path, kMaxOpenAttempts); } Result LockFileManager::AcquireLock( const std::string& lock_file_path) { - SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); - CF_EXPECT(fd->Flock(LOCK_EX)); - return LockFile(fd, lock_file_path); + while (true) { + SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); + CF_EXPECT(fd->Flock(LOCK_EX)); + if (IsSameInode(fd, lock_file_path)) { + return LockFile(fd, lock_file_path); + } + (void)fd->Flock(LOCK_UN | LOCK_NB); + } } Result> LockFileManager::TryAcquireLock( const std::string& lock_file_path) { - SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); - Result flock_result = fd->Flock(LOCK_EX | LOCK_NB); - if (flock_result) { - return std::optional(LockFile(fd, lock_file_path)); - } else if (!flock_result && *flock_result.error() == EWOULDBLOCK) { + for (int attempt = 0; attempt < 8; ++attempt) { + SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); + Result flock_result = fd->Flock(LOCK_EX | LOCK_NB); + if (flock_result) { + if (IsSameInode(fd, lock_file_path)) { + return std::optional(LockFile(fd, lock_file_path)); + } + (void)fd->Flock(LOCK_UN | LOCK_NB); + continue; + } else if (*flock_result.error() == EWOULDBLOCK || + *flock_result.error() == EAGAIN) { + return {}; + } + CF_EXPECT(std::move(flock_result)); return {}; } - CF_EXPECT(std::move(flock_result)); return {}; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h index c7af9e58853..ef34394f99e 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h @@ -15,6 +15,8 @@ */ #pragma once +#include + #include #include #include @@ -66,6 +68,7 @@ class LockFile { const auto& LockFilePath() const { return lock_file_path_; } Result Status() const; Result Status(InUseState); + Result OwnerUid() const; // to put this into a set bool operator<(const LockFile& other) const; diff --git a/base/cvd/cuttlefish/metrics/BUILD.bazel b/base/cvd/cuttlefish/metrics/BUILD.bazel index 048269d03a5..2c6b45cc0de 100644 --- a/base/cvd/cuttlefish/metrics/BUILD.bazel +++ b/base/cvd/cuttlefish/metrics/BUILD.bazel @@ -19,6 +19,7 @@ cf_cc_library( "//cuttlefish/metrics/guest:device_event_type", "//cuttlefish/metrics/guest:guest_metrics", "//cuttlefish/metrics/guest:parsed_flags", + "//cuttlefish/metrics/io:enabled", "//cuttlefish/metrics/io:notification", "//cuttlefish/result", "@abseil-cpp//absl/log", diff --git a/base/cvd/cuttlefish/metrics/device_metrics_orchestration.cc b/base/cvd/cuttlefish/metrics/device_metrics_orchestration.cc index 22890a77b3f..5c81ab28907 100644 --- a/base/cvd/cuttlefish/metrics/device_metrics_orchestration.cc +++ b/base/cvd/cuttlefish/metrics/device_metrics_orchestration.cc @@ -31,6 +31,7 @@ #include "cuttlefish/metrics/guest/device_event_type.h" #include "cuttlefish/metrics/guest/guest_metrics.h" #include "cuttlefish/metrics/guest/parsed_flags.h" +#include "cuttlefish/metrics/io/enabled.h" #include "cuttlefish/metrics/io/notification.h" #include "cuttlefish/metrics/metrics_orchestration.h" #include "cuttlefish/result/result.h" @@ -94,12 +95,15 @@ Result GatherDeviceMetrics( } Result RunMetrics(const DeviceMetricsInput& metrics_input) { - ScopedLogger logger = - CreateLogger(metrics_input.base_input.metrics_directory); CF_EXPECTF(FileExists(metrics_input.base_input.metrics_directory), "Metrics directory({}) does not exist. Perhaps metrics were not " "initialized?", metrics_input.base_input.metrics_directory); + ScopedLogger logger = + CreateLogger(metrics_input.base_input.metrics_directory); + if (!AreMetricsEnabled()) { + return {}; + } const std::string event_type_label = DeviceEventTypeString(metrics_input.guests.event_type); @@ -117,6 +121,15 @@ Result RunMetrics(const DeviceMetricsInput& metrics_input) { void GatherVmInstantiationMetrics(const LocalInstanceGroup& instance_group) { DisplayPrivacyNotice(); + Result metrics_setup_result = SetUpMetrics(instance_group.MetricsDir()); + if (!metrics_setup_result.has_value()) { + VLOG(0) << fmt::format("Failed to initialize metrics. Error: {}", + metrics_setup_result.error().FormatForEnv()); + return; + } + if (!AreMetricsEnabled()) { + return; + } Result metrics_input_result = GetDeviceMetricsInput( instance_group, DeviceEventType::DeviceInstantiation); if (!metrics_input_result.has_value()) { @@ -124,13 +137,6 @@ void GatherVmInstantiationMetrics(const LocalInstanceGroup& instance_group) { metrics_input_result.error().FormatForEnv()); return; } - Result metrics_setup_result = - SetUpMetrics(metrics_input_result->base_input.metrics_directory); - if (!metrics_setup_result.has_value()) { - VLOG(0) << fmt::format("Failed to initialize metrics. Error: {}", - metrics_setup_result.error().FormatForEnv()); - return; - } Result run_metrics_result = RunMetrics(*metrics_input_result); if (!run_metrics_result.has_value()) { VLOG(0) << fmt::format( @@ -141,6 +147,15 @@ void GatherVmInstantiationMetrics(const LocalInstanceGroup& instance_group) { } void GatherVmStartMetrics(const LocalInstanceGroup& instance_group) { + Result metrics_setup_result = SetUpMetrics(instance_group.MetricsDir()); + if (!metrics_setup_result.has_value()) { + VLOG(0) << fmt::format("Failed to initialize metrics. Error: {}", + metrics_setup_result.error().FormatForEnv()); + return; + } + if (!AreMetricsEnabled()) { + return; + } Result metrics_input_result = GetDeviceMetricsInput(instance_group, DeviceEventType::DeviceBootStart); if (!metrics_input_result.has_value()) { @@ -158,6 +173,9 @@ void GatherVmStartMetrics(const LocalInstanceGroup& instance_group) { } void GatherVmBootCompleteMetrics(const LocalInstanceGroup& instance_group) { + if (!AreMetricsEnabled()) { + return; + } Result metrics_input_result = GetDeviceMetricsInput( instance_group, DeviceEventType::DeviceBootComplete); if (!metrics_input_result.has_value()) { @@ -175,6 +193,9 @@ void GatherVmBootCompleteMetrics(const LocalInstanceGroup& instance_group) { } void GatherVmBootFailedMetrics(const LocalInstanceGroup& instance_group) { + if (!AreMetricsEnabled()) { + return; + } Result metrics_input_result = GetDeviceMetricsInput(instance_group, DeviceEventType::DeviceBootFailed); if (!metrics_input_result.has_value()) { @@ -192,6 +213,9 @@ void GatherVmBootFailedMetrics(const LocalInstanceGroup& instance_group) { } void GatherVmStopMetrics(const LocalInstanceGroup& instance_group) { + if (!AreMetricsEnabled()) { + return; + } Result metrics_input_result = GetDeviceMetricsInput(instance_group, DeviceEventType::DeviceStop); if (!metrics_input_result.has_value()) {