diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/cli/BUILD.bazel index 389a8ab21be..72559c79323 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/BUILD.bazel @@ -119,8 +119,8 @@ cf_cc_library( "//cuttlefish/host/commands/cvd/cli/commands:load_configs.h", ], deps = [ + "//cuttlefish/common/libs/fs", "//cuttlefish/common/libs/utils:contains", - "//cuttlefish/common/libs/utils:environment", "//cuttlefish/common/libs/utils:files", "//cuttlefish/common/libs/utils:users", "//cuttlefish/files:copy", @@ -130,7 +130,6 @@ cf_cc_library( "//cuttlefish/flag_parser", "//cuttlefish/host/commands/cvd/cli:command_request", "//cuttlefish/host/commands/cvd/cli:help_format", - "//cuttlefish/host/commands/cvd/cli:utils", "//cuttlefish/host/commands/cvd/cli/commands:bugreport", "//cuttlefish/host/commands/cvd/cli/commands:cache", "//cuttlefish/host/commands/cvd/cli/commands:clear", @@ -159,7 +158,6 @@ cf_cc_library( "//cuttlefish/host/commands/cvd/cli/commands:version", "//cuttlefish/host/commands/cvd/cli/parser:load_config_cc_proto", "//cuttlefish/host/commands/cvd/cli/parser:load_configs_parser", - "//cuttlefish/host/commands/cvd/cli/selector", "//cuttlefish/host/commands/cvd/cli/selector:creation_analyzer", "//cuttlefish/host/commands/cvd/cli/selector:num_instances_parser", "//cuttlefish/host/commands/cvd/cli/selector:selector_common_parser", @@ -171,6 +169,7 @@ cf_cc_library( "//cuttlefish/host/commands/cvd/instances/lock", "//cuttlefish/host/commands/cvd/utils:common", "//cuttlefish/host/commands/cvd/utils:interrupt_listener", + "//cuttlefish/posix:readlink", "//cuttlefish/posix:realpath", "//cuttlefish/posix:remove", "//cuttlefish/posix:strerror", @@ -185,6 +184,34 @@ cf_cc_library( }), ) +cf_cc_test( + name = "create_test", + srcs = ["create_test.cpp"], + deps = [ + "//cuttlefish/common/libs/fs", + "//cuttlefish/common/libs/fs:fd", + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/common/libs/utils:users", + "//cuttlefish/files:directory_exists", + "//cuttlefish/files:file_exists", + "//cuttlefish/files:recursively_remove_directory", + "//cuttlefish/files:write_new_file", + "//cuttlefish/host/commands/cvd/cli:command_request", + "//cuttlefish/host/commands/cvd/cli:nesting_commands", + "//cuttlefish/host/commands/cvd/cli/commands:remove", + "//cuttlefish/host/commands/cvd/cli/commands:start", + "//cuttlefish/host/commands/cvd/cli/commands:stop", + "//cuttlefish/host/commands/cvd/cli/selector:selector_common_parser", + "//cuttlefish/host/commands/cvd/instances", + "//cuttlefish/host/commands/cvd/instances:cvd_persistent_data", + "//cuttlefish/host/commands/cvd/instances:instance_manager", + "//cuttlefish/host/commands/cvd/instances/lock", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "@abseil-cpp//absl/strings", + ], +) + cf_cc_library( name = "utils", srcs = ["utils.cpp"], diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/create.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/create.cpp index c8597e12b07..44688456137 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/create.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/create.cpp @@ -19,11 +19,9 @@ #include #include #include +#include #include -#include -#include -#include #include #include #include @@ -38,8 +36,8 @@ #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/common/libs/utils/contains.h" -#include "cuttlefish/common/libs/utils/environment.h" #include "cuttlefish/common/libs/utils/files.h" #include "cuttlefish/common/libs/utils/users.h" #include "cuttlefish/files/copy.h" @@ -56,14 +54,14 @@ #include "cuttlefish/host/commands/cvd/cli/help_format.h" #include "cuttlefish/host/commands/cvd/cli/selector/creation_analyzer.h" #include "cuttlefish/host/commands/cvd/cli/selector/num_instances_parser.h" -#include "cuttlefish/host/commands/cvd/cli/selector/selector.h" #include "cuttlefish/host/commands/cvd/cli/selector/selector_common_parser.h" #include "cuttlefish/host/commands/cvd/cli/selector/selector_constants.h" -#include "cuttlefish/host/commands/cvd/cli/utils.h" #include "cuttlefish/host/commands/cvd/instances/cvd_persistent_data.pb.h" #include "cuttlefish/host/commands/cvd/instances/instance_manager.h" +#include "cuttlefish/host/commands/cvd/instances/local_instance.h" #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" #include "cuttlefish/host/commands/cvd/utils/common.h" +#include "cuttlefish/posix/readlink.h" #include "cuttlefish/posix/realpath.h" #include "cuttlefish/posix/remove.h" #include "cuttlefish/posix/strerror.h" @@ -129,11 +127,12 @@ Result StartGroup( const LocalInstanceGroup& group, const std::vector& subcmd_args, const std::unordered_map& envs, - InstanceManager& instance_manager) { + InstanceManager& instance_manager, SharedFD in_progress_lock) { const CommandRequest start_cmd = CF_EXPECT(CreateStartCommand(group, subcmd_args, envs)); std::unique_ptr start_handler = - std::make_unique(instance_manager); + std::make_unique(instance_manager, + std::move(in_progress_lock)); CF_EXPECT(start_handler->Handle(start_cmd)); return {}; } @@ -173,9 +172,16 @@ Result EnsureSymlink(const std::string& target, const std::string link) { return {}; } -Result IsDefaultGroup(const CommandRequest& request) { - return StringFromEnv("HOME", "") == CF_EXPECT(SystemWideUserHome()) && - !request.Selectors().HasOptions(); +Result IsDefaultGroup( + const std::unordered_map& envs, + const CommandRequest& request, const LocalInstanceGroup& group) { + if (request.Selectors().HasOptions() || + !DirectoryExists(group.HomeDir(), /*follow_symlinks=*/false)) { + return false; + } + std::string sys_home = CF_EXPECT(SystemWideUserHome()); + auto home_it = envs.find("HOME"); + return home_it == envs.end() || home_it->second == sys_home; } // For backward compatibility, we add extra symlink in home dir @@ -263,31 +269,99 @@ bool IsDeviceRunning(const LocalInstanceGroup& group) { return std::ranges::any_of(group.Instances(), &LocalInstance::IsActive); } +bool HasCustomHome(const LocalInstanceGroup& group) { + return !DirectoryExists(group.HomeDir(), /*follow_symlinks=*/false); +} + +std::optional GroupCustomHome(const LocalInstanceGroup& group) { + if (!HasCustomHome(group)) { + return std::nullopt; + } + if (Result real_home = RealPath(group.HomeDir()); + real_home.has_value()) { + return std::move(*real_home); + } + if (Result link_target = ReadLink(group.HomeDir()); + link_target.has_value()) { + return absl::StartsWith(*link_target, "/") + ? AbsolutePath(*link_target) + : AbsolutePath( + fmt::format("{}/{}", group.BaseDir(), *link_target)); + } + return group.HomeDir(); +} + +bool InstancesMatch( + const LocalInstanceGroup& group, const size_t num_instances, + const std::vector& requested_ids, + const std::optional>& requested_names) { + if (group.Instances().size() != num_instances) { + return false; + } + for (size_t i = 0; i < requested_ids.size(); ++i) { + if (group.Instances()[i].Id() != requested_ids[i]) { + return false; + } + } + if (requested_names.has_value()) { + if (requested_names->size() != num_instances) { + return false; + } + for (size_t i = 0; i < num_instances; ++i) { + if (group.Instances()[i].Name() != (*requested_names)[i]) { + return false; + } + } + } + return true; +} + +bool GroupHasAllInstanceNames(const LocalInstanceGroup& group, + const std::vector& requested_names) { + for (const std::string& requested_name : requested_names) { + if (group.FindByInstanceName(requested_name).empty()) { + return false; + } + } + return true; +} + +bool GroupOverlapsRequestedIds(const LocalInstanceGroup& group, + const std::vector& requested_ids) { + for (const LocalInstance& instance : group.Instances()) { + if (Contains(requested_ids, instance.Id())) { + return true; + } + } + return false; +} + Result MatchPaths(const LocalInstanceGroup& group, const std::string& target_host, const std::vector& target_products) { - const std::string real_group_host = - CF_EXPECT(RealPath(group.HostArtifactsPath())); const std::string real_target_host = CF_EXPECT(RealPath(target_host)); - - if (real_group_host != real_target_host) { + Result real_group_host = RealPath(group.HostArtifactsPath()); + if (!real_group_host.has_value() || *real_group_host != real_target_host) { return false; } const std::vector group_products = absl::StrSplit(group.ProductOutPath(), ','); - - std::vector real_group_products; - for (const auto& path : group_products) { - real_group_products.push_back(CF_EXPECT(RealPath(path))); + if (group_products.size() != target_products.size()) { + return false; } - std::vector real_target_products; - for (const auto& path : target_products) { - real_target_products.push_back(CF_EXPECT(RealPath(path))); + for (size_t i = 0; i < target_products.size(); ++i) { + const std::string real_target_product = + CF_EXPECT(RealPath(target_products[i])); + Result real_group_product = RealPath(group_products[i]); + if (!real_group_product.has_value() || + *real_group_product != real_target_product) { + return false; + } } - return real_group_products == real_target_products; + return true; } } // namespace @@ -299,33 +373,121 @@ CvdCreateCommandHandler::CvdCreateCommandHandler( Result> CvdCreateCommandHandler::FindReusableGroup( const selector::SelectorOptions& selectors, - const std::unordered_map& envs) { - InstanceDatabase::Filter filter = - selector::BuildFilterFromSelectors(selectors); + const GroupCreationInfo& creation_info) { + std::optional target_custom_home; + if (creation_info.group_directories.home.has_value()) { + target_custom_home = + CF_EXPECT(RealPath(*creation_info.group_directories.home)); + } + const std::string& target_host = + *creation_info.group_directories.host_artifacts_path; + std::vector target_products; + target_products.reserve( + creation_info.group_directories.product_out_paths.size()); + for (const std::optional& product_path : + creation_info.group_directories.product_out_paths) { + target_products.push_back(*product_path); + } + const size_t num_instances = num_instances_parser_.NumInstances(); + const std::vector requested_ids = + num_instances_parser_.InstanceIds(); + + if (selectors.group_name.has_value()) { + const std::vector named_groups = CF_EXPECT( + instance_manager_.FindGroups({.group_name = *selectors.group_name})); + if (named_groups.empty()) { + return std::nullopt; + } + CF_EXPECT_EQ(named_groups.size(), 1u, + "Unclear which group to reuse, try `cvd reset -y`"); + const LocalInstanceGroup& candidate = named_groups[0]; + CF_EXPECT(!IsDeviceRunning(candidate), + "Group is already running, try `cvd stop`"); + bool home_matches = (GroupCustomHome(candidate) == target_custom_home); + bool paths_match = + home_matches && + CF_EXPECT(MatchPaths(candidate, target_host, target_products)); + CF_EXPECT(paths_match == true, + "Trying to change the host paths on an existing group"); + CF_EXPECT(InstancesMatch(candidate, num_instances, requested_ids, + selectors.instance_names), + "Trying to change the instances on an existing group"); + return candidate; + } const std::vector groups = - CF_EXPECT(instance_manager_.FindGroups(filter)); + CF_EXPECT(instance_manager_.FindGroups({})); if (groups.empty()) { return std::nullopt; } - CF_EXPECT_EQ(groups.size(), 1, - "Unclear which group to reuse, try `cvd reset -y`"); - auto target_host_it = envs.find(kAndroidHostOut); - CF_EXPECT(target_host_it != envs.end()); + std::vector slot_groups; + for (const LocalInstanceGroup& group : groups) { + std::optional group_custom_home = GroupCustomHome(group); + bool overlaps_ids = GroupOverlapsRequestedIds(group, requested_ids); + bool belongs_to_slot = false; + if (target_custom_home.has_value()) { + belongs_to_slot = + (group_custom_home == target_custom_home) || overlaps_ids; + } else if (!requested_ids.empty()) { + belongs_to_slot = overlaps_ids; + } else { + belongs_to_slot = + !group_custom_home.has_value() && + (!selectors.instance_names.has_value() || + GroupHasAllInstanceNames(group, *selectors.instance_names)); + } + if (belongs_to_slot) { + slot_groups.push_back(group); + } + } + + if (slot_groups.empty()) { + return std::nullopt; + } + + if (!target_custom_home.has_value() && requested_ids.empty()) { + CF_EXPECT_EQ(slot_groups.size(), 1u, + "Unclear which group to reuse, try `cvd reset -y`"); + } + + for (const LocalInstanceGroup& group : slot_groups) { + CF_EXPECT(!IsDeviceRunning(group), + "Group is already running, try `cvd stop`"); + } - auto target_product_it = envs.find(kAndroidProductOut); - CF_EXPECT(target_product_it != envs.end()); + if (slot_groups.size() == 1) { + const LocalInstanceGroup& candidate = slot_groups[0]; + bool home_matches = (GroupCustomHome(candidate) == target_custom_home); + bool paths_match = + home_matches && + CF_EXPECT(MatchPaths(candidate, target_host, target_products)); + bool instances_match = InstancesMatch( + candidate, num_instances, requested_ids, selectors.instance_names); + if (paths_match && instances_match) { + return candidate; + } + if (!HasCustomHome(candidate)) { + CF_EXPECT(paths_match == true, + "Trying to change the host paths on an existing group"); + CF_EXPECT(instances_match == true, + "Trying to change the instances on an existing group"); + } + } - const std::vector target_products = ExpandProductPaths( - target_product_it->second, num_instances_parser_.NumInstances()); - bool match = - CF_EXPECT(MatchPaths(groups[0], target_host_it->second, target_products)); - CF_EXPECT(std::move(match), - "Trying to change the host paths on an existing group"); - CF_EXPECT(!IsDeviceRunning(groups[0]), - "Group is already running, try `cvd stop`"); - return groups[0]; + for (const LocalInstanceGroup& group : slot_groups) { + CF_EXPECT(HasCustomHome(group), + "Refusing to replace default-HOME instance group; try `cvd rm` " + "or `cvd reset -y`"); + } + for (LocalInstanceGroup& group : slot_groups) { + bool removed = + CF_EXPECT(instance_manager_.RemoveInstanceGroup(std::move(group))); + CF_EXPECT(removed == true, + "Group is already running or changed concurrently, try `cvd " + "stop`"); + } + return std::nullopt; } Result CvdCreateCommandHandler::Handle(const CommandRequest& request) { @@ -363,19 +525,41 @@ Result CvdCreateCommandHandler::Handle(const CommandRequest& request) { envs[kAndroidHostOut] = AbsolutePath(own_flags_.host_path); envs[kAndroidProductOut] = AbsolutePath(own_flags_.product_path); + GroupCreationInfo creation_info = CF_EXPECT(AnalyzeCreation({ + .envs = envs, + .selectors = request.Selectors(), + .num_instances = num_instances_parser_.NumInstances(), + .instance_ids = num_instances_parser_.InstanceIds(), + })); + CF_EXPECT(creation_info.group_directories.host_artifacts_path.has_value()); + CF_EXPECTF( + DirectoryExists(*creation_info.group_directories.host_artifacts_path), + "Host artifacts directory '{}' does not exist", + *creation_info.group_directories.host_artifacts_path); + for (const std::optional& product_path : + creation_info.group_directories.product_out_paths) { + CF_EXPECT(product_path.has_value()); + CF_EXPECTF(DirectoryExists(*product_path), + "Product directory '{}' does not exist", *product_path); + } + + CF_EXPECT(instance_manager_.PruneBrokenInstanceGroups()); + + SharedFD in_progress_lock; std::optional group; if (own_flags_.reuse) { - group = CF_EXPECT(FindReusableGroup(request.Selectors(), envs)); + group = CF_EXPECT(FindReusableGroup(request.Selectors(), creation_info)); } if (!group.has_value()) { - group = CF_EXPECT(CreateGroup(subcmd_args, envs, request)); + group = CF_EXPECT(CreateGroup(std::move(creation_info), &in_progress_lock)); } CF_EXPECT(group.has_value()); if (own_flags_.start) { - CF_EXPECT(StartGroup(*group, subcmd_args, envs, instance_manager_)); + CF_EXPECT(StartGroup(*group, subcmd_args, envs, instance_manager_, + std::move(in_progress_lock))); - if (CF_EXPECT(IsDefaultGroup(request))) { + if (CF_EXPECT(IsDefaultGroup(envs, request, *group))) { // For backward compatibility, we add extra symlink in system wide home // when HOME is NOT overridden and selector flags are NOT given. auto symlink_res = CreateSymlinks(*group); @@ -390,24 +574,18 @@ Result CvdCreateCommandHandler::Handle(const CommandRequest& request) { } Result CvdCreateCommandHandler::CreateGroup( - const std::vector& subcmd_args, - const std::unordered_map& envs, - const CommandRequest& request) { - GroupCreationInfo creation_info = CF_EXPECT(AnalyzeCreation({ - .envs = envs, - .selectors = request.Selectors(), - .num_instances = num_instances_parser_.NumInstances(), - .instance_ids = num_instances_parser_.InstanceIds(), - })); - - auto groups = CF_EXPECT(instance_manager_.FindGroups( - {.group_name = creation_info.group_creation_params.group_name})); - CF_EXPECTF(groups.empty(), "Group named '{}' already exists", - creation_info.group_creation_params.group_name); + GroupCreationInfo creation_info, SharedFD* out_in_progress_lock) { + if (!creation_info.group_creation_params.group_name.empty()) { + std::vector groups = + CF_EXPECT(instance_manager_.FindGroups( + {.group_name = creation_info.group_creation_params.group_name})); + CF_EXPECTF(groups.empty(), "Group named '{}' already exists", + creation_info.group_creation_params.group_name); + } LocalInstanceGroup group = CF_EXPECT(instance_manager_.CreateInstanceGroup( std::move(creation_info.group_creation_params), - std::move(creation_info.group_directories))); + std::move(creation_info.group_directories), out_in_progress_lock)); group.SetAllStates(cvd::INSTANCE_STATE_STOPPED); CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); @@ -513,9 +691,12 @@ std::vector CvdCreateCommandHandler::FlagModeFlags( .Help("Whether to start the instance group.")); flags.emplace_back( GflagsCompatFlag("reuse", own_flags_.reuse) - .Help("Whether to attempt reusing an existing group. Will fail if " - "there are multiple matching groups, or the chosen group has a " - "host tools / guest image mismatch.")); + .Help("Whether to attempt reusing an existing stopped group " + "matching the target HOME / instance slot (or replacing a " + "stopped custom-HOME slot group if paths or instances " + "changed). Fails if the group is running, if multiple " + "default-HOME groups match, or if an explicitly selected or " + "default-HOME group has a path or instance mismatch.")); return flags; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/create.h b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/create.h index cc3fef70777..26f3dde63d2 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/create.h +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/create.h @@ -18,13 +18,18 @@ #include #include +#include #include +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/flag_parser/flag.h" #include "cuttlefish/host/commands/cvd/cli/command_request.h" #include "cuttlefish/host/commands/cvd/cli/commands/command_handler.h" +#include "cuttlefish/host/commands/cvd/cli/selector/creation_analyzer.h" #include "cuttlefish/host/commands/cvd/cli/selector/num_instances_parser.h" +#include "cuttlefish/host/commands/cvd/cli/selector/selector_common_parser.h" #include "cuttlefish/host/commands/cvd/instances/instance_manager.h" +#include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" #include "cuttlefish/result/result.h" namespace cuttlefish { @@ -54,11 +59,10 @@ class CvdCreateCommandHandler : public CvdCommandHandler { const selector::SelectorOptions&); Result> FindReusableGroup( const selector::SelectorOptions& selectors, - const std::unordered_map& envs); + const selector::GroupCreationInfo& creation_info); Result CreateGroup( - const std::vector& subcmd_args, - const std::unordered_map& envs, - const CommandRequest& request); + selector::GroupCreationInfo creation_info, + SharedFD* out_in_progress_lock); InstanceManager& instance_manager_; CreateFlags own_flags_; diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp index 05821dc3bca..42746a8ead4 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp @@ -26,6 +26,7 @@ #include "fmt/core.h" #include "fmt/format.h" +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/common/libs/utils/files.h" #include "cuttlefish/flag_parser/flag.h" #include "cuttlefish/host/commands/cvd/cli/command_request.h" @@ -93,9 +94,10 @@ Result BuildStartCommand(const CommandRequest& request, .Build()); } -Result CreateGroup( - InstanceManager& instance_manager, const std::string& base_dir, - const EnvironmentSpecification& env_spec) { +Result CreateGroup(InstanceManager& instance_manager, + const std::string& base_dir, + const EnvironmentSpecification& env_spec, + SharedFD* out_in_progress_lock) { InstanceGroupParams group_params{ .group_name = env_spec.common().group_name(), }; @@ -106,7 +108,8 @@ Result CreateGroup( } return CF_EXPECT(instance_manager.CreateInstanceGroup( std::move(group_params), - CF_EXPECT(GetGroupCreationDirectories(base_dir, env_spec)))); + CF_EXPECT(GetGroupCreationDirectories(base_dir, env_spec)), + out_in_progress_lock)); } } // namespace @@ -167,9 +170,11 @@ Result LoadConfigsCommand::Handle(const CommandRequest& request) { }); auto listener_handle = CF_EXPECT(std::move(push_result)); + SharedFD in_progress_lock; group_creation_mtx.lock(); // Don't use CF_EXPECT here or the mutex will be left locked. - auto group_res = CreateGroup(instance_manager_, flags_.base_dir, env_spec); + auto group_res = CreateGroup(instance_manager_, flags_.base_dir, env_spec, + &in_progress_lock); if (group_res.has_value()) { // Have to initialize the group_name variable before releasing the mutex. group_name = (*group_res).GroupName(); @@ -177,9 +182,14 @@ Result LoadConfigsCommand::Handle(const CommandRequest& request) { group_creation_mtx.unlock(); auto group = CF_EXPECT(std::move(group_res)); - auto cvd_flags = CF_EXPECT(ParseCvdConfigs(env_spec, group)); + Result cvd_flags_res = ParseCvdConfigs(env_spec, group); + if (!cvd_flags_res.has_value()) { + group.SetAllStates(cvd::INSTANCE_STATE_PREPARE_FAILED); + (void)instance_manager_.UpdateInstanceGroup(group); + } + CvdFlags cvd_flags = CF_EXPECT(std::move(cvd_flags_res)); - auto res = LoadGroup(request, group, std::move(cvd_flags)); + auto res = LoadGroup(request, group, std::move(cvd_flags), in_progress_lock); if (!res.has_value()) { auto first_instance_state = group.Instances()[0].State(); // The failure could have occurred during prepare(fetch) or start @@ -197,7 +207,8 @@ Result LoadConfigsCommand::Handle(const CommandRequest& request) { Result LoadConfigsCommand::LoadGroup(const CommandRequest& request, LocalInstanceGroup& group, - CvdFlags cvd_flags) { + CvdFlags cvd_flags, + SharedFD& in_progress_lock) { auto mkdir_res = EnsureDirectoryExists(group.HomeDir(), 0775, /* group_name */ ""); if (!mkdir_res.has_value()) { @@ -229,7 +240,8 @@ Result LoadConfigsCommand::LoadGroup(const CommandRequest& request, CommandRequest start_cmd = CF_EXPECT(BuildStartCommand(request, cvd_flags, group)); std::unique_ptr start_handler = - std::make_unique(instance_manager_); + std::make_unique(instance_manager_, + std::move(in_progress_lock)); CF_EXPECT(start_handler->Handle(start_cmd)); return {}; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.h b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.h index 24d1806e1db..16bfb650af5 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.h +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.h @@ -17,6 +17,7 @@ #include +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/flag_parser/flag.h" #include "cuttlefish/host/commands/cvd/cli/command_request.h" #include "cuttlefish/host/commands/cvd/cli/commands/command_handler.h" @@ -48,7 +49,8 @@ class LoadConfigsCommand : public CvdCommandHandler { private: Result LoadGroup(const CommandRequest& request, - LocalInstanceGroup& group, CvdFlags cvd_flags); + LocalInstanceGroup& group, CvdFlags cvd_flags, + SharedFD& in_progress_lock); InstanceManager& instance_manager_; LoadFlags flags_; diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/remove.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/remove.cpp index 747c7a415e0..60ad8abec30 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/remove.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/remove.cpp @@ -74,6 +74,7 @@ Result RemoveCvdCommandHandler::Handle(const CommandRequest& request) { return CF_ERR(NoGroupMessage(request)); } auto group = CF_EXPECT(selector::SelectGroup(instance_manager_, request)); + group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(group)); Result stop_res = StopGroup(group); if (!stop_res.has_value()) { @@ -82,7 +83,9 @@ Result RemoveCvdCommandHandler::Handle(const CommandRequest& request) { "kill any remaining device processes."; } - CF_EXPECT(instance_manager_.RemoveInstanceGroup(group)); + bool removed = CF_EXPECT(instance_manager_.RemoveInstanceGroup(group)); + CF_EXPECT(removed == true, + "Instance group no longer exists or changed concurrently"); return {}; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.cpp index 4aea9fc938b..441106c191c 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -376,8 +377,9 @@ bool CanBypassToSingleInstance(const LocalInstance& instance, } // namespace CvdStartCommandHandler::CvdStartCommandHandler( - InstanceManager& instance_manager) - : instance_manager_(instance_manager) { + InstanceManager& instance_manager, SharedFD preacquired_in_progress_lock) + : instance_manager_(instance_manager), + preacquired_in_progress_lock_(std::move(preacquired_in_progress_lock)) { own_flags_.daemon = true; own_flags_.print_group_format = isatty(0) ? "human" : "json"; } @@ -396,6 +398,13 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { request.Selectors().instance_names->size() == 1) { auto [instance, group] = CF_EXPECT(selector::SelectInstance(instance_manager_, request)); + group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(group)); + for (const LocalInstance& refreshed_inst : group.Instances()) { + if (refreshed_inst.Id() == instance.Id()) { + instance = refreshed_inst; + break; + } + } if (CanBypassToSingleInstance(instance, group, subcmd_args)) { CF_EXPECT(LaunchSingleInstance(instance, group, request)); @@ -413,9 +422,18 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { CF_EXPECT(selector::SelectGroup(instance_manager_, request), "Failed to select group to start, did you mean 'cvd create'?"); + SharedFD in_progress_lock = std::move(preacquired_in_progress_lock_); + if (!in_progress_lock->IsOpen()) { + in_progress_lock = + CF_EXPECT(InstanceManager::AcquireGroupInProgressLock(group)); + } + group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState( + group, /*caller_holds_in_progress_lock=*/true)); + CF_EXPECT(!group.HasActiveInstances(), "Selected instance group is already started, use `cvd create` to " "create a new one."); + CF_EXPECT(instance_manager_.EnsureInstanceLocksInUse(group)); std::unordered_map envs = request.Env(); @@ -443,8 +461,10 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { Command command = CF_EXPECT(ConstructCvdNonHelpCommand( bin, group, subcmd_args, envs, request, memfd)); + std::mutex group_mutex; Result> handle_res = - PushInterruptListener([this, &group, stop_monitor](int signal) { + PushInterruptListener([this, &group, &group_mutex, + stop_monitor](int signal) { LOG(WARNING) << strsignal(signal) << " signal received, cleanning up"; stop_monitor(); Result interrupt_res = subprocess_waiter_.Interrupt(); @@ -455,38 +475,45 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { "run `cvd reset` to ensure a clean state"; } - group.SetAllStates(cvd::INSTANCE_STATE_CANCELLED); - Result update_res = instance_manager_.UpdateInstanceGroup(group); - if (!update_res.has_value()) { - LOG(ERROR) << "Failed to update group status: " << update_res.error(); + { + std::lock_guard lock(group_mutex); + group.SetAllStates(cvd::INSTANCE_STATE_CANCELLED); + Result update_res = + instance_manager_.UpdateInstanceGroup(group); + if (!update_res.has_value()) { + LOG(ERROR) << "Failed to update group status: " + << update_res.error(); + } } // It's technically possible for the group's state to be set to - // "running" before abort has a chance to run, but that can only happen - // if the instances are indeed running, so it's OK. + // "running" before abort has a chance to run, but that can only + // happen if the instances are indeed running, so it's OK. std::abort(); }); std::unique_ptr listener_handle = CF_EXPECT(std::move(handle_res)); - group.SetAllStates(cvd::INSTANCE_STATE_STARTING); - group.SetStartTime(CvdServerClock::now()); - CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); + { + std::lock_guard lock(group_mutex); + group.SetAllStates(cvd::INSTANCE_STATE_STARTING); + group.SetStartTime(CvdServerClock::now()); + CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); + } Result monitor_res; #ifdef __linux__ std::thread monitor_thread; if (!own_flags_.daemon) { - const LocalInstance& instance = *group.Instances().begin(); - const std::string assemble_log = - absl::StrCat(instance.AssemblyDirectory(), "/", kLogNameAssembleCvd); + const LocalInstance first_instance = *group.Instances().begin(); + const std::string assemble_log = absl::StrCat( + first_instance.AssemblyDirectory(), "/", kLogNameAssembleCvd); (void)RemoveFile(assemble_log); - for (const auto& log : CF_EXPECT(instance.LogsFilenames())) { + for (const auto& log : CF_EXPECT(first_instance.LogsFilenames())) { (void)RemoveFile(log); } - monitor_thread = std::thread([&group, stop_read, &monitor_res]() { - const LocalInstance& first_instance = *group.Instances().begin(); + monitor_thread = std::thread([first_instance, stop_read, &monitor_res]() { monitor_res = MonitorLogs(first_instance, stop_read); }); } @@ -505,8 +532,8 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { }; #endif - Result launch_res = - LaunchDeviceInterruptible(std::move(command), group, envs, request); + Result launch_res = LaunchDeviceInterruptible( + std::move(command), group, group_mutex, envs, request); if (!launch_res.has_value()) { if (!own_flags_.daemon) { @@ -518,8 +545,12 @@ Result CvdStartCommandHandler::Handle(const CommandRequest& request) { return launch_res; } - group.SetAllStates(cvd::INSTANCE_STATE_RUNNING); - CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); + { + std::lock_guard lock(group_mutex); + group.SetAllStates(cvd::INSTANCE_STATE_RUNNING); + CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); + } + in_progress_lock = SharedFD(); if (!own_flags_.daemon) { listener_handle.reset(); @@ -607,7 +638,7 @@ Result CvdStartCommandHandler::LaunchDevice( } Result CvdStartCommandHandler::LaunchDeviceInterruptible( - Command command, LocalInstanceGroup& group, + Command command, LocalInstanceGroup& group, std::mutex& group_mutex, const std::unordered_map& envs, const CommandRequest& request) { // cvd_internal_start uses the config from the previous invocation to @@ -624,6 +655,7 @@ Result CvdStartCommandHandler::LaunchDeviceInterruptible( const Result start_res = LaunchDevice(std::move(command), group, group.Instances()); if (!start_res.has_value()) { + std::lock_guard lock(group_mutex); group.SetAllStates(cvd::INSTANCE_STATE_BOOT_FAILED); CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); return start_res; @@ -635,6 +667,28 @@ Result CvdStartCommandHandler::LaunchDeviceInterruptible( Result CvdStartCommandHandler::LaunchSingleInstance( LocalInstance& instance, LocalInstanceGroup& group, const CommandRequest& request) { + SharedFD in_progress_lock = std::move(preacquired_in_progress_lock_); + if (!in_progress_lock->IsOpen()) { + in_progress_lock = + CF_EXPECT(InstanceManager::AcquireGroupInProgressLock(group)); + } + group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState( + group, /*caller_holds_in_progress_lock=*/true)); + bool found_instance = false; + for (const LocalInstance& refreshed_inst : group.Instances()) { + if (refreshed_inst.Id() == instance.Id()) { + instance = refreshed_inst; + found_instance = true; + break; + } + } + CF_EXPECT(found_instance == true, + "Selected instance no longer exists in group"); + CF_EXPECT( + CanBypassToSingleInstance(instance, group, request.SubcommandArguments()), + "Selected instance cannot be started individually in its current state"); + CF_EXPECT(instance_manager_.EnsureInstanceLocksInUse(group)); + const std::string bin_path = group.HostArtifactsPath() + "/bin/run_cvd"; std::unordered_map run_cvd_envs = request.Env(); run_cvd_envs[kCuttlefishInstanceEnvVarName] = std::to_string(instance.Id()); @@ -679,7 +733,6 @@ Result CvdStartCommandHandler::LaunchSingleInstance( }; set_instance_state(cvd::INSTANCE_STATE_STARTING); - group.SetStartTime(CvdServerClock::now()); CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); const Result start_res = @@ -692,6 +745,7 @@ Result CvdStartCommandHandler::LaunchSingleInstance( set_instance_state(cvd::INSTANCE_STATE_RUNNING); CF_EXPECT(instance_manager_.UpdateInstanceGroup(group)); + in_progress_lock = SharedFD(); const Json::Value group_json = CF_EXPECT(group.FetchStatus()); std::cout << group_json.toStyledString(); diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.h b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.h index b8f2b229aa0..103f5ab3305 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.h +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/start.h @@ -16,10 +16,13 @@ #pragma once +#include #include #include +#include #include +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/flag_parser/flag.h" #include "cuttlefish/host/commands/cvd/cli/commands/command_handler.h" #include "cuttlefish/host/commands/cvd/instances/instance_manager.h" @@ -32,7 +35,8 @@ namespace cuttlefish { class CvdStartCommandHandler : public CvdCommandHandler { public: - CvdStartCommandHandler(InstanceManager& instance_manager); + CvdStartCommandHandler(InstanceManager& instance_manager, + SharedFD preacquired_in_progress_lock = {}); Result Handle(const CommandRequest& request) override; std::vector CmdList() const override; @@ -54,7 +58,7 @@ class CvdStartCommandHandler : public CvdCommandHandler { const std::vector& instances); Result LaunchDeviceInterruptible( - Command command, LocalInstanceGroup& group, + Command command, LocalInstanceGroup& group, std::mutex& group_mutex, const std::unordered_map& envs, const CommandRequest& request); @@ -62,6 +66,7 @@ class CvdStartCommandHandler : public CvdCommandHandler { std::vector BuildOwnFlags(); InstanceManager& instance_manager_; + SharedFD preacquired_in_progress_lock_; SubprocessWaiter subprocess_waiter_; struct { std::vector host_substitutions; diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/stop.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/stop.cpp index a8603c7521f..2cd844df4d7 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/stop.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/stop.cpp @@ -54,6 +54,7 @@ Result CvdStopCommandHandler::Handle(const CommandRequest& request) { } auto group = CF_EXPECT(selector::SelectGroup(instance_manager_, request)); + group = CF_EXPECT(instance_manager_.RefreshInstanceGroupState(group)); CF_EXPECT(group.HasActiveInstances(), "Selected group is not running"); CF_EXPECT(ConsumeFlags(CF_EXPECT(Flags(request)), cmd_args, diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/create_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/create_test.cpp new file mode 100644 index 00000000000..a313948269c --- /dev/null +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/create_test.cpp @@ -0,0 +1,911 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/commands/cvd/cli/commands/create.h" + +#include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/common/libs/fs/shared_fd.h" +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/common/libs/utils/users.h" +#include "cuttlefish/files/directory_exists.h" +#include "cuttlefish/files/file_exists.h" +#include "cuttlefish/files/recursively_remove_directory.h" +#include "cuttlefish/files/write_new_file.h" +#include "cuttlefish/host/commands/cvd/cli/command_request.h" +#include "cuttlefish/host/commands/cvd/cli/commands/remove.h" +#include "cuttlefish/host/commands/cvd/cli/commands/start.h" +#include "cuttlefish/host/commands/cvd/cli/commands/stop.h" +#include "cuttlefish/host/commands/cvd/cli/selector/selector_common_parser.h" +#include "cuttlefish/host/commands/cvd/instances/cvd_persistent_data.pb.h" +#include "cuttlefish/host/commands/cvd/instances/instance_database.h" +#include "cuttlefish/host/commands/cvd/instances/instance_manager.h" +#include "cuttlefish/host/commands/cvd/instances/local_instance.h" +#include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" +#include "cuttlefish/host/commands/cvd/instances/lock/instance_lock.h" +#include "cuttlefish/host/commands/cvd/instances/lock/lock_file.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::HasSubstr; + +class ScopedTestWorkspace { + public: + ScopedTestWorkspace() { + // Keep the root short enough to bind AF_UNIX `launcher_monitor.sock` + // sockets (< 108 bytes); Bazel's sandbox `TEST_TMPDIR` is ~185 bytes. + const char* test_tmpdir = getenv("TEST_TMPDIR"); + std::string base = (test_tmpdir != nullptr && test_tmpdir[0] != '\0' && + std::string(test_tmpdir).size() < 50) + ? test_tmpdir + : "/tmp"; + std::string tmpl = absl::StrCat(base, "/cf_create_test.XXXXXX"); + char* created = mkdtemp(tmpl.data()); + if (created != nullptr) { + root_ = created; + setenv("CUTTLEFISH_METRICS_TRANSMITTER_PATH", + absl::StrCat(root_, "/no_metrics_transmitter").c_str(), 1); + setenv("XDG_DATA_HOME", absl::StrCat(root_, "/xdg_data").c_str(), 1); + } + } + + ~ScopedTestWorkspace() { + if (!root_.empty()) { + (void)RecursivelyRemoveDirectory(root_); + } + } + + ScopedTestWorkspace(const ScopedTestWorkspace&) = delete; + ScopedTestWorkspace& operator=(const ScopedTestWorkspace&) = delete; + + bool Valid() const { return !root_.empty(); } + const std::string& Root() const { return root_; } + + Result MakeArtifactsDir(const std::string& name) const { + std::string dir = absl::StrCat(root_, "/", name); + std::string bin_dir = absl::StrCat(dir, "/bin"); + CF_EXPECT(EnsureDirectoryExists(bin_dir)); + CF_EXPECT(Fd::Creat(absl::StrCat(bin_dir, "/cvd_internal_start"), + S_IRWXU | S_IRGRP | S_IXGRP)); + return dir; + } + + private: + std::string root_; +}; + +struct HermeticCvdContext { + explicit HermeticCvdContext(const std::string& root) + : locks_dir(absl::StrCat(root, "/locks/")), + db_path(absl::StrCat(root, "/instance_database.binpb")), + per_user_dir(absl::StrCat(root, "/per_user")), + lock_manager(locks_dir), + instance_db(db_path), + instance_manager(lock_manager, instance_db, per_user_dir) {} + + std::string locks_dir; + std::string db_path; + std::string per_user_dir; + InstanceLockFileManager lock_manager; + InstanceDatabase instance_db; + InstanceManager instance_manager; +}; + +Result RunCreateNoStart( + InstanceManager& instance_manager, + const std::unordered_map& env, + const std::vector& extra_args, + selector::SelectorOptions selectors = {}) { + CommandRequest request = + CF_EXPECT(CommandRequestBuilder() + .SetEnv(env) + .AddArguments({"cvd", "create", "--nostart"}) + .AddArguments(extra_args) + .SetSelectorOptions(std::move(selectors)) + .Build()); + CvdCreateCommandHandler handler(instance_manager); + CF_EXPECT(handler.Handle(request)); + return {}; +} + +Result StartMonitorSocketForInstance(const std::string& ws_root, + const LocalInstance& instance) { + std::string uds_base = absl::StrCat(ws_root, "/u"); + std::string inst_uds = absl::StrCat(uds_base, "/cvd-", instance.Id()); + CF_EXPECT(EnsureDirectoryExists(inst_uds)); + std::string config_json = + absl::StrCat("{\"instances_uds_dir\": \"", uds_base, "\"}\n"); + CF_EXPECT(WriteNewFile( + absl::StrCat(instance.HomeDirectory(), "/.cuttlefish_config.json"), + config_json)); + std::string sock_path = absl::StrCat(inst_uds, "/launcher_monitor.sock"); + return CF_EXPECT(Fd::SocketLocalServer(sock_path, /*is_abstract=*/false, + SOCK_STREAM, 0600)); +} + +TEST(CvdCreateReuseTest, ReusesStoppedGroupInSameCustomHome) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot = ws.MakeArtifactsDir("acloud_cf_25"); + ASSERT_THAT(slot, IsOk()); + + std::unordered_map env = {{"HOME", *slot}}; + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=25"}), + IsOk()); + + Result> groups1 = + ctx.instance_manager.FindGroups({}); + ASSERT_THAT(groups1, IsOk()); + ASSERT_EQ(groups1->size(), 1u); + EXPECT_EQ((*groups1)[0].GroupName(), "cvd_1"); + ASSERT_EQ((*groups1)[0].Instances().size(), 1u); + EXPECT_EQ((*groups1)[0].Instances()[0].Id(), 25u); + + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=25"}), + IsOk()); + + Result> groups2 = + ctx.instance_manager.FindGroups({}); + ASSERT_THAT(groups2, IsOk()); + ASSERT_EQ(groups2->size(), 1u); + EXPECT_EQ((*groups2)[0].GroupName(), "cvd_1"); + EXPECT_EQ((*groups2)[0].BaseDir(), (*groups1)[0].BaseDir()); +} + +TEST(CvdCreateReuseTest, RefreshesStaleRunningStateLeftByStopCvd) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot = ws.MakeArtifactsDir("acloud_cf_25"); + ASSERT_THAT(slot, IsOk()); + std::unordered_map env = {{"HOME", *slot}}; + + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=25"}), + IsOk()); + + Result group = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(group, IsOk()); + group->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*group), IsOk()); + + // No monitor socket or run_cvd process exists (simulating stop_cvd teardown). + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=25"}), + IsOk()); + + Result refreshed = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(refreshed, IsOk()); + ASSERT_EQ(refreshed->Instances().size(), 1u); + EXPECT_EQ(refreshed->Instances()[0].State(), cvd::INSTANCE_STATE_STOPPED); +} + +TEST(CvdCreateReuseTest, RejectsReuseWhenLauncherMonitorIsAlive) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot = ws.MakeArtifactsDir("acloud_cf_10"); + ASSERT_THAT(slot, IsOk()); + std::unordered_map env = {{"HOME", *slot}}; + + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=10"}), + IsOk()); + + Result group = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(group, IsOk()); + group->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*group), IsOk()); + + Result monitor_server = + StartMonitorSocketForInstance(ws.Root(), group->Instances()[0]); + ASSERT_THAT(monitor_server, IsOk()); + + Result second_res = RunCreateNoStart( + ctx.instance_manager, env, {"--reuse=true", "--base_instance_num=10"}); + ASSERT_FALSE(second_res.has_value()); + EXPECT_THAT(second_res.error().Message(), + HasSubstr("Group is already running")); +} + +TEST(CvdCreateReuseTest, RejectsReuseWhenGroupStartIsInProgress) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot = ws.MakeArtifactsDir("acloud_cf_11"); + ASSERT_THAT(slot, IsOk()); + std::unordered_map env = {{"HOME", *slot}}; + + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=11"}), + IsOk()); + + Result group = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(group, IsOk()); + Result in_progress = + InstanceManager::AcquireGroupInProgressLock(*group); + ASSERT_THAT(in_progress, IsOk()); + group->SetAllStates(cvd::INSTANCE_STATE_STARTING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*group), IsOk()); + + Result second_res = RunCreateNoStart( + ctx.instance_manager, env, {"--reuse=true", "--base_instance_num=11"}); + ASSERT_FALSE(second_res.has_value()); + EXPECT_THAT(second_res.error().Message(), + HasSubstr("Group is already running")); +} + +TEST(CvdCreateReuseTest, AllowsMultipleConcurrentSlotsUnderSameUser) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot17 = ws.MakeArtifactsDir("acloud_cf_17"); + ASSERT_THAT(slot17, IsOk()); + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *slot17}}, + {"--reuse=true", "--base_instance_num=17"}), + IsOk()); + + Result group17 = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(group17, IsOk()); + group17->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*group17), IsOk()); + Result monitor17 = + StartMonitorSocketForInstance(ws.Root(), group17->Instances()[0]); + ASSERT_THAT(monitor17, IsOk()); + + const std::vector shards = {44, 58, 1, 4, 45, 33, 56, 5}; + for (int shard : shards) { + SCOPED_TRACE(shard); + Result slot = + ws.MakeArtifactsDir(absl::StrCat("acloud_cf_", shard)); + ASSERT_THAT(slot, IsOk()); + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *slot}}, + {"--reuse=true", + absl::StrCat("--base_instance_num=", shard)}), + IsOk()); + } + + Result> all_groups = + ctx.instance_manager.FindGroups({}); + ASSERT_THAT(all_groups, IsOk()); + EXPECT_EQ(all_groups->size(), 9u); +} + +TEST(CvdCreateReuseTest, PrunesDeletedGroupDirectoryAndRecyclesInstanceId) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result temp1 = ws.MakeArtifactsDir("temp_run_1"); + ASSERT_THAT(temp1, IsOk()); + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *temp1}}, + {"--reuse=true"}), + IsOk()); + + Result group1 = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(group1, IsOk()); + EXPECT_EQ(group1->Instances()[0].Id(), 1u); + // Simulate stop_cvd leaving state RUNNING followed by temp dir removal. + group1->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*group1), IsOk()); + ASSERT_THAT(RecursivelyRemoveDirectory(*temp1), IsOk()); + + Result temp2 = ws.MakeArtifactsDir("temp_run_2"); + ASSERT_THAT(temp2, IsOk()); + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *temp2}}, + {"--reuse=true"}), + IsOk()); + + Result> groups = + ctx.instance_manager.FindGroups({}); + ASSERT_THAT(groups, IsOk()); + ASSERT_EQ(groups->size(), 1u); + EXPECT_EQ((*groups)[0].Instances()[0].Id(), 1u); +} + +TEST(CvdCreateReuseTest, ReplacesStoppedSlotGroupWhenProductOrCountChanges) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot = ws.MakeArtifactsDir("dev_home"); + Result prod_a = ws.MakeArtifactsDir("prod_a"); + Result prod_b = ws.MakeArtifactsDir("prod_b"); + ASSERT_THAT(slot, IsOk()); + ASSERT_THAT(prod_a, IsOk()); + ASSERT_THAT(prod_b, IsOk()); + + std::unordered_map env = {{"HOME", *slot}}; + ASSERT_THAT( + RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--host_path=" + *slot, + "--product_path=" + *prod_a, "--num_instances=2"}), + IsOk()); + + // Invalid product_path must fail BEFORE removing the existing stopped group. + ASSERT_FALSE( + RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--host_path=" + *slot, + "--product_path=" + ws.Root() + "/missing_dir"}) + .has_value()); + Result> kept_groups = + ctx.instance_manager.FindGroups({}); + ASSERT_THAT(kept_groups, IsOk()); + ASSERT_EQ(kept_groups->size(), 1u); + EXPECT_EQ((*kept_groups)[0].Instances().size(), 2u); + + // Explicit --group_name selector with mismatched product_path must fail. + Result selector_mismatch = + RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--host_path=" + *slot, + "--product_path=" + *prod_b, "--num_instances=2"}, + {.group_name = "cvd_1"}); + ASSERT_FALSE(selector_mismatch.has_value()); + EXPECT_THAT(selector_mismatch.error().Message(), + HasSubstr("Trying to change the host paths")); + + // Slot-based reuse (no selector) replaces the stopped group without leaking. + ASSERT_THAT( + RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--host_path=" + *slot, + "--product_path=" + *prod_b, "--num_instances=1"}), + IsOk()); + Result> replaced_groups = + ctx.instance_manager.FindGroups({}); + ASSERT_THAT(replaced_groups, IsOk()); + ASSERT_EQ(replaced_groups->size(), 1u); + EXPECT_EQ((*replaced_groups)[0].Instances().size(), 1u); + EXPECT_EQ((*replaced_groups)[0].Instances()[0].Id(), 1u); +} + +TEST(CvdCreateReuseTest, FailsWhenMultipleDefaultHomeGroupsAreAmbiguous) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result sys_home = SystemWideUserHome(); + ASSERT_THAT(sys_home, IsOk()); + Result artifacts = ws.MakeArtifactsDir("artifacts"); + ASSERT_THAT(artifacts, IsOk()); + + std::unordered_map env = { + {"HOME", *sys_home}, + {"ANDROID_HOST_OUT", *artifacts}, + {"ANDROID_PRODUCT_OUT", *artifacts}, + }; + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, {"--reuse=false"}), + IsOk()); + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, {"--reuse=false"}), + IsOk()); + + // Bare --reuse=true with two default-HOME groups is ambiguous. + Result ambiguous = + RunCreateNoStart(ctx.instance_manager, env, {"--reuse=true"}); + ASSERT_FALSE(ambiguous.has_value()); + EXPECT_THAT(ambiguous.error().Message(), + HasSubstr("Unclear which group to reuse")); + + // Explicit --base_instance_num=2 disambiguates cleanly! + EXPECT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=2"}), + IsOk()); +} + +TEST(CvdCreateReuseTest, ConcurrentMultiProcessCreatesPreserveAllGroups) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + + constexpr int kNumProcs = 12; + std::vector slot_dirs; + slot_dirs.reserve(kNumProcs); + for (int i = 1; i <= kNumProcs; ++i) { + Result slot = + ws.MakeArtifactsDir(absl::StrCat("concurrent_slot_", i)); + ASSERT_THAT(slot, IsOk()); + slot_dirs.push_back(*slot); + } + + std::vector pids; + pids.reserve(kNumProcs); + for (int i = 1; i <= kNumProcs; ++i) { + pid_t pid = fork(); + ASSERT_GE(pid, 0); + if (pid == 0) { + HermeticCvdContext child_ctx(ws.Root()); + Result res = RunCreateNoStart( + child_ctx.instance_manager, {{"HOME", slot_dirs[i - 1]}}, + {"--reuse=true", absl::StrCat("--base_instance_num=", i)}); + if (!res.has_value()) { + std::cerr << "Child " << i << " failed: " << res.error().FormatForEnv() + << std::endl; + } + _exit(res.has_value() ? 0 : 1); + } + pids.push_back(pid); + } + + for (pid_t pid : pids) { + int status = 0; + ASSERT_EQ(waitpid(pid, &status, 0), pid); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0) + << "Child pid " << pid << " failed with status " << status; + } + + HermeticCvdContext verify_ctx(ws.Root()); + Result> groups = + verify_ctx.instance_manager.FindGroups({}); + ASSERT_THAT(groups, IsOk()); + EXPECT_EQ(groups->size(), static_cast(kNumProcs)); +} + +TEST(CvdCreateReuseTest, + HandlesLongGroupPathsWithoutAbortingWhenMonitorMissing) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + std::string deep_root = + absl::StrCat(ws.Root(), + "/very_long_per_user_directory_name_exceeding_unix_socket_" + "limit_0123456789"); + ASSERT_THAT(EnsureDirectoryExists(deep_root), IsOk()); + HermeticCvdContext ctx(deep_root); + + Result slot = ws.MakeArtifactsDir("acloud_cf_99"); + ASSERT_THAT(slot, IsOk()); + std::unordered_map env = {{"HOME", *slot}}; + + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=99"}), + IsOk()); + + Result group = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(group, IsOk()); + std::string fallback_inst_dir = + absl::StrCat(group->HomeDir(), "/cuttlefish/instances/cvd-99"); + ASSERT_THAT(EnsureDirectoryExists(fallback_inst_dir), IsOk()); + std::string long_monitor_path = + absl::StrCat(fallback_inst_dir, "/launcher_monitor.sock"); + ASSERT_GE(long_monitor_path.size(), sizeof(sockaddr_un::sun_path)); + ASSERT_THAT(WriteNewFile(long_monitor_path, ""), IsOk()); + group->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*group), IsOk()); + + EXPECT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=99"}), + IsOk()); +} + +TEST(CvdCreateReuseTest, + StandaloneCvdStartRefreshesStaleRunningStateAndRestoresBootFailedLock) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot = ws.MakeArtifactsDir("acloud_cf_25"); + ASSERT_THAT(slot, IsOk()); + + std::unordered_map env = { + {"HOME", *slot}, + {"ANDROID_HOST_OUT", *slot}, + {"ANDROID_PRODUCT_OUT", *slot}, + }; + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=25"}), + IsOk()); + + // Simulate `stop_cvd` leaving stale RUNNING state AND `CvdResetGroup` having + // removed `local-instance-25.lock`. + Result group = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(group, IsOk()); + group->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*group), IsOk()); + ASSERT_THAT(ctx.lock_manager.RemoveLockFile(25), IsOk()); + + Result start_req = + CommandRequestBuilder() + .SetEnv(env) + .AddArguments( + {"cvd", "start", "--daemon=true", "--print_group_format=none"}) + .SetSelectorOptions({.group_name = "cvd_1"}) + .Build(); + ASSERT_THAT(start_req, IsOk()); + CvdStartCommandHandler start_handler(ctx.instance_manager); + // In the unit test binary (`create_test` rather than + // `cuttlefish-common/bin/cvd`), `Handle` must succeed through + // `AcquireGroupInProgressLock`, + // `RefreshInstanceGroupState(caller_holds_in_progress_lock=true)` (clearing + // stale RUNNING -> STOPPED instead of self-deadlocking on + // `in_progress.lock`), and `EnsureInstanceLocksInUse`, before + // `HostPackageSubstitution` checks the executable path. + Result start_res = start_handler.Handle(*start_req); + ASSERT_FALSE(start_res.has_value()); + EXPECT_THAT(start_res.error().Message(), HasSubstr("cuttlefish-common/bin")); + + Result refreshed = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(refreshed, IsOk()); + EXPECT_EQ(refreshed->Instances()[0].State(), cvd::INSTANCE_STATE_STOPPED); + + Result lock = ctx.lock_manager.AcquireLock(25); + ASSERT_THAT(lock, IsOk()); + Result lock_state = lock->Status(); + ASSERT_THAT(lock_state, IsOk()); + EXPECT_EQ(*lock_state, InUseState::kInUse); +} + +TEST(CvdCreateReuseTest, StandaloneCvdStopAndRemoveHandleStaleRunningState) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot = ws.MakeArtifactsDir("acloud_cf_30"); + ASSERT_THAT(slot, IsOk()); + std::unordered_map env = {{"HOME", *slot}}; + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=30"}), + IsOk()); + + Result group = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(group, IsOk()); + group->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*group), IsOk()); + + // `cvd stop` refreshes stale RUNNING -> STOPPED and reports not running. + Result stop_req = + CommandRequestBuilder() + .SetEnv(env) + .AddArguments({"cvd", "stop"}) + .SetSelectorOptions({.group_name = "cvd_1"}) + .Build(); + ASSERT_THAT(stop_req, IsOk()); + CvdStopCommandHandler stop_handler(ctx.instance_manager); + Result stop_res = stop_handler.Handle(*stop_req); + ASSERT_FALSE(stop_res.has_value()); + EXPECT_THAT(stop_res.error().Message(), + HasSubstr("Selected group is not running")); + + Result after_stop = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(after_stop, IsOk()); + EXPECT_EQ(after_stop->Instances()[0].State(), cvd::INSTANCE_STATE_STOPPED); + + // Mark stale RUNNING again and verify `cvd rm` removes it cleanly without + // needing a working `stop_cvd` binary. + after_stop->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*after_stop), IsOk()); + + Result rm_req = + CommandRequestBuilder() + .SetEnv(env) + .AddArguments({"cvd", "rm"}) + .SetSelectorOptions({.group_name = "cvd_1"}) + .Build(); + ASSERT_THAT(rm_req, IsOk()); + RemoveCvdCommandHandler rm_handler(ctx.instance_manager); + ASSERT_THAT(rm_handler.Handle(*rm_req), IsOk()); + + Result> remaining = + ctx.instance_manager.FindGroups({}); + ASSERT_THAT(remaining, IsOk()); + EXPECT_TRUE(remaining->empty()); + EXPECT_FALSE( + DirectoryExists(after_stop->BaseDir(), /*follow_symlinks=*/false)); +} + +TEST(CvdCreateReuseTest, + PreparingGroupHoldingInProgressLockSurvivesConcurrentPrune) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + // Create an empty fetch target dir (no `bin/cvd_internal_start` yet), + // simulating `cvd load` while `cvd fetch` is in progress. + std::string fetching_dir = absl::StrCat(ws.Root(), "/fetching_artifacts"); + ASSERT_THAT(EnsureDirectoryExists(fetching_dir), IsOk()); + + SharedFD in_progress_lock; + Result preparing_group = + ctx.instance_manager.CreateInstanceGroup( + InstanceGroupParams{ + .group_name = "loading_grp", + .instances = {InstanceParams{.instance_id = 50, + .per_instance_name = "ins-1"}}, + }, + InstanceManager::GroupDirectories{ + .home = fetching_dir, + .host_artifacts_path = fetching_dir, + .product_out_paths = {fetching_dir}, + }, + &in_progress_lock); + ASSERT_THAT(preparing_group, IsOk()); + ASSERT_TRUE(in_progress_lock->IsOpen()); + + // Concurrent `cvd create --reuse=true` on another slot triggers + // `PruneBrokenInstanceGroups()`, which must NOT prune `loading_grp` while + // `in_progress_lock` is held. + Result slot2 = ws.MakeArtifactsDir("acloud_cf_51"); + ASSERT_THAT(slot2, IsOk()); + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *slot2}}, + {"--reuse=true", "--base_instance_num=51"}), + IsOk()); + + Result still_preparing = + ctx.instance_manager.FindGroup({.group_name = "loading_grp"}); + ASSERT_THAT(still_preparing, IsOk()); + EXPECT_EQ(still_preparing->Instances()[0].State(), + cvd::INSTANCE_STATE_PREPARING); + EXPECT_TRUE(DirectoryExists(still_preparing->BaseDir())); + + // Once `in_progress_lock` is dropped (e.g. `cvd load` was SIGKILLed mid-fetch + // leaving empty artifacts), `PruneBrokenInstanceGroups` cleans it up. + in_progress_lock = SharedFD(); + ASSERT_THAT(ctx.instance_manager.PruneBrokenInstanceGroups(), IsOk()); + Result> after_prune = + ctx.instance_manager.FindGroups({.group_name = "loading_grp"}); + ASSERT_THAT(after_prune, IsOk()); + EXPECT_TRUE(after_prune->empty()); +} + +TEST(CvdCreateReuseTest, + DefaultReuseFalsePrunesBrokenStaleGroupsAndRecyclesInstanceId) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result temp1 = ws.MakeArtifactsDir("ephemeral_1"); + ASSERT_THAT(temp1, IsOk()); + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *temp1}}, + {"--reuse=false"}), + IsOk()); + + Result group1 = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(group1, IsOk()); + EXPECT_EQ(group1->Instances()[0].Id(), 1u); + group1->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*group1), IsOk()); + ASSERT_THAT(RecursivelyRemoveDirectory(*temp1), IsOk()); + + Result temp2 = ws.MakeArtifactsDir("ephemeral_2"); + ASSERT_THAT(temp2, IsOk()); + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *temp2}}, + {"--reuse=false"}), + IsOk()); + + Result> groups = + ctx.instance_manager.FindGroups({}); + ASSERT_THAT(groups, IsOk()); + ASSERT_EQ(groups->size(), 1u); + EXPECT_EQ((*groups)[0].Instances()[0].Id(), 1u); +} + +TEST(CvdCreateReuseTest, MultiSlotPerInstanceNameIsolationAndMismatch) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot1 = ws.MakeArtifactsDir("slot_phone_1"); + Result slot2 = ws.MakeArtifactsDir("slot_phone_2"); + ASSERT_THAT(slot1, IsOk()); + ASSERT_THAT(slot2, IsOk()); + + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *slot1}}, + {"--reuse=true", "--base_instance_num=1"}, + {.instance_names = {{"phone"}}}), + IsOk()); + Result grp1 = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(grp1, IsOk()); + grp1->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*grp1), IsOk()); + Result mon1 = + StartMonitorSocketForInstance(ws.Root(), grp1->Instances()[0]); + ASSERT_THAT(mon1, IsOk()); + + // Second slot with the SAME `--instance_name=phone` creates its own group. + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *slot2}}, + {"--reuse=true", "--base_instance_num=2"}, + {.instance_names = {{"phone"}}}), + IsOk()); + Result> groups = + ctx.instance_manager.FindGroups({}); + ASSERT_THAT(groups, IsOk()); + EXPECT_EQ(groups->size(), 2u); + + // Explicit `--group_name=cvd_2` with mismatched `--instance_name=tablet` + // fails. + Result name_mismatch = + RunCreateNoStart(ctx.instance_manager, {{"HOME", *slot2}}, + {"--reuse=true", "--base_instance_num=2"}, + {.group_name = "cvd_2", .instance_names = {{"tablet"}}}); + ASSERT_FALSE(name_mismatch.has_value()); + EXPECT_THAT(name_mismatch.error().Message(), + HasSubstr("Trying to change the instances")); +} + +TEST(CvdCreateReuseTest, + UnrelatedActiveGroupWithDeletedCustomHomeDoesNotBreakOtherSlots) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot17 = ws.MakeArtifactsDir("acloud_cf_17"); + Result slot44 = ws.MakeArtifactsDir("acloud_cf_44"); + ASSERT_THAT(slot17, IsOk()); + ASSERT_THAT(slot44, IsOk()); + + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *slot17}}, + {"--reuse=true", "--base_instance_num=17"}), + IsOk()); + Result grp17 = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(grp17, IsOk()); + // Hold `in_progress.lock` so `grp17` stays active even after its custom HOME + // directory is deleted on disk (creating a dangling `/home` + // symlink). + Result lock17 = InstanceManager::AcquireGroupInProgressLock(*grp17); + ASSERT_THAT(lock17, IsOk()); + grp17->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(ctx.instance_manager.UpdateInstanceGroup(*grp17), IsOk()); + ASSERT_THAT(RecursivelyRemoveDirectory(*slot17), IsOk()); + + EXPECT_THAT(RunCreateNoStart(ctx.instance_manager, {{"HOME", *slot44}}, + {"--reuse=true", "--base_instance_num=44"}), + IsOk()); +} + +TEST(CvdCreateReuseTest, + DefaultHomeStoppedGroupPreservesUserdataOnPathMismatch) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result sys_home = SystemWideUserHome(); + ASSERT_THAT(sys_home, IsOk()); + Result prod_a = ws.MakeArtifactsDir("default_prod_a"); + Result prod_b = ws.MakeArtifactsDir("default_prod_b"); + ASSERT_THAT(prod_a, IsOk()); + ASSERT_THAT(prod_b, IsOk()); + + std::unordered_map env = {{"HOME", *sys_home}}; + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--host_path=" + *prod_a, + "--product_path=" + *prod_a}), + IsOk()); + + Result grp = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(grp, IsOk()); + std::string userdata_sentinel = absl::StrCat(grp->HomeDir(), "/userdata.img"); + ASSERT_THAT(WriteNewFile(userdata_sentinel, "precious-userdata"), IsOk()); + + Result mismatch = RunCreateNoStart( + ctx.instance_manager, env, + {"--reuse=true", "--host_path=" + *prod_a, "--product_path=" + *prod_b}); + ASSERT_FALSE(mismatch.has_value()); + EXPECT_THAT(mismatch.error().Message(), + HasSubstr("Trying to change the host paths")); + EXPECT_TRUE(FileExists(userdata_sentinel)); +} + +TEST(CvdCreateReuseTest, DuplicateCustomHomeAcrossGroupsIsRejectedByDatabase) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot = ws.MakeArtifactsDir("shared_slot"); + Result slot2 = ws.MakeArtifactsDir("other_slot"); + ASSERT_THAT(slot, IsOk()); + ASSERT_THAT(slot2, IsOk()); + std::unordered_map env = {{"HOME", *slot}}; + + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=false", "--base_instance_num=1"}), + IsOk()); + Result dup_home = RunCreateNoStart( + ctx.instance_manager, env, {"--reuse=false", "--base_instance_num=2"}); + ASSERT_FALSE(dup_home.has_value()); + EXPECT_THAT(dup_home.error().Message(), + HasSubstr("An instance group already exists with HOME")); + // Newly allocated lock file for failed group (`2`) must be cleaned up! + EXPECT_FALSE( + FileExists(absl::StrCat(ctx.locks_dir, "local-instance-2.lock"))); + + // Same-user duplicate instance ID (`1`) must fail without releasing group 1's + // existing `kInUse` lock! + Result dup_id = + RunCreateNoStart(ctx.instance_manager, {{"HOME", *slot2}}, + {"--reuse=false", "--base_instance_num=1"}); + ASSERT_FALSE(dup_id.has_value()); + EXPECT_TRUE(FileExists(absl::StrCat(ctx.locks_dir, "local-instance-1.lock"))); + Result lock1 = ctx.lock_manager.AcquireLock(1); + ASSERT_THAT(lock1, IsOk()); + Result status1 = lock1->Status(); + ASSERT_THAT(status1, IsOk()); + EXPECT_EQ(*status1, InUseState::kInUse); +} + +TEST(CvdCreateReuseTest, RemoveCvdRejectsGroupWhenTransitionIsInProgress) { + ScopedTestWorkspace ws; + ASSERT_TRUE(ws.Valid()); + HermeticCvdContext ctx(ws.Root()); + + Result slot = ws.MakeArtifactsDir("acloud_cf_60"); + ASSERT_THAT(slot, IsOk()); + std::unordered_map env = {{"HOME", *slot}}; + ASSERT_THAT(RunCreateNoStart(ctx.instance_manager, env, + {"--reuse=true", "--base_instance_num=60"}), + IsOk()); + + Result group = + ctx.instance_manager.FindGroup({.group_name = "cvd_1"}); + ASSERT_THAT(group, IsOk()); + Result in_progress = + InstanceManager::AcquireGroupInProgressLock(*group); + ASSERT_THAT(in_progress, IsOk()); + + Result rm_req = + CommandRequestBuilder() + .SetEnv(env) + .AddArguments({"cvd", "rm"}) + .SetSelectorOptions({.group_name = "cvd_1"}) + .Build(); + ASSERT_THAT(rm_req, IsOk()); + RemoveCvdCommandHandler rm_handler(ctx.instance_manager); + Result rm_res = rm_handler.Handle(*rm_req); + ASSERT_FALSE(rm_res.has_value()); + EXPECT_THAT(rm_res.error().Message(), + HasSubstr("Group transition is currently in progress")); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/instances/BUILD.bazel index 3f2384370c4..7c3d2ea587e 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/BUILD.bazel @@ -47,6 +47,19 @@ cf_cc_library( ], ) +cf_cc_test( + name = "data_viewer_test", + srcs = ["data_viewer_test.cpp"], + deps = [ + ":cvd_persistent_data", + ":data_viewer", + "//cuttlefish/common/libs/fs:fd", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "@abseil-cpp//absl/strings", + ], +) + cf_cc_library( name = "device_name", srcs = ["device_name.cpp"], @@ -138,6 +151,7 @@ cf_cc_library( "//cuttlefish/host/libs/log_names", "//cuttlefish/host/libs/screen_recording_controls", "//cuttlefish/io:write_exact", + "//cuttlefish/posix:realpath", "//cuttlefish/posix:symlink", "//cuttlefish/process:command", "//cuttlefish/process:managed_stdio", @@ -163,12 +177,14 @@ cf_cc_library( ":cvd_persistent_data", ":instances", "//cuttlefish/common/libs/fs", + "//cuttlefish/common/libs/fs:fd", "//cuttlefish/common/libs/utils:contains", "//cuttlefish/common/libs/utils:files", "//cuttlefish/common/libs/utils:json", "//cuttlefish/common/libs/utils:signals", "//cuttlefish/common/libs/utils:users", "//cuttlefish/files:directory_exists", + "//cuttlefish/files:file_exists", "//cuttlefish/files:recursively_remove_directory", "//cuttlefish/host/commands/cvd/cli:command_request", "//cuttlefish/host/commands/cvd/cli:utils", @@ -186,6 +202,7 @@ cf_cc_library( "//cuttlefish/host/libs/config:cuttlefish_config", "//cuttlefish/host/libs/screen_recording_controls", "//cuttlefish/metrics:device_metrics_orchestration", + "//cuttlefish/posix:realpath", "//cuttlefish/posix:remove", "//cuttlefish/posix:symlink", "//cuttlefish/process:command", @@ -215,7 +232,20 @@ cf_cc_test( cf_cc_test( name = "local_instance_test", srcs = ["local_instance_test.cpp"], - deps = ["//cuttlefish/host/commands/cvd/instances"], + deps = [ + ":cvd_persistent_data", + ":data_viewer", + "//cuttlefish/common/libs/fs:fd", + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/files:directory_exists", + "//cuttlefish/files:recursively_remove_directory", + "//cuttlefish/files:write_new_file", + "//cuttlefish/host/commands/cvd/instances", + "//cuttlefish/metrics:device_metrics_orchestration", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "@abseil-cpp//absl/strings", + ], ) cf_cc_library( diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.cpp index e7dcbcff133..b452d911e29 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.cpp @@ -16,6 +16,10 @@ #include "cuttlefish/host/commands/cvd/instances/data_viewer.h" +#include +#include +#include + #include #include @@ -23,13 +27,15 @@ #include "cuttlefish/common/libs/fs/fd.h" #include "cuttlefish/common/libs/fs/shared_fd.h" +#include "cuttlefish/common/libs/utils/signals.h" #include "cuttlefish/io/string.h" #include "cuttlefish/io/write_exact.h" namespace cuttlefish { Result DataViewer::LockBackingFile(int op) const { - Fd fd = CF_EXPECT(Fd::Open(backing_file_, O_CREAT | O_RDWR, 0640)); + Fd fd = + CF_EXPECT(Fd::Open(backing_file_, O_CREAT | O_RDWR | O_CLOEXEC, 0640)); CF_EXPECTF(fd.Flock(op), "Failed to acquire lock for instance database backing file: {}", fd.StrError()); @@ -48,6 +54,15 @@ Result DataViewer::StoreData(SharedFD fd, const cvd::PersistentData& data) { std::string str; CF_EXPECT(data.SerializeToString(&str), "Failed to serialize data"); + // Block signals while writing to the instance database file. This reduces + // the chances of corrupting the file. + sigset_t all_signals; + sigfillset(&all_signals); + SignalMasker blocker(all_signals); + // Overwrite the file contents, don't append. + CF_EXPECTF(fd->Truncate(0), "Failed to truncate fd: {}", fd->StrError()); + CF_EXPECTF(fd->LSeek(0, SEEK_SET) >= 0, "Failed to seek to 0: {}", + fd->StrError()); CF_EXPECT(WriteExact(*fd, str), "Failed to write to backing file"); return {}; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.h b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.h index 7982a4a02dc..06c36e8dd07 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/common/libs/utils/signals.h" @@ -69,26 +70,35 @@ class DataViewer { Result WithExclusiveLock( std::function(cvd::PersistentData&)> task) { DeadlockProtector dp(*this); - auto fd = CF_EXPECT(LockBackingFile(LOCK_SH)); + auto fd = CF_EXPECT(LockBackingFile(LOCK_EX)); auto data = CF_EXPECT(LoadData(fd)); auto res = task(data); if (!res.has_value()) { // Don't update if there is an error return res; } - // Block signals while writing to the instance database file. This reduces - // the chances of corrupting the file. - sigset_t all_signals; - sigfillset(&all_signals); - SignalMasker blocker(all_signals); - // Overwrite the file contents, don't append - CF_EXPECTF(fd->Truncate(0), "Failed to truncate fd: {}", fd->StrError()); - CF_EXPECTF(fd->LSeek(0, SEEK_SET) >= 0, "Failed to seek to 0: {}", - fd->StrError()); CF_EXPECT(StoreData(fd, data)); return res; } + /** + * Like `WithExclusiveLock`, except the backing file is only truncated and + * rewritten when the second element of the returned pair is `true` + * (indicating `data` was modified). + */ + template + Result WithConditionalExclusiveLock( + std::function>(cvd::PersistentData&)> task) { + DeadlockProtector dp(*this); + auto fd = CF_EXPECT(LockBackingFile(LOCK_EX)); + auto data = CF_EXPECT(LoadData(fd)); + auto [value, modified] = CF_EXPECT(task(data)); + if (modified) { + CF_EXPECT(StoreData(fd, data)); + } + return std::move(value); + } + private: // Opens and locks the backing file. The lock will be dropped when the file // descriptor closes. diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer_test.cpp new file mode 100644 index 00000000000..988284fa539 --- /dev/null +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/data_viewer_test.cpp @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/commands/cvd/instances/data_viewer.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "gmock/gmock-matchers.h" +#include "gtest/gtest.h" + +#include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/host/commands/cvd/instances/cvd_persistent_data.pb.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +class TempDbFile { + public: + TempDbFile() { + const char* tmpdir = getenv("TEST_TMPDIR"); + std::string base = + (tmpdir != nullptr && tmpdir[0] != '\0') ? tmpdir : "/tmp"; + std::string tmpl = absl::StrCat(base, "/cf_data_viewer_test.XXXXXX"); + int fd = mkstemp(tmpl.data()); + if (fd >= 0) { + close(fd); + path_ = tmpl; + } + } + + ~TempDbFile() { + if (!path_.empty()) { + unlink(path_.c_str()); + } + } + + const std::string& Path() const { return path_; } + + private: + std::string path_; +}; + +TEST(DataViewerTest, WithExclusiveLockAcquiresExclusiveFlock) { + TempDbFile db_file; + ASSERT_FALSE(db_file.Path().empty()); + DataViewer viewer(db_file.Path()); + + Result res = viewer.WithExclusiveLock( + [&db_file](cvd::PersistentData& data) -> Result { + // An independent file description must fail `LOCK_SH | LOCK_NB` while + // `WithExclusiveLock` holds `LOCK_EX`. + Fd probe = CF_EXPECT(Fd::Open(db_file.Path(), O_RDONLY | O_CLOEXEC)); + Result probe_lock = probe.Flock(LOCK_SH | LOCK_NB); + CF_EXPECT(!probe_lock.has_value(), + "Expected LOCK_SH | LOCK_NB to block against LOCK_EX"); + CF_EXPECT(*probe_lock.error() == EWOULDBLOCK || + *probe_lock.error() == EAGAIN); + data.add_instance_groups()->set_name("grp_1"); + return {}; + }); + ASSERT_THAT(res, IsOk()); + + Result count = viewer.WithSharedLock( + [](const cvd::PersistentData& data) -> Result { + return data.instance_groups_size(); + }); + ASSERT_THAT(count, IsOkAndValue(1)); +} + +TEST(DataViewerTest, ConcurrentWritersPreserveAllUpdates) { + TempDbFile db_file; + ASSERT_FALSE(db_file.Path().empty()); + + constexpr int kNumThreads = 8; + constexpr int kIterationsPerThread = 25; + std::vector threads; + threads.reserve(kNumThreads); + for (int t = 0; t < kNumThreads; ++t) { + threads.emplace_back([&db_file, t]() { + DataViewer viewer(db_file.Path()); + for (int i = 0; i < kIterationsPerThread; ++i) { + Result res = viewer.WithExclusiveLock( + [t, i](cvd::PersistentData& data) -> Result { + auto* grp = data.add_instance_groups(); + grp->set_name(absl::StrCat("grp_", t, "_", i)); + grp->set_home_directory(absl::StrCat("/tmp/home_", t, "_", i)); + return {}; + }); + EXPECT_THAT(res, IsOk()); + } + }); + } + for (std::thread& th : threads) { + th.join(); + } + + DataViewer reader(db_file.Path()); + Result total = reader.WithSharedLock( + [](const cvd::PersistentData& data) -> Result { + return data.instance_groups_size(); + }); + ASSERT_THAT(total, IsOkAndValue(kNumThreads * kIterationsPerThread)); +} + +TEST(DataViewerTest, WithConditionalExclusiveLockSkipsRewriteWhenUnmodified) { + TempDbFile db_file; + ASSERT_FALSE(db_file.Path().empty()); + DataViewer viewer(db_file.Path()); + + ASSERT_THAT(viewer.WithExclusiveLock( + [](cvd::PersistentData& data) -> Result { + data.add_instance_groups()->set_name("grp_initial"); + return {}; + }), + IsOk()); + + // Make the backing file's mtime in the past and confirm an unmodified + // `WithConditionalExclusiveLock` does not truncate or rewrite it. + struct timespec past[2] = {{.tv_sec = 1000000, .tv_nsec = 0}, + {.tv_sec = 1000000, .tv_nsec = 0}}; + ASSERT_EQ(utimensat(AT_FDCWD, db_file.Path().c_str(), past, 0), 0); + + Result noop = viewer.WithConditionalExclusiveLock( + [](cvd::PersistentData& data) -> Result> { + data.add_instance_groups()->set_name("unpersisted"); + return std::make_pair(42, false); + }); + ASSERT_THAT(noop, IsOkAndValue(42)); + + struct stat st{}; + ASSERT_EQ(stat(db_file.Path().c_str(), &st), 0); + EXPECT_EQ(st.st_mtim.tv_sec, 1000000); + + Result groups_after_noop = viewer.WithSharedLock( + [](const cvd::PersistentData& data) -> Result { + return data.instance_groups_size(); + }); + ASSERT_THAT(groups_after_noop, IsOkAndValue(1)); + + Result modified = viewer.WithConditionalExclusiveLock( + [](cvd::PersistentData& data) -> Result> { + data.add_instance_groups()->set_name("grp_second"); + return std::make_pair(99, true); + }); + ASSERT_THAT(modified, IsOkAndValue(99)); + + Result groups_after_mod = viewer.WithSharedLock( + [](const cvd::PersistentData& data) -> Result { + return data.instance_groups_size(); + }); + ASSERT_THAT(groups_after_mod, IsOkAndValue(2)); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.cpp index 5da732b86e7..6312d5ed85d 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.cpp @@ -16,6 +16,7 @@ #include "cuttlefish/host/commands/cvd/instances/instance_database.h" +#include #include #include #include @@ -33,6 +34,7 @@ #include "cuttlefish/host/commands/cvd/instances/local_instance.h" #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" #include "cuttlefish/host/libs/config/config_constants.h" +#include "cuttlefish/posix/realpath.h" namespace cuttlefish { @@ -127,7 +129,12 @@ Result InstanceDatabase::AddInstanceGroup(LocalInstanceGroup group) { CF_EXPECTF(IsValidInstanceName(instance_proto.name()), "instance_name \"{}\" is invalid", instance_proto.name()); } - auto add_res = viewer_.WithExclusiveLock([&group]( + std::optional new_real_home; + if (Result resolved = RealPath(group.HomeDir()); + resolved.has_value()) { + new_real_home = std::move(*resolved); + } + auto add_res = viewer_.WithExclusiveLock([&group, &new_real_home]( cvd::PersistentData& data) -> Result { if (group.group_proto_->name().empty()) { @@ -140,8 +147,17 @@ Result InstanceDatabase::AddInstanceGroup(LocalInstanceGroup group) { for (const auto& existing_group : data.instance_groups()) { CF_EXPECT_NE(existing_group.home_directory(), group.group_proto_->home_directory(), - "An instance group already exists with HOME directory: {}" + "An instance group already exists with HOME directory: " << group.group_proto_->home_directory()); + if (new_real_home.has_value()) { + Result existing_real_home = + RealPath(existing_group.home_directory()); + if (existing_real_home.has_value()) { + CF_EXPECT_NE(*existing_real_home, *new_real_home, + "An instance group already exists with HOME directory: " + << *new_real_home); + } + } } std::unordered_map ids_to_name_map; for (const auto& group_proto : data.instance_groups()) { @@ -186,19 +202,118 @@ Result InstanceDatabase::UpdateInstanceGroup( return {}; } -Result InstanceDatabase::RemoveInstanceGroup( - const std::string& group_name) { - return viewer_.WithExclusiveLock([&group_name]( - cvd::PersistentData& data) { - auto mutable_groups = data.mutable_instance_groups(); - for (auto it = mutable_groups->begin(); it != mutable_groups->end(); ++it) { - if (it->name() == group_name) { - mutable_groups->erase(it); - return true; - } - } +Result InstanceDatabase::BatchCompareAndUpdateInstanceStates( + const std::vector& updates) { + if (updates.empty()) { + return false; + } + return viewer_.WithConditionalExclusiveLock( + [&updates](cvd::PersistentData& data) -> Result> { + bool any_updated = false; + for (const GroupStateUpdate& update : updates) { + if (update.updated_states.empty()) { + continue; + } + const cvd::InstanceGroup& expected = update.snapshot.Proto(); + for (cvd::InstanceGroup& group_proto : + *data.mutable_instance_groups()) { + if (group_proto.name() != expected.name() || + group_proto.home_directory() != expected.home_directory() || + group_proto.start_time_sec() != expected.start_time_sec() || + group_proto.instances_size() != expected.instances_size()) { + continue; + } + bool states_match = true; + for (int i = 0; i < group_proto.instances_size(); ++i) { + const cvd::Instance& inst = group_proto.instances(i); + const cvd::Instance& expected_inst = expected.instances(i); + if (inst.id() != expected_inst.id() || + inst.state() != expected_inst.state()) { + states_match = false; + break; + } + } + if (!states_match) { + break; + } + if (update.precondition && !update.precondition()) { + break; + } + for (int i = 0; i < group_proto.instances_size(); ++i) { + cvd::Instance* inst = group_proto.mutable_instances(i); + auto state_it = update.updated_states.find(inst->id()); + if (state_it != update.updated_states.end() && + inst->state() != state_it->second) { + inst->set_state(state_it->second); + any_updated = true; + } + } + break; + } + } + return std::make_pair(any_updated, any_updated); + }); +} + +Result InstanceDatabase::CompareAndUpdateInstanceStates( + const LocalInstanceGroup& snapshot, + const std::unordered_map& updated_states, + std::function precondition) { + if (updated_states.empty()) { return false; + } + std::vector batch; + batch.push_back(GroupStateUpdate{ + .snapshot = snapshot, + .updated_states = updated_states, + .precondition = std::move(precondition), }); + return BatchCompareAndUpdateInstanceStates(batch); +} + +Result InstanceDatabase::RemoveInstanceGroup( + const std::string& group_name) { + return viewer_.WithConditionalExclusiveLock( + [&group_name]( + cvd::PersistentData& data) -> Result> { + auto mutable_groups = data.mutable_instance_groups(); + for (auto it = mutable_groups->begin(); it != mutable_groups->end(); + ++it) { + if (it->name() == group_name) { + mutable_groups->erase(it); + return std::make_pair(true, true); + } + } + return std::make_pair(false, false); + }); +} + +Result InstanceDatabase::RemoveInstanceGroup( + const LocalInstanceGroup& group, std::function precondition) { + return viewer_.WithConditionalExclusiveLock( + [&group, &precondition]( + cvd::PersistentData& data) -> Result> { + auto mutable_groups = data.mutable_instance_groups(); + for (auto it = mutable_groups->begin(); it != mutable_groups->end(); + ++it) { + if (it->name() != group.GroupName() || + it->home_directory() != group.HomeDir() || + it->start_time_sec() != group.Proto().start_time_sec()) { + continue; + } + LocalInstanceGroup existing = + CF_EXPECT(LocalInstanceGroup::Create(*it)); + CF_EXPECT(!existing.HasActiveInstances(), + "Group still contains active instances"); + if (precondition) { + CF_EXPECT(precondition() == true, + "Group transition is currently in progress"); + } + mutable_groups->erase(it); + return std::make_pair(true, true); + } + return std::make_pair(false, false); + }); } Result> InstanceDatabase::FindGroups( diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.h b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.h index 56d8dada844..fb88cfda816 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database.h @@ -18,8 +18,10 @@ #include +#include #include #include +#include #include #include #include @@ -60,9 +62,33 @@ class InstanceDatabase { */ Result AddInstanceGroup(LocalInstanceGroup group); Result UpdateInstanceGroup(const LocalInstanceGroup& group); + struct GroupStateUpdate { + LocalInstanceGroup snapshot; + std::unordered_map updated_states; + std::function precondition; + }; + + // Atomically updates instance states across one or more groups in a single + // exclusive database lock acquisition. Each entry in `updates` is applied iff + // the stored group still matches `snapshot` and satisfies `precondition`. + Result BatchCompareAndUpdateInstanceStates( + const std::vector& updates); + + // Convenience wrapper around `BatchCompareAndUpdateInstanceStates` for a + // single group. + Result CompareAndUpdateInstanceStates( + const LocalInstanceGroup& snapshot, + const std::unordered_map& updated_states, + std::function precondition = nullptr); Result> InstanceGroups() const; Result RemoveInstanceGroup(const std::string& group_name); + // Removes `group` if it still matches `group` in the database, has no active + // instances, and satisfies `precondition` (if provided) under the exclusive + // database lock. + Result RemoveInstanceGroup( + const LocalInstanceGroup& group, + std::function precondition = nullptr); /** * Empties the database and returns the recently deleted instance groups. */ diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database_test.cpp index 2fadfd891ce..0ea57228fa8 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database_test.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_database_test.cpp @@ -406,5 +406,42 @@ TEST_F(CvdInstanceDatabaseTest, UpdateInstances) { EXPECT_EQ(find_res->Instances()[1].State(), cvd::INSTANCE_STATE_STARTING); } +TEST_F(CvdInstanceDatabaseTest, CompareAndUpdateInstanceStatesUsesSnapshotCas) { + if (!SetUpOk()) { + GTEST_SKIP() << Error().msg; + } + auto& db = GetDb(); + + Result grp_res = + LocalInstanceGroup::Builder("cas_grp").AddInstance(1, "ins1").Build(); + ASSERT_THAT(grp_res, IsOk()); + ASSERT_THAT(db.AddInstanceGroup(*grp_res), IsOk()); + + Result stale_snapshot = + db.FindGroup({.group_name = "cas_grp"}); + ASSERT_THAT(stale_snapshot, IsOk()); + + // Advance state in DB to RUNNING via an independent lookup + // (`LocalInstanceGroup` shares `group_proto_` across shallow copies), then + // verify CAS against the stale PREPARING snapshot is a no-op while CAS + // against RUNNING succeeds. + Result running = db.FindGroup({.group_name = "cas_grp"}); + ASSERT_THAT(running, IsOk()); + running->SetAllStates(cvd::INSTANCE_STATE_RUNNING); + ASSERT_THAT(db.UpdateInstanceGroup(*running), IsOk()); + + ASSERT_THAT(db.CompareAndUpdateInstanceStates( + *stale_snapshot, {{1, cvd::INSTANCE_STATE_STOPPED}}), + IsOkAndValue(false)); + ASSERT_THAT(db.CompareAndUpdateInstanceStates( + *running, {{1, cvd::INSTANCE_STATE_STOPPED}}), + IsOkAndValue(true)); + + Result final_grp = + db.FindGroup({.group_name = "cas_grp"}); + ASSERT_THAT(final_grp, IsOk()); + EXPECT_EQ(final_grp->Instances()[0].State(), cvd::INSTANCE_STATE_STOPPED); +} + } // namespace selector } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.cpp index 752d6a81220..a24584599f7 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.cpp @@ -16,54 +16,278 @@ #include "cuttlefish/host/commands/cvd/instances/instance_manager.h" +#include +#include +#include +#include +#include + #include +#include #include #include #include #include +#include +#include +#include #include #include #include "absl/log/log.h" #include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" #include "android-base/file.h" #include "fmt/format.h" +#include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/common/libs/utils/files.h" #include "cuttlefish/files/directory_exists.h" +#include "cuttlefish/files/file_exists.h" #include "cuttlefish/files/recursively_remove_directory.h" #include "cuttlefish/host/commands/cvd/cli/commands/host_tool_target.h" +#include "cuttlefish/host/commands/cvd/instances/cvd_persistent_data.pb.h" #include "cuttlefish/host/commands/cvd/instances/local_instance.h" #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" #include "cuttlefish/host/commands/cvd/instances/lock/instance_lock.h" #include "cuttlefish/host/commands/cvd/instances/lock/lock_file.h" #include "cuttlefish/host/commands/cvd/instances/stop.h" #include "cuttlefish/host/commands/cvd/utils/common.h" +#include "cuttlefish/host/libs/config/config_constants.h" #include "cuttlefish/host/libs/config/config_utils.h" #include "cuttlefish/metrics/device_metrics_orchestration.h" +#include "cuttlefish/posix/realpath.h" #include "cuttlefish/posix/remove.h" #include "cuttlefish/posix/symlink.h" +#include "cuttlefish/process/proc_file_utils.h" #include "cuttlefish/result/result.h" namespace cuttlefish { namespace { -Result RemoveGroupDirectory(const LocalInstanceGroup& group) { - std::string per_user_dir = PerUserDir(); - if (!absl::StartsWith(group.HomeDir(), per_user_dir)) { +constexpr absl::string_view kGroupInProgressLockName = "in_progress.lock"; + +struct ActiveRunCvdProc { + std::string home; + std::optional real_home; + std::optional instance_id; +}; + +Result> CollectActiveRunCvdProcs() { + std::vector procs; + std::vector pids = + CF_EXPECT(CollectPidsByExecName("run_cvd", getuid())); + for (pid_t pid : pids) { + Result> envs = GetEnvs(pid); + if (!envs.has_value()) { + continue; + } + auto home_it = envs->find("HOME"); + if (home_it == envs->end() || home_it->second.empty()) { + continue; + } + ActiveRunCvdProc proc{ + .home = home_it->second, + }; + if (Result real_home = RealPath(proc.home); + real_home.has_value()) { + proc.real_home = std::move(*real_home); + } + if (auto id_it = envs->find(kCuttlefishInstanceEnvVarName); + id_it != envs->end()) { + unsigned parsed_id = 0; + if (absl::SimpleAtoi(id_it->second, &parsed_id)) { + proc.instance_id = parsed_id; + } + } + procs.push_back(std::move(proc)); + } + return procs; +} + +bool MatchesActiveRunCvd(const LocalInstanceGroup& group, + const LocalInstance& instance, + const std::vector& procs) { + std::optional real_group_home; + if (Result real_home = RealPath(group.HomeDir()); + real_home.has_value()) { + real_group_home = std::move(*real_home); + } + for (const ActiveRunCvdProc& proc : procs) { + bool home_matches = + (proc.home == group.HomeDir()) || + (real_group_home.has_value() && proc.real_home.has_value() && + *real_group_home == *proc.real_home); + if (!home_matches) { + continue; + } + if (!proc.instance_id.has_value() || *proc.instance_id == instance.Id()) { + return true; + } + } + return false; +} + +bool IsGroupBackingDirsIntact(const LocalInstanceGroup& group) { + if (!DirectoryExists(group.HomeDir())) { + return false; + } + if (!HostToolTarget(group.HostArtifactsPath()) + .GetStartBinName() + .has_value()) { + return false; + } + for (absl::string_view product_dir : + absl::StrSplit(group.ProductOutPath(), ',')) { + if (product_dir.empty() || !DirectoryExists(std::string(product_dir))) { + return false; + } + } + return true; +} + +Result> ProbeGroupStateUpdates( + const LocalInstanceGroup& group, + std::optional>& cached_procs, + bool caller_holds_in_progress_lock) { + std::unordered_map updated_states; + if (!caller_holds_in_progress_lock && + InstanceManager::IsGroupTransitionInProgress(group)) { + return updated_states; + } + + for (const LocalInstance& instance : group.Instances()) { + const cvd::InstanceState state = instance.State(); + const bool check_dead = instance.IsActive(); + const bool check_alive = state == cvd::INSTANCE_STATE_BOOT_FAILED || + state == cvd::INSTANCE_STATE_CANCELLED || + state == cvd::INSTANCE_STATE_STOPPED; + if (!check_dead && !check_alive) { + continue; + } + const auto probe_start = std::chrono::steady_clock::now(); + bool is_alive = + instance.CanConnectToLauncherMonitor(std::chrono::seconds(1)); + if (std::chrono::steady_clock::now() - probe_start > + std::chrono::milliseconds(10)) { + cached_procs.reset(); + } + if (!is_alive && state != cvd::INSTANCE_STATE_STOPPED) { + if (!cached_procs.has_value()) { + cached_procs = CF_EXPECT(CollectActiveRunCvdProcs()); + } + is_alive = MatchesActiveRunCvd(group, instance, *cached_procs); + } + if (check_dead && !is_alive) { + updated_states[instance.Id()] = cvd::INSTANCE_STATE_STOPPED; + } else if (check_alive && is_alive) { + updated_states[instance.Id()] = cvd::INSTANCE_STATE_RUNNING; + } + } + return updated_states; +} + +Result RefreshGroupWithProcs( + LocalInstanceGroup& group, + std::optional>& cached_procs, + InstanceDatabase& instance_db, bool caller_holds_in_progress_lock) { + if (caller_holds_in_progress_lock) { + std::vector reloaded = + CF_EXPECT(instance_db.FindGroups({.group_name = group.GroupName()})); + CF_EXPECTF( + reloaded.size() == 1 && reloaded.front().HomeDir() == group.HomeDir(), + "Instance group '{}' no longer exists in database", group.GroupName()); + group = std::move(reloaded.front()); + } + + std::unordered_map updated_states = + CF_EXPECT(ProbeGroupStateUpdates(group, cached_procs, + caller_holds_in_progress_lock)); + if (updated_states.empty()) { + return {}; + } + + std::function precondition; + if (!caller_holds_in_progress_lock) { + precondition = [&group]() { + return !InstanceManager::IsGroupTransitionInProgress(group); + }; + } + bool updated = CF_EXPECT(instance_db.CompareAndUpdateInstanceStates( + group, updated_states, std::move(precondition))); + if (updated) { + for (LocalInstance& instance : group.Instances()) { + if (auto it = updated_states.find(instance.Id()); + it != updated_states.end()) { + instance.SetState(it->second); + } + } + } else { + Result> reloaded = + instance_db.FindGroups({.group_name = group.GroupName()}); + if (reloaded.has_value() && reloaded->size() == 1 && + reloaded->front().HomeDir() == group.HomeDir()) { + group = std::move(reloaded->front()); + } + } + return {}; +} + +Result RemoveGroupDirectory(const LocalInstanceGroup& group, + const std::string& base_root_dir) { + std::string prefix = absl::EndsWith(base_root_dir, "/") + ? base_root_dir + : absl::StrCat(base_root_dir, "/"); + if (!absl::StartsWith(group.HomeDir(), prefix)) { LOG(WARNING) << "Instance group home directory not under user specific directory(" - << per_user_dir << "), artifacts not deleted"; + << base_root_dir << "), artifacts not deleted"; return {}; } std::string group_directory = CF_EXPECT(GroupDirFromHome(group.HomeDir())); - if (DirectoryExists(group_directory)) { + if (DirectoryExists(group_directory, /*follow_symlinks=*/false)) { CF_EXPECT(RecursivelyRemoveDirectory(group_directory), "Failed to remove group directory"); + } else if (FileExists(group_directory, /*follow_symlinks=*/false)) { + CF_EXPECT(RemoveFile(group_directory), + "Failed to remove group directory symlink"); } return {}; } +enum class ClaimBaseDirResult { + kClaimed, + kAlreadyExists, +}; + +Result TryClaimGroupBaseDir( + const std::optional& custom_base_dir, + const std::string& base_dir, const std::string& base_root_dir) { + CF_EXPECT(EnsureDirectoryExists(base_root_dir)); + if (custom_base_dir.has_value()) { + CF_EXPECT(DirectoryExists(*custom_base_dir)); + if (symlink(custom_base_dir->c_str(), base_dir.c_str()) == 0) { + return ClaimBaseDirResult::kClaimed; + } + if (errno == EEXIST) { + return ClaimBaseDirResult::kAlreadyExists; + } + return CF_ERRNO("Failed to symlink '" << *custom_base_dir << "' to '" + << base_dir << "'"); + } + if (mkdir(base_dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == 0) { + return ClaimBaseDirResult::kClaimed; + } + if (errno == EEXIST) { + return ClaimBaseDirResult::kAlreadyExists; + } + return CF_ERRNO("Failed to create group base directory '" << base_dir << "'"); +} + Result LinkOrMakeDir(std::optional target, const std::string& path) { if (target.has_value()) { @@ -76,11 +300,9 @@ Result LinkOrMakeDir(std::optional target, return {}; } -Result CreateOrLinkGroupDirectories( +Result PopulateGroupSubdirectories( const LocalInstanceGroup& group, InstanceManager::GroupDirectories directories) { - CF_EXPECT( - LinkOrMakeDir(std::move(directories.base_directory), group.BaseDir())); CF_EXPECT(LinkOrMakeDir(std::move(directories.home), group.HomeDir())); CF_EXPECT(EnsureDirectoryExists(group.ArtifactsDir())); CF_EXPECT(LinkOrMakeDir(std::move(directories.host_artifacts_path), @@ -100,7 +322,64 @@ Result StopBin(const std::string& host_android_out) { InstanceManager::InstanceManager(InstanceLockFileManager& lock_manager, InstanceDatabase& instance_db) - : lock_manager_(lock_manager), instance_db_(instance_db) {} + : InstanceManager(lock_manager, instance_db, PerUserDir()) {} + +InstanceManager::InstanceManager(InstanceLockFileManager& lock_manager, + InstanceDatabase& instance_db, + std::string base_root_dir) + : lock_manager_(lock_manager), + instance_db_(instance_db), + base_root_dir_(std::move(base_root_dir)) {} + +Result InstanceManager::AcquireGroupInProgressLock( + const LocalInstanceGroup& group) { + CF_EXPECTF(DirectoryExists(group.BaseDir()), + "Instance group base directory '{}' does not exist", + group.BaseDir()); + std::string lock_path = + fmt::format("{}/{}", group.BaseDir(), kGroupInProgressLockName); + SharedFD fd = + CF_EXPECT(Fd::Open(lock_path, O_CREAT | O_RDWR | O_CLOEXEC, 0600)); + constexpr int kMaxLockRetries = 50; + Result flock_res = fd->Flock(LOCK_EX | LOCK_NB); + for (int retry = 0; + retry < kMaxLockRetries && !flock_res.has_value() && + (*flock_res.error() == EWOULDBLOCK || *flock_res.error() == EAGAIN); + ++retry) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + flock_res = fd->Flock(LOCK_EX | LOCK_NB); + } + if (!flock_res.has_value() && + (*flock_res.error() == EWOULDBLOCK || *flock_res.error() == EAGAIN)) { + return CF_ERR( + "Selected instance group is already starting, use `cvd create` to " + "create a new one."); + } + CF_EXPECT(std::move(flock_res)); + return fd; +} + +bool InstanceManager::IsGroupTransitionInProgress( + const LocalInstanceGroup& group) { + std::string lock_path = + fmt::format("{}/{}", group.BaseDir(), kGroupInProgressLockName); + if (!FileExists(lock_path)) { + return false; + } + Result fd = Fd::Open(lock_path, O_RDONLY | O_CLOEXEC); + if (!fd.has_value()) { + return false; + } + Result flock_res = fd->Flock(LOCK_SH | LOCK_NB); + if (!flock_res.has_value() && + (*flock_res.error() == EWOULDBLOCK || *flock_res.error() == EAGAIN)) { + return true; + } + if (flock_res.has_value()) { + (void)fd->Flock(LOCK_UN | LOCK_NB); + } + return false; +} Result> InstanceManager::FindInstanceWithGroup( @@ -121,36 +400,41 @@ Result> InstanceManager::AllocateAndLockInstanceIds( std::vector instances) { std::set requested; - std::vector requested_lock_files; - // Acquire requested locks first for (const auto& instance : instances) { if (instance.instance_id.has_value()) { unsigned id = instance.instance_id.value(); auto [_, inserted] = requested.insert(id); - // This check avoids a possible deadlock when trying to acquire the lock a - // second time CF_EXPECTF(std::move(inserted), "Requested instance ids must be distinct, but {} is repeated", id); - requested_lock_files.emplace_back( - CF_EXPECT(lock_manager_.AcquireLock(id))); } } + // Acquire requested locks in ascending ID order to avoid ABBA deadlocks. + std::unordered_map> + requested_lock_files; + requested_lock_files.reserve(requested.size()); + for (unsigned id : requested) { + InstanceLockFile lock = CF_EXPECT(lock_manager_.AcquireCurrentUserLock(id)); + bool was_unused = (CF_EXPECT(lock.Status()) == InUseState::kNotInUse); + requested_lock_files.emplace(id, + std::make_pair(std::move(lock), was_unused)); + } std::vector ret; ret.reserve(instances.size()); - auto requested_it = requested_lock_files.begin(); for (auto& instance : instances) { if (instance.instance_id.has_value()) { - CF_EXPECT(requested_it != requested_lock_files.end()); + auto lock_it = requested_lock_files.find(*instance.instance_id); + CF_EXPECT(lock_it != requested_lock_files.end()); ret.emplace_back(InternalInstanceDesc{ - .lock_file = std::move(*requested_it), + .lock_file = std::move(lock_it->second.first), .name = std::move(instance.per_instance_name), + .release_on_error = lock_it->second.second, }); - ++requested_it; } else { ret.emplace_back(InternalInstanceDesc{ .lock_file = CF_EXPECT(lock_manager_.AcquireUnusedLock()), .name = std::move(instance.per_instance_name), + .release_on_error = true, }); } } @@ -158,55 +442,210 @@ InstanceManager::AllocateAndLockInstanceIds( } Result InstanceManager::CreateInstanceGroup( - InstanceGroupParams group_params, GroupDirectories directories) { + InstanceGroupParams group_params, GroupDirectories directories, + SharedFD* out_in_progress_lock) { CF_EXPECT_EQ( group_params.instances.size(), directories.product_out_paths.size(), "Number of product directories doesn't match number of instances"); std::vector instance_descs = CF_EXPECT(AllocateAndLockInstanceIds(std::move(group_params.instances))); + auto release_unused_locks = [this, &instance_descs]() { + for (InternalInstanceDesc& desc : instance_descs) { + if (desc.release_on_error) { + (void)lock_manager_.ReleaseHeldLock(desc.lock_file); + } + } + }; + + constexpr int kMaxBaseDirAttempts = 1024; + std::optional built_group; + for (int attempt = 0; attempt < kMaxBaseDirAttempts; ++attempt) { + LocalInstanceGroup::Builder group_builder(group_params.group_name, + base_root_dir_); + for (const auto& instance_desc : instance_descs) { + if (instance_desc.name.has_value()) { + group_builder.AddInstance(instance_desc.lock_file.Instance(), + instance_desc.name.value()); + } else { + group_builder.AddInstance(instance_desc.lock_file.Instance()); + } + } + Result build_res = group_builder.Build(); + if (!build_res.has_value()) { + release_unused_locks(); + } + LocalInstanceGroup candidate = CF_EXPECT(std::move(build_res)); - LocalInstanceGroup::Builder group_builder(std::move(group_params.group_name)); - for (const auto& instance_desc : instance_descs) { - if (instance_desc.name.has_value()) { - group_builder.AddInstance(instance_desc.lock_file.Instance(), - instance_desc.name.value()); - } else { - group_builder.AddInstance(instance_desc.lock_file.Instance()); + Result claim_res = TryClaimGroupBaseDir( + directories.base_directory, candidate.BaseDir(), base_root_dir_); + if (!claim_res.has_value()) { + release_unused_locks(); } + if (CF_EXPECT(std::move(claim_res)) == ClaimBaseDirResult::kClaimed) { + built_group.emplace(std::move(candidate)); + break; + } + } + if (!built_group.has_value()) { + release_unused_locks(); + return CF_ERR("Failed to allocate unique group base directory"); } - LocalInstanceGroup group = CF_EXPECT(group_builder.Build()); + LocalInstanceGroup group = std::move(*built_group); + + auto cleanup_group_and_locks = [this, &group, &release_unused_locks]() { + (void)RemoveGroupDirectory(group, base_root_dir_); + release_unused_locks(); + }; // The base and other directories are always set to the default location, if // the user provides custom directories the ones in the default locations // become symbolic links to those. - CF_EXPECT(CreateOrLinkGroupDirectories(group, std::move(directories))); + Result dirs_res = + PopulateGroupSubdirectories(group, std::move(directories)); + if (!dirs_res.has_value()) { + cleanup_group_and_locks(); + } + CF_EXPECT(std::move(dirs_res)); - CF_EXPECT(instance_db_.AddInstanceGroup(group)); + SharedFD in_progress_lock; + if (out_in_progress_lock != nullptr) { + Result lock_res = AcquireGroupInProgressLock(group); + if (!lock_res.has_value()) { + cleanup_group_and_locks(); + } + in_progress_lock = CF_EXPECT(std::move(lock_res)); + } + + Result add_res = instance_db_.AddInstanceGroup(group); + if (!add_res.has_value()) { + in_progress_lock = SharedFD(); + cleanup_group_and_locks(); + } + CF_EXPECT(std::move(add_res)); for (auto& instance_desc : instance_descs) { CF_EXPECT(instance_desc.lock_file.Status(InUseState::kInUse)); } + if (out_in_progress_lock != nullptr) { + *out_in_progress_lock = std::move(in_progress_lock); + } GatherVmInstantiationMetrics(group); return group; } +Result InstanceManager::EnsureInstanceLocksInUse( + const LocalInstanceGroup& group) { + std::set sorted_ids; + for (const LocalInstance& instance : group.Instances()) { + if (instance.Id() != 0) { + sorted_ids.insert(instance.Id()); + } + } + std::vector locks; + locks.reserve(sorted_ids.size()); + for (unsigned id : sorted_ids) { + locks.push_back(CF_EXPECT(lock_manager_.AcquireCurrentUserLock(id))); + } + for (InstanceLockFile& lock : locks) { + CF_EXPECT(lock.Status(InUseState::kInUse)); + } + return {}; +} + Result InstanceManager::RemoveInstanceGroup(LocalInstanceGroup group) { CF_EXPECT(!group.HasActiveInstances(), "Group still contains active instances"); - for (auto& instance : group.Instances()) { - if (instance.Id() == 0) { - continue; + CF_EXPECT(!IsGroupTransitionInProgress(group), + "Group transition is currently in progress"); + std::set sorted_ids; + for (const LocalInstance& instance : group.Instances()) { + if (instance.Id() != 0) { + sorted_ids.insert(instance.Id()); + } + } + std::vector held_locks; + held_locks.reserve(sorted_ids.size()); + for (unsigned id : sorted_ids) { + std::optional lock_opt = + CF_EXPECT(lock_manager_.AcquireOwnedLockIfPresent(id)); + if (lock_opt.has_value()) { + held_locks.emplace_back(std::move(*lock_opt)); } - if (auto res = lock_manager_.RemoveLockFile(instance.Id()); + } + + bool removed = CF_EXPECT(instance_db_.RemoveInstanceGroup( + group, [&group]() { return !IsGroupTransitionInProgress(group); })); + if (!removed) { + return false; + } + + for (InstanceLockFile& lock : held_locks) { + if (Result res = lock_manager_.ReleaseHeldLock(lock); !res.has_value()) { - LOG(ERROR) << "Failed to remove instance id lock: " << res.error(); + LOG(ERROR) << "Failed to release instance id lock " << lock.Instance() + << ": " << res.error(); + } + } + CF_EXPECT(RemoveGroupDirectory(group, base_root_dir_)); + return true; +} + +Result InstanceManager::RefreshInstanceGroupState( + LocalInstanceGroup group, bool caller_holds_in_progress_lock) { + std::optional> cached_procs; + CF_EXPECT(RefreshGroupWithProcs(group, cached_procs, instance_db_, + caller_holds_in_progress_lock)); + return group; +} + +Result InstanceManager::RefreshInstanceGroups() { + std::vector groups = + CF_EXPECT(instance_db_.FindGroups({})); + std::optional> cached_procs; + std::vector batch_updates; + for (const LocalInstanceGroup& group : groups) { + std::unordered_map updates = CF_EXPECT( + ProbeGroupStateUpdates(group, cached_procs, + /*caller_holds_in_progress_lock=*/false)); + if (!updates.empty()) { + batch_updates.push_back(InstanceDatabase::GroupStateUpdate{ + .snapshot = group, + .updated_states = std::move(updates), + .precondition = + [group]() { + return !InstanceManager::IsGroupTransitionInProgress(group); + }, + }); } } - CF_EXPECT(RemoveGroupDirectory(group)); + if (!batch_updates.empty()) { + (void)CF_EXPECT( + instance_db_.BatchCompareAndUpdateInstanceStates(batch_updates)); + } + return {}; +} - return CF_EXPECT(instance_db_.RemoveInstanceGroup(group.GroupName())); +Result InstanceManager::PruneBrokenInstanceGroups() { + CF_EXPECT(RefreshInstanceGroups()); + std::vector groups = + CF_EXPECT(instance_db_.FindGroups({})); + for (LocalInstanceGroup& group : groups) { + if (group.HasActiveInstances()) { + continue; + } + if (!IsGroupBackingDirsIntact(group)) { + LOG(INFO) << "Pruning inactive instance group '" << group.GroupName() + << "' with missing backing directories"; + Result remove_res = RemoveInstanceGroup(std::move(group)); + if (!remove_res.has_value()) { + LOG(WARNING) << "Failed to prune broken instance group: " + << remove_res.error(); + } + } + } + return {}; } Result InstanceManager::UpdateInstanceGroup( @@ -263,6 +702,11 @@ Result InstanceManager::StopInstanceGroup( } Result InstanceManager::Clear() { + if (Result refresh_res = RefreshInstanceGroups(); + !refresh_res.has_value()) { + LOG(WARNING) << "Failed to refresh instance groups before clear: " + << refresh_res.error(); + } const std::string config_json_name = android::base::Basename(GetGlobalConfigFileLink()); auto instance_groups = @@ -280,9 +724,9 @@ Result InstanceManager::Clear() { if (instance.Id() <= 0) { continue; } - auto res = lock_manager_.RemoveLockFile(instance.Id()); + auto res = lock_manager_.ReleaseInstanceLock(instance.Id()); if (!res.has_value()) { - LOG(ERROR) << "Failed to remove lock file for instance: " + LOG(ERROR) << "Failed to release lock file for instance: " << res.error(); } } @@ -295,7 +739,7 @@ Result InstanceManager::Clear() { LOG(ERROR) << res.error(); } // TODO: b/471069557 - diagnose unused - Result unused = RemoveGroupDirectory(group); + Result unused = RemoveGroupDirectory(group, base_root_dir_); } std::cerr << "Stopped all known instances\n"; return {}; diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.h b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.h index 8e68c678bc4..c6358f79378 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/instance_manager.h @@ -24,6 +24,7 @@ #include #include +#include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/host/commands/cvd/instances/instance_database.h" #include "cuttlefish/host/commands/cvd/instances/local_instance.h" #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" @@ -56,12 +57,39 @@ class InstanceManager { std::vector> product_out_paths; }; InstanceManager(InstanceLockFileManager&, InstanceDatabase& instance_db); + InstanceManager(InstanceLockFileManager&, InstanceDatabase& instance_db, + std::string base_root_dir); + + // Acquires an exclusive non-blocking flock on `/in_progress.lock` + // to guard an in-progress prepare or boot transition against concurrent + // state refreshes and duplicate starts. + static Result AcquireGroupInProgressLock( + const LocalInstanceGroup& group); + // Returns true if another open file description currently holds an exclusive + // flock on `/in_progress.lock`. + static bool IsGroupTransitionInProgress(const LocalInstanceGroup& group); Result HasInstanceGroups() const; + // Creates and registers an instance group. If `out_in_progress_lock` is + // non-null, acquires `in_progress.lock` before inserting the group into + // `instance_db_` so no unlocked `PREPARING`/`STOPPED` window exists. Result CreateInstanceGroup( - InstanceGroupParams group_params, GroupDirectories group_directories); + InstanceGroupParams group_params, GroupDirectories group_directories, + SharedFD* out_in_progress_lock = nullptr); Result UpdateInstanceGroup(const LocalInstanceGroup& group); Result RemoveInstanceGroup(LocalInstanceGroup group); + // Verifies that all instance ID lock files for `group` are not owned by + // another user and marks them `InUseState::kInUse`. + Result EnsureInstanceLocksInUse(const LocalInstanceGroup& group); + // Reconciles `group`'s persisted states against live launcher monitor sockets + // and `run_cvd` processes. Set `caller_holds_in_progress_lock` to true when + // the caller already holds `AcquireGroupInProgressLock(group)`. + Result RefreshInstanceGroupState( + LocalInstanceGroup group, bool caller_holds_in_progress_lock = false); + Result RefreshInstanceGroups(); + // Refreshes all groups and removes inactive groups whose backing directories + // (`HomeDir`, `HostArtifactsPath`, or `ProductOutPath`) no longer exist. + Result PruneBrokenInstanceGroups(); // Stops and removes all known instance instance groups Result Clear(); @@ -91,6 +119,7 @@ class InstanceManager { struct InternalInstanceDesc { InstanceLockFile lock_file; std::optional name; + bool release_on_error = false; }; Result> AllocateAndLockInstanceIds( @@ -98,6 +127,7 @@ class InstanceManager { InstanceLockFileManager& lock_manager_; InstanceDatabase& instance_db_; + std::string base_root_dir_; }; } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.cpp index 359be9c31da..4b388854b8b 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -115,6 +116,11 @@ bool LocalInstance::IsActive() const { return false; } +bool LocalInstance::CanConnectToLauncherMonitor( + std::chrono::seconds timeout) const { + return GetLauncherMonitor(timeout).has_value(); +} + Result LocalInstance::FetchStatus(std::chrono::seconds timeout) { return CF_EXPECT(FetchInstanceStatus(*this, timeout)); } @@ -278,13 +284,18 @@ Result LocalInstance::GetLauncherMonitor( // under /tmp, and store this path in the config. Older instances just put // them in the instance directory. std::string uds_dir = InstanceDirectory(); - Json::Value config = CF_EXPECT(ReadJsonConfig()); - if (config.isMember("instances_uds_dir") && - config["instances_uds_dir"].isString()) { - uds_dir = - fmt::format("{}/cvd-{}", config["instances_uds_dir"].asString(), Id()); + if (Result config = ReadJsonConfig(); + config.has_value() && config->isMember("instances_uds_dir") && + (*config)["instances_uds_dir"].isString()) { + uds_dir = fmt::format("{}/cvd-{}", + (*config)["instances_uds_dir"].asString(), Id()); } std::string monitor_path = uds_dir + "/launcher_monitor.sock"; + CF_EXPECTF(FileExists(monitor_path), + "Launcher monitor socket ({}) does not exist", monitor_path); + CF_EXPECTF(monitor_path.size() < sizeof(sockaddr_un::sun_path), + "Launcher monitor socket path ({}) exceeds AF_UNIX limit", + monitor_path); SharedFD monitor = SharedFD::SocketLocalClient(monitor_path, false, SOCK_STREAM, timeout.count()); CF_EXPECTF(monitor->IsOpen(), diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.h b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.h index da97d991761..d5048f6a8b8 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance.h @@ -60,6 +60,8 @@ class LocalInstance { std::string AssemblyDirectory() const; bool IsActive() const; + bool CanConnectToLauncherMonitor( + std::chrono::seconds timeout = std::chrono::seconds(1)) const; // Contacts run_cvd to query the instance status. Returns a JSON object with // a description of the instance properties. Waits for run_cvd to respond for // at most timeout seconds. diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.cpp index 9a8e17039e4..b40efc8526c 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.cpp @@ -17,6 +17,8 @@ #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" #include +#include +#include #include #include #include @@ -47,9 +49,17 @@ std::vector Filter( return ret; } -std::string DefaultBaseDir() { - auto time = std::chrono::system_clock::now().time_since_epoch().count(); - return fmt::format("{}/{}", PerUserDir(), time); +std::string DefaultBaseDir(const std::string& base_root_dir) { + static std::atomic last_time{0}; + const int64_t now = + std::chrono::system_clock::now().time_since_epoch().count(); + int64_t prev = last_time.load(std::memory_order_relaxed); + int64_t chosen = std::max(now, prev + 1); + while (!last_time.compare_exchange_weak(prev, chosen, + std::memory_order_relaxed)) { + chosen = std::max(now, prev + 1); + } + return fmt::format("{}/{}", base_root_dir, chosen); } std::string HomeDirFromBase(const std::string& base_dir) { @@ -94,7 +104,11 @@ Result LocalInstanceGroup::Create( } LocalInstanceGroup::Builder::Builder(std::string group_name) - : base_dir_(DefaultBaseDir()) { + : Builder(std::move(group_name), PerUserDir()) {} + +LocalInstanceGroup::Builder::Builder(std::string group_name, + const std::string& base_root_dir) + : base_dir_(DefaultBaseDir(base_root_dir)) { group_proto_.set_name(std::move(group_name)); group_proto_.set_home_directory(HomeDirFromBase(base_dir_)); group_proto_.set_host_artifacts_path(HostArtifactsDirFromBase(base_dir_)); diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.h b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.h index b18d05218cb..f30d7aa47dd 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group.h @@ -33,7 +33,8 @@ class LocalInstanceGroup { public: class Builder { public: - Builder(std::string group_name); + explicit Builder(std::string group_name); + Builder(std::string group_name, const std::string& base_root_dir); Builder& AddInstance(unsigned id) &; Builder& AddInstance(unsigned id, std::string name) &; diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group_test.cpp index aba929d5239..6e021957351 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group_test.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_group_test.cpp @@ -82,5 +82,35 @@ TEST_F(LocalInstanceGroupUnitTest, SearchById) { } } +TEST_F(LocalInstanceGroupUnitTest, + BuilderProducesDistinctPureDigitBaseDirBasenamesForPodcvd) { + Result g1 = + LocalInstanceGroup::Builder("grp1", "/var/tmp/cvd/0") + .AddInstance(1) + .Build(); + Result g2 = + LocalInstanceGroup::Builder("grp2", "/var/tmp/cvd/0") + .AddInstance(2) + .Build(); + ASSERT_THAT(g1, IsOk()); + ASSERT_THAT(g2, IsOk()); + EXPECT_NE(g1->BaseDir(), g2->BaseDir()); + + // `podcvd` translates `/var/tmp/cvd//` via + // `/var/tmp/cvd/[0-9]+/[0-9]+`, so the basename must consist strictly of + // ASCII digits with no underscores or non-numeric suffixes. + for (const LocalInstanceGroup& grp : {*g1, *g2}) { + const std::string base = grp.BaseDir(); + const std::string prefix = "/var/tmp/cvd/0/"; + ASSERT_EQ(base.rfind(prefix, 0), 0u); + const std::string leaf = base.substr(prefix.size()); + ASSERT_FALSE(leaf.empty()); + for (char ch : leaf) { + EXPECT_GE(ch, '0'); + EXPECT_LE(ch, '9'); + } + } +} + } // namespace selector } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_test.cpp index 42fdfa5976f..1b9501a2e83 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_test.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/local_instance_test.cpp @@ -13,12 +13,83 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "cuttlefish/host/commands/cvd/instances/local_instance.h" + +#include +#include + +#include +#include + +#include "absl/strings/str_cat.h" +#include "gmock/gmock-matchers.h" #include "gtest/gtest.h" +#include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/files/directory_exists.h" +#include "cuttlefish/files/recursively_remove_directory.h" +#include "cuttlefish/files/write_new_file.h" +#include "cuttlefish/host/commands/cvd/instances/cvd_persistent_data.pb.h" +#include "cuttlefish/host/commands/cvd/instances/data_viewer.h" +#include "cuttlefish/host/commands/cvd/instances/instance_database.h" #include "cuttlefish/host/commands/cvd/instances/local_instance_group.h" +#include "cuttlefish/metrics/device_metrics_orchestration.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" namespace cuttlefish { namespace selector { +namespace { + +class ScopedShortTempDir { + public: + ScopedShortTempDir() { + std::string tmpl = "/tmp/cf_inst_test.XXXXXX"; + char* created = mkdtemp(tmpl.data()); + if (created != nullptr) { + root_ = created; + setenv("CUTTLEFISH_METRICS_TRANSMITTER_PATH", + absl::StrCat(root_, "/no_metrics_transmitter").c_str(), 1); + setenv("XDG_DATA_HOME", absl::StrCat(root_, "/xdg_data").c_str(), 1); + } + } + + ~ScopedShortTempDir() { + if (!root_.empty()) { + (void)RecursivelyRemoveDirectory(root_); + } + } + + const std::string& Root() const { return root_; } + + private: + std::string root_; +}; + +Result BuildGroupWithHome(const std::string& db_path, + const std::string& name, + const std::string& home_dir, + unsigned id) { + DataViewer viewer(db_path); + CF_EXPECT(viewer.WithExclusiveLock( + [&](cvd::PersistentData& data) -> Result { + cvd::InstanceGroup* grp = data.add_instance_groups(); + grp->set_name(name); + grp->set_home_directory(home_dir); + grp->set_host_artifacts_path(home_dir); + grp->set_product_out_path(home_dir); + cvd::Instance* inst = grp->add_instances(); + inst->set_id(id); + inst->set_name(absl::StrCat(id)); + inst->set_state(cvd::INSTANCE_STATE_RUNNING); + return {}; + })); + InstanceDatabase db(db_path); + return CF_EXPECT(db.FindGroup({.group_name = name})); +} + +} // namespace /** * Note that invalid inputs must be tested at the InstanceDatabase level @@ -27,15 +98,6 @@ TEST(LocalinstanceTest, Fields) { auto parent_group_res = LocalInstanceGroup::Builder("super").AddInstance(3, "phone").Build(); if (!parent_group_res.has_value()) { - /* - * Here's why we skip the test rather than see it as a failure. - * - * 1. The test is specifically designed for operations in - * LocalInstanceRecord. - * 2. Adding instance to a group is tested in another test suites designed - * for LocalInstanceGroup. It's a failure there but not here. - * - */ GTEST_SKIP() << "Failed to add instance group. Set up failed."; } const auto& parent_group = *parent_group_res; @@ -47,5 +109,73 @@ TEST(LocalinstanceTest, Fields) { EXPECT_EQ(parent_group.Proto().name(), "super"); } +TEST(LocalinstanceTest, CanConnectToLauncherMonitorHandlesMissingAndLongPaths) { + ScopedShortTempDir tmp; + ASSERT_FALSE(tmp.Root().empty()); + + std::string long_home = absl::StrCat( + tmp.Root(), + "/very_long_home_directory_exceeding_unix_domain_socket_sun_path_" + "limit_0123456789abcdef"); + std::string fallback_inst_dir = + absl::StrCat(long_home, "/cuttlefish/instances/cvd-7"); + ASSERT_THAT(EnsureDirectoryExists(fallback_inst_dir), IsOk()); + std::string long_sock = + absl::StrCat(fallback_inst_dir, "/launcher_monitor.sock"); + ASSERT_GE(long_sock.size(), sizeof(sockaddr_un::sun_path)); + ASSERT_THAT(WriteNewFile(long_sock, ""), IsOk()); + + Result group = BuildGroupWithHome( + absl::StrCat(tmp.Root(), "/db.binpb"), "grp_long", long_home, 7); + ASSERT_THAT(group, IsOk()); + EXPECT_FALSE(group->Instances()[0].CanConnectToLauncherMonitor( + std::chrono::seconds(1))); + + // Now point `.cuttlefish_config.json` at a short `instances_uds_dir` with a + // live listening AF_UNIX socket and verify `CanConnectToLauncherMonitor` + // succeeds. + std::string uds_root = absl::StrCat(tmp.Root(), "/uds"); + std::string cvd_uds = absl::StrCat(uds_root, "/cvd-7"); + ASSERT_THAT(EnsureDirectoryExists(cvd_uds), IsOk()); + ASSERT_THAT( + WriteNewFile(absl::StrCat(long_home, "/.cuttlefish_config.json"), + absl::StrCat("{\"instances_uds_dir\":\"", uds_root, "\"}")), + IsOk()); + std::string live_sock_path = absl::StrCat(cvd_uds, "/launcher_monitor.sock"); + Result server_sock = Fd::SocketLocalServer( + live_sock_path, /*is_abstract=*/false, SOCK_STREAM, 0600); + ASSERT_THAT(server_sock, IsOk()); + + EXPECT_TRUE(group->Instances()[0].CanConnectToLauncherMonitor( + std::chrono::seconds(1))); +} + +TEST(LocalinstanceTest, MetricsSetupRecreatesWipedDirectoryWithoutAborting) { + ScopedShortTempDir tmp; + ASSERT_FALSE(tmp.Root().empty()); + + std::string home = absl::StrCat(tmp.Root(), "/slot_home"); + ASSERT_THAT(EnsureDirectoryExists(home), IsOk()); + Result group = BuildGroupWithHome( + absl::StrCat(tmp.Root(), "/metrics_db.binpb"), "grp_metrics", home, 1); + ASSERT_THAT(group, IsOk()); + + // Even when `AreMetricsEnabled()` is false, `GatherVmInstantiationMetrics` + // creates `/metrics`. + GatherVmInstantiationMetrics(*group); + EXPECT_TRUE(DirectoryExists(group->MetricsDir())); + + // Simulate a caller wiping `$HOME/*` between runs: `GatherVmStartMetrics` + // recreates `/metrics` without crashing, and `GatherVmStopMetrics` + // does not abort if `/metrics` was deleted mid-run. + ASSERT_THAT(RecursivelyRemoveDirectory(group->MetricsDir()), IsOk()); + EXPECT_FALSE(DirectoryExists(group->MetricsDir())); + GatherVmStartMetrics(*group); + EXPECT_TRUE(DirectoryExists(group->MetricsDir())); + + ASSERT_THAT(RecursivelyRemoveDirectory(group->MetricsDir()), IsOk()); + GatherVmStopMetrics(*group); +} + } // namespace selector } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel index 875163b687e..7eabe3615a3 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel @@ -1,5 +1,5 @@ load("//:build_variables.bzl", "COPTS") -load("//cuttlefish/bazel:rules.bzl", "cf_cc_library") +load("//cuttlefish/bazel:rules.bzl", "cf_cc_library", "cf_cc_test") package( default_visibility = ["//:android_cuttlefish"], @@ -33,3 +33,16 @@ cf_cc_library( "@android_system_libbase//:file_logging", ], ) + +cf_cc_test( + name = "instance_lock_test", + srcs = ["instance_lock_test.cpp"], + deps = [ + ":lock", + "//cuttlefish/files:file_exists", + "//cuttlefish/files:recursively_remove_directory", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "@abseil-cpp//absl/strings", + ], +) diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp index 773e25c6a0d..8a0dce2e3cd 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp @@ -17,6 +17,7 @@ #include "cuttlefish/host/commands/cvd/instances/lock/instance_lock.h" #include +#include #include #include @@ -36,6 +37,10 @@ InstanceLockFile::InstanceLockFile(LockFile&& lock_file, unsigned InstanceLockFile::Instance() const { return instance_num_; } +const std::string& InstanceLockFile::LockFilePath() const { + return lock_file_.LockFilePath(); +} + Result InstanceLockFile::Status() const { auto in_use_state = CF_EXPECT(lock_file_.Status()); return in_use_state; @@ -46,6 +51,10 @@ Result InstanceLockFile::Status(InUseState state) { return {}; } +Result InstanceLockFile::OwnerUid() const { + return CF_EXPECT(lock_file_.OwnerUid()); +} + InstanceLockFileManager::InstanceLockFileManager( std::string instance_locks_path) : instance_locks_path_(std::move(instance_locks_path)) {}; @@ -59,18 +68,97 @@ Result InstanceLockFileManager::LockFilePath( return path.str(); } +Result> +InstanceLockFileManager::AcquireOwnedLockIfPresent(unsigned instance_num) { + const auto lock_file_path = CF_EXPECT(LockFilePath(instance_num)); + Result path_owner = FileOwner(lock_file_path); + if (!path_owner.has_value() || *path_owner != geteuid()) { + return std::nullopt; + } + InstanceLockFile lock = CF_EXPECT(AcquireLock(instance_num)); + uid_t fd_owner = CF_EXPECT(lock.OwnerUid()); + if (fd_owner != geteuid()) { + return std::nullopt; + } + return lock; +} + +Result InstanceLockFileManager::ReleaseHeldLock(InstanceLockFile& lock) { + uid_t fd_owner = CF_EXPECT(lock.OwnerUid()); + if (fd_owner != geteuid()) { + return {}; + } + CF_EXPECT(lock.Status(InUseState::kNotInUse)); + CF_EXPECT(RemoveFile(lock.LockFilePath()), StrError()); + return {}; +} + +Result InstanceLockFileManager::ReleaseInstanceLock( + unsigned instance_num) { + std::optional lock = + CF_EXPECT(AcquireOwnedLockIfPresent(instance_num)); + if (!lock.has_value()) { + return {}; + } + CF_EXPECT(ReleaseHeldLock(*lock)); + return {}; +} + Result InstanceLockFileManager::RemoveLockFile(unsigned instance_num) { const auto lock_file_path = CF_EXPECT(LockFilePath(instance_num)); CF_EXPECT(RemoveFile(lock_file_path), StrError()); return {}; } +Result InstanceLockFileManager::AcquireCurrentUserLock( + unsigned instance_num) { + constexpr int kMaxAttempts = 16; + for (int attempt = 0; attempt < kMaxAttempts; ++attempt) { + InstanceLockFile lock = CF_EXPECT(AcquireLock(instance_num)); + InUseState status = CF_EXPECT(lock.Status()); + uid_t owner = CF_EXPECT(lock.OwnerUid()); + if (owner == geteuid()) { + return lock; + } + if (status == InUseState::kInUse) { + return CF_ERRF( + "Instance ID {} is currently in use by another user (uid {})", + instance_num, owner); + } + // Stale `kNotInUse` file owned by another UID: unlink while holding + // `LOCK_EX` so the next iteration creates a fresh inode owned by geteuid(). + CF_EXPECTF(RemoveFile(lock.LockFilePath()), + "Failed to replace stale lock file '{}': {}", + lock.LockFilePath(), StrError()); + } + return CF_ERRF("Failed to acquire instance lock {} for current user", + instance_num); +} + Result InstanceLockFileManager::AcquireUnusedLock() { for (unsigned i = 1;; i++) { auto lock = CF_EXPECT(TryAcquireLock(i)); - if (lock && CF_EXPECT(lock->Status()) == InUseState::kNotInUse) { + if (!lock) { + continue; + } + if (CF_EXPECT(lock->Status()) != InUseState::kNotInUse) { + continue; + } + if (CF_EXPECT(lock->OwnerUid()) == geteuid()) { return *lock; } + // Replace stale `kNotInUse` inode owned by another UID so `st_uid` matches + // `geteuid()`. If unlinking fails (e.g. sticky custom dir), skip to i + 1. + if (!RemoveFile(lock->LockFilePath()).has_value()) { + continue; + } + lock.reset(); + auto fresh_lock = CF_EXPECT(TryAcquireLock(i)); + if (fresh_lock && + CF_EXPECT(fresh_lock->Status()) == InUseState::kNotInUse && + CF_EXPECT(fresh_lock->OwnerUid()) == geteuid()) { + return *fresh_lock; + } } } diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h index 721dd188050..05ab4b263cf 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h @@ -15,6 +15,8 @@ */ #pragma once +#include + #include #include @@ -28,8 +30,10 @@ class InstanceLockFile { public: unsigned Instance() const; + const std::string& LockFilePath() const; Result Status() const; Result Status(InUseState); + Result OwnerUid() const; private: InstanceLockFile(LockFile&& lock_file, unsigned instance_num); @@ -42,7 +46,24 @@ class InstanceLockFileManager { InstanceLockFileManager(std::string instance_locks_path); Result AcquireLock(unsigned instance_num); + // Acquires `instance_num` for the current user (`geteuid()`): fails if + // marked `kInUse` by another UID, and replaces any stale `kNotInUse` inode + // owned by another UID with a fresh inode owned by `geteuid()`. + Result AcquireCurrentUserLock(unsigned instance_num); Result AcquireUnusedLock(); + Result> TryAcquireLock(unsigned instance_num); + + // Acquires `instance_num` if its lock file exists and is owned by `geteuid()` + // so the caller can release it while holding the lock. + Result> AcquireOwnedLockIfPresent( + unsigned instance_num); + + // Resets and removes an acquired lock file while holding its exclusive flock. + Result ReleaseHeldLock(InstanceLockFile& lock); + + // Releases `instance_num` if its lock file exists and is owned by + // `geteuid()`. + Result ReleaseInstanceLock(unsigned instance_num); // TODO: This routine should be removed and replaced with allocd // The caller must check if the instance_num belongs to the user, before @@ -51,7 +72,6 @@ class InstanceLockFileManager { private: Result LockFilePath(unsigned instance_num); - Result> TryAcquireLock(unsigned instance_num); std::string instance_locks_path_; LockFileManager lock_file_manager_; diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock_test.cpp new file mode 100644 index 00000000000..0d18e9a30f5 --- /dev/null +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock_test.cpp @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/commands/cvd/instances/lock/instance_lock.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "gmock/gmock-matchers.h" +#include "gtest/gtest.h" + +#include "cuttlefish/files/file_exists.h" +#include "cuttlefish/files/recursively_remove_directory.h" +#include "cuttlefish/host/commands/cvd/instances/lock/lock_file.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +class TempLocksDir { + public: + TempLocksDir() { + const char* tmpdir = getenv("TEST_TMPDIR"); + std::string base = + (tmpdir != nullptr && tmpdir[0] != '\0') ? tmpdir : "/tmp"; + std::string tmpl = absl::StrCat(base, "/cf_lock_test.XXXXXX"); + char* created = mkdtemp(tmpl.data()); + if (created != nullptr) { + dir_ = absl::StrCat(created, "/"); + } + } + + ~TempLocksDir() { + if (!dir_.empty()) { + (void)RecursivelyRemoveDirectory(dir_); + } + } + + const std::string& Dir() const { return dir_; } + + private: + std::string dir_; +}; + +TEST(InstanceLockTest, AcquireCurrentUserAndReleaseHeldLock) { + TempLocksDir tmp; + ASSERT_FALSE(tmp.Dir().empty()); + InstanceLockFileManager mgr(tmp.Dir()); + + Result lock = mgr.AcquireCurrentUserLock(42); + ASSERT_THAT(lock, IsOk()); + EXPECT_EQ(lock->Instance(), 42u); + ASSERT_THAT(lock->OwnerUid(), IsOkAndValue(geteuid())); + ASSERT_THAT(lock->Status(), IsOkAndValue(InUseState::kNotInUse)); + ASSERT_THAT(lock->Status(InUseState::kInUse), IsOk()); + + // Non-blocking `TryAcquireLock` from another manager fails while held. + InstanceLockFileManager other_mgr(tmp.Dir()); + Result> contended = + other_mgr.TryAcquireLock(42); + ASSERT_THAT(contended, IsOk()); + EXPECT_FALSE(contended->has_value()); + + std::string lock_path = lock->LockFilePath(); + EXPECT_TRUE(FileExists(lock_path)); + ASSERT_THAT(mgr.ReleaseHeldLock(*lock), IsOk()); + EXPECT_FALSE(FileExists(lock_path)); +} + +TEST(InstanceLockTest, ConcurrentAcquireAndReleaseNeverSplitBrain) { + TempLocksDir tmp; + ASSERT_FALSE(tmp.Dir().empty()); + + constexpr int kNumThreads = 8; + constexpr int kIterations = 40; + std::atomic active_holders{0}; + std::atomic overlap_detected{false}; + + std::vector threads; + threads.reserve(kNumThreads); + for (int t = 0; t < kNumThreads; ++t) { + threads.emplace_back([&]() { + InstanceLockFileManager mgr(tmp.Dir()); + for (int i = 0; i < kIterations; ++i) { + Result lock = mgr.AcquireCurrentUserLock(7); + ASSERT_THAT(lock, IsOk()); + int prev = active_holders.fetch_add(1, std::memory_order_seq_cst); + if (prev != 0) { + overlap_detected.store(true, std::memory_order_seq_cst); + } + std::this_thread::yield(); + active_holders.fetch_sub(1, std::memory_order_seq_cst); + if ((i % 2) == 0) { + EXPECT_THAT(mgr.ReleaseHeldLock(*lock), IsOk()); + } + } + }); + } + for (std::thread& th : threads) { + th.join(); + } + + EXPECT_FALSE(overlap_detected.load()); +} + +TEST(InstanceLockTest, AcquireUnusedLockSkipsInUseIds) { + TempLocksDir tmp; + ASSERT_FALSE(tmp.Dir().empty()); + InstanceLockFileManager mgr(tmp.Dir()); + + { + // Drop flock at scope exit while leaving `1` marked `kInUse`. + Result lock1 = mgr.AcquireCurrentUserLock(1); + ASSERT_THAT(lock1, IsOk()); + ASSERT_THAT(lock1->Status(InUseState::kInUse), IsOk()); + } + + Result unused = mgr.AcquireUnusedLock(); + ASSERT_THAT(unused, IsOk()); + EXPECT_EQ(unused->Instance(), 2u); + ASSERT_THAT(mgr.ReleaseInstanceLock(1), IsOk()); + EXPECT_FALSE(FileExists(absl::StrCat(tmp.Dir(), "local-instance-1.lock"))); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp index 4066f6425f4..32bd1d68cbe 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp @@ -16,7 +16,11 @@ #include "cuttlefish/host/commands/cvd/instances/lock/lock_file.h" +#include +#include #include +#include +#include #include #include @@ -87,6 +91,23 @@ Result LockFile::Status(InUseState state) { return {}; } +Result LockFile::OwnerUid() const { + return CF_EXPECT(fd_->Fstat()).st_uid; +} + +static bool IsSameInode(const SharedFD& fd, const std::string& file_path) { + Result fd_stat = fd->Fstat(); + if (!fd_stat.has_value()) { + return false; + } + struct stat path_stat{}; + if (stat(file_path.c_str(), &path_stat) != 0) { + return false; + } + return fd_stat->st_dev == path_stat.st_dev && + fd_stat->st_ino == path_stat.st_ino; +} + bool LockFile::operator<(const LockFile& other) const { if (this == std::addressof(other)) { return false; @@ -103,35 +124,75 @@ bool LockFile::operator<(const LockFile& other) const { Result LockFileManager::OpenLockFile(const std::string& file_path) { auto parent_dir = android::base::Dirname(file_path); CF_EXPECT(EnsureDirectoryExists(parent_dir, 0777)); - int result = chmod(parent_dir.c_str(), 0777); - if (result) { - VLOG(0) << "failed: chmod 777 " << parent_dir; + if (Result dir_owner = FileOwner(parent_dir); + dir_owner.has_value() && *dir_owner == geteuid()) { + int result = chmod(parent_dir.c_str(), 0777); + if (result != 0) { + VLOG(0) << "failed: chmod 777 " << parent_dir; + } } - Fd fd = CF_EXPECT(Fd::Open(file_path.data(), O_CREAT | O_RDWR, 0666)); - result = chmod(file_path.c_str(), 0666); - if (result) { - VLOG(0) << "failed: chmod 666 " << file_path; + + // Open existing lock files WITHOUT `O_CREAT` first so Linux + // `fs.protected_regular=2` (`may_create_in_sticky`) does not reject opening a + // `0666` file owned by another UID inside the `0777` shared lock directory. + constexpr int kMaxOpenAttempts = 16; + for (int attempt = 0; attempt < kMaxOpenAttempts; ++attempt) { + Result existing_fd = + Fd::Open(file_path.data(), O_RDWR | O_CLOEXEC); + if (existing_fd.has_value()) { + return std::move(*existing_fd); + } + if (*existing_fd.error() != ENOENT) { + return CF_EXPECT(std::move(existing_fd)); + } + + Result created_fd = + Fd::Open(file_path.data(), O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0666); + if (created_fd.has_value()) { + if (!created_fd->Chmod(0666)) { + VLOG(0) << "failed: fchmod 666 " << file_path; + } + return std::move(*created_fd); + } + if (*created_fd.error() == EEXIST) { + continue; + } + return CF_EXPECT(std::move(created_fd)); } - return fd; + return CF_ERRF("Failed to open or create lock file '{}' after {} attempts", + file_path, kMaxOpenAttempts); } Result LockFileManager::AcquireLock( const std::string& lock_file_path) { - SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); - CF_EXPECT(fd->Flock(LOCK_EX)); - return LockFile(fd, lock_file_path); + while (true) { + SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); + CF_EXPECT(fd->Flock(LOCK_EX)); + if (IsSameInode(fd, lock_file_path)) { + return LockFile(fd, lock_file_path); + } + (void)fd->Flock(LOCK_UN | LOCK_NB); + } } Result> LockFileManager::TryAcquireLock( const std::string& lock_file_path) { - SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); - Result flock_result = fd->Flock(LOCK_EX | LOCK_NB); - if (flock_result) { - return std::optional(LockFile(fd, lock_file_path)); - } else if (!flock_result && *flock_result.error() == EWOULDBLOCK) { + for (int attempt = 0; attempt < 8; ++attempt) { + SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); + Result flock_result = fd->Flock(LOCK_EX | LOCK_NB); + if (flock_result) { + if (IsSameInode(fd, lock_file_path)) { + return std::optional(LockFile(fd, lock_file_path)); + } + (void)fd->Flock(LOCK_UN | LOCK_NB); + continue; + } else if (*flock_result.error() == EWOULDBLOCK || + *flock_result.error() == EAGAIN) { + return {}; + } + CF_EXPECT(std::move(flock_result)); return {}; } - CF_EXPECT(std::move(flock_result)); return {}; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h index c7af9e58853..ef34394f99e 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h @@ -15,6 +15,8 @@ */ #pragma once +#include + #include #include #include @@ -66,6 +68,7 @@ class LockFile { const auto& LockFilePath() const { return lock_file_path_; } Result Status() const; Result Status(InUseState); + Result OwnerUid() const; // to put this into a set bool operator<(const LockFile& other) const; diff --git a/base/cvd/cuttlefish/metrics/BUILD.bazel b/base/cvd/cuttlefish/metrics/BUILD.bazel index 048269d03a5..2c6b45cc0de 100644 --- a/base/cvd/cuttlefish/metrics/BUILD.bazel +++ b/base/cvd/cuttlefish/metrics/BUILD.bazel @@ -19,6 +19,7 @@ cf_cc_library( "//cuttlefish/metrics/guest:device_event_type", "//cuttlefish/metrics/guest:guest_metrics", "//cuttlefish/metrics/guest:parsed_flags", + "//cuttlefish/metrics/io:enabled", "//cuttlefish/metrics/io:notification", "//cuttlefish/result", "@abseil-cpp//absl/log", diff --git a/base/cvd/cuttlefish/metrics/device_metrics_orchestration.cc b/base/cvd/cuttlefish/metrics/device_metrics_orchestration.cc index 22890a77b3f..5c81ab28907 100644 --- a/base/cvd/cuttlefish/metrics/device_metrics_orchestration.cc +++ b/base/cvd/cuttlefish/metrics/device_metrics_orchestration.cc @@ -31,6 +31,7 @@ #include "cuttlefish/metrics/guest/device_event_type.h" #include "cuttlefish/metrics/guest/guest_metrics.h" #include "cuttlefish/metrics/guest/parsed_flags.h" +#include "cuttlefish/metrics/io/enabled.h" #include "cuttlefish/metrics/io/notification.h" #include "cuttlefish/metrics/metrics_orchestration.h" #include "cuttlefish/result/result.h" @@ -94,12 +95,15 @@ Result GatherDeviceMetrics( } Result RunMetrics(const DeviceMetricsInput& metrics_input) { - ScopedLogger logger = - CreateLogger(metrics_input.base_input.metrics_directory); CF_EXPECTF(FileExists(metrics_input.base_input.metrics_directory), "Metrics directory({}) does not exist. Perhaps metrics were not " "initialized?", metrics_input.base_input.metrics_directory); + ScopedLogger logger = + CreateLogger(metrics_input.base_input.metrics_directory); + if (!AreMetricsEnabled()) { + return {}; + } const std::string event_type_label = DeviceEventTypeString(metrics_input.guests.event_type); @@ -117,6 +121,15 @@ Result RunMetrics(const DeviceMetricsInput& metrics_input) { void GatherVmInstantiationMetrics(const LocalInstanceGroup& instance_group) { DisplayPrivacyNotice(); + Result metrics_setup_result = SetUpMetrics(instance_group.MetricsDir()); + if (!metrics_setup_result.has_value()) { + VLOG(0) << fmt::format("Failed to initialize metrics. Error: {}", + metrics_setup_result.error().FormatForEnv()); + return; + } + if (!AreMetricsEnabled()) { + return; + } Result metrics_input_result = GetDeviceMetricsInput( instance_group, DeviceEventType::DeviceInstantiation); if (!metrics_input_result.has_value()) { @@ -124,13 +137,6 @@ void GatherVmInstantiationMetrics(const LocalInstanceGroup& instance_group) { metrics_input_result.error().FormatForEnv()); return; } - Result metrics_setup_result = - SetUpMetrics(metrics_input_result->base_input.metrics_directory); - if (!metrics_setup_result.has_value()) { - VLOG(0) << fmt::format("Failed to initialize metrics. Error: {}", - metrics_setup_result.error().FormatForEnv()); - return; - } Result run_metrics_result = RunMetrics(*metrics_input_result); if (!run_metrics_result.has_value()) { VLOG(0) << fmt::format( @@ -141,6 +147,15 @@ void GatherVmInstantiationMetrics(const LocalInstanceGroup& instance_group) { } void GatherVmStartMetrics(const LocalInstanceGroup& instance_group) { + Result metrics_setup_result = SetUpMetrics(instance_group.MetricsDir()); + if (!metrics_setup_result.has_value()) { + VLOG(0) << fmt::format("Failed to initialize metrics. Error: {}", + metrics_setup_result.error().FormatForEnv()); + return; + } + if (!AreMetricsEnabled()) { + return; + } Result metrics_input_result = GetDeviceMetricsInput(instance_group, DeviceEventType::DeviceBootStart); if (!metrics_input_result.has_value()) { @@ -158,6 +173,9 @@ void GatherVmStartMetrics(const LocalInstanceGroup& instance_group) { } void GatherVmBootCompleteMetrics(const LocalInstanceGroup& instance_group) { + if (!AreMetricsEnabled()) { + return; + } Result metrics_input_result = GetDeviceMetricsInput( instance_group, DeviceEventType::DeviceBootComplete); if (!metrics_input_result.has_value()) { @@ -175,6 +193,9 @@ void GatherVmBootCompleteMetrics(const LocalInstanceGroup& instance_group) { } void GatherVmBootFailedMetrics(const LocalInstanceGroup& instance_group) { + if (!AreMetricsEnabled()) { + return; + } Result metrics_input_result = GetDeviceMetricsInput(instance_group, DeviceEventType::DeviceBootFailed); if (!metrics_input_result.has_value()) { @@ -192,6 +213,9 @@ void GatherVmBootFailedMetrics(const LocalInstanceGroup& instance_group) { } void GatherVmStopMetrics(const LocalInstanceGroup& instance_group) { + if (!AreMetricsEnabled()) { + return; + } Result metrics_input_result = GetDeviceMetricsInput(instance_group, DeviceEventType::DeviceStop); if (!metrics_input_result.has_value()) {