From cbc55290ced93b9177e2b0ecc74f635db1e38ce7 Mon Sep 17 00:00:00 2001 From: Ram Muthiah Date: Tue, 17 Nov 2026 18:10:16 +0000 Subject: [PATCH] cvd: Harden shared instance locks for multi-user hosts and concurrency Follow up on 2545596ec0f1 (`cvd: Use 0777 permissions on shared instance locks directory`) to fix three multi-user and concurrency hazards on shared `/var/tmp/cvd/lock/local-instance-.lock` files: 1. `LockFileManager::OpenLockFile` called `open(O_CREAT | O_RDWR, 0666)` even when the lock file already existed. On kernels with `fs.protected_regular = 2` (default on Debian 13), opening an existing regular file owned by another UID inside a world-writable (`0777`) directory with `O_CREAT` without `O_EXCL` fails with `EACCES`. Open existing lock files with `O_RDWR | O_CLOEXEC` first, and only fall back to `O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC` (using `fchmod(0666)` on the newly created fd and skipping `chmod(parent, 0777)` when `parent_dir` is owned by another UID). 2. `AcquireLock` and `TryAcquireLock` did not re-check that the locked fd still matched the path's inode (`fstat` vs `stat`), allowing a concurrent `RemoveLockFile` / `DeleteLockFile` to unlink a locked inode and let a second caller create and lock a distinct inode for the same instance ID simultaneously. Retry if `IsSameInode` fails after `flock(LOCK_EX)`. 3. Add `AcquireCurrentUserLock`, `AcquireOwnedLockIfPresent`, `ReleaseHeldLock`, and `ReleaseInstanceLock` on `InstanceLockFileManager`, and update `AcquireUnusedLock` to reject active (`kInUse`) locks held by other UIDs while replacing stale (`kNotInUse`) foreign-UID lock files with fresh inodes owned by `geteuid()` so subsequent `fchmod` status updates cannot fail with `EPERM`. Add `instance_lock_test` (`instances/lock/BUILD.bazel`) covering `O_CLOEXEC`, concurrent `AcquireLock` + `RemoveLockFile` inode races, and `AcquireCurrentUserLock` / `ReleaseInstanceLock` semantics. Bug: b/554100008 Bug: b/584825999 Bug: b/586523782 Test: bazel test //cuttlefish/host/commands/cvd/instances/lock/... --- .../commands/cvd/instances/lock/BUILD.bazel | 15 +- .../cvd/instances/lock/instance_lock.cpp | 90 ++++++++++- .../cvd/instances/lock/instance_lock.h | 22 ++- .../cvd/instances/lock/instance_lock_test.cpp | 147 ++++++++++++++++++ .../commands/cvd/instances/lock/lock_file.cpp | 95 +++++++++-- .../commands/cvd/instances/lock/lock_file.h | 3 + 6 files changed, 352 insertions(+), 20 deletions(-) create mode 100644 base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock_test.cpp diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel index 875163b687e..7eabe3615a3 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/BUILD.bazel @@ -1,5 +1,5 @@ load("//:build_variables.bzl", "COPTS") -load("//cuttlefish/bazel:rules.bzl", "cf_cc_library") +load("//cuttlefish/bazel:rules.bzl", "cf_cc_library", "cf_cc_test") package( default_visibility = ["//:android_cuttlefish"], @@ -33,3 +33,16 @@ cf_cc_library( "@android_system_libbase//:file_logging", ], ) + +cf_cc_test( + name = "instance_lock_test", + srcs = ["instance_lock_test.cpp"], + deps = [ + ":lock", + "//cuttlefish/files:file_exists", + "//cuttlefish/files:recursively_remove_directory", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "@abseil-cpp//absl/strings", + ], +) diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp index 773e25c6a0d..8a0dce2e3cd 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.cpp @@ -17,6 +17,7 @@ #include "cuttlefish/host/commands/cvd/instances/lock/instance_lock.h" #include +#include #include #include @@ -36,6 +37,10 @@ InstanceLockFile::InstanceLockFile(LockFile&& lock_file, unsigned InstanceLockFile::Instance() const { return instance_num_; } +const std::string& InstanceLockFile::LockFilePath() const { + return lock_file_.LockFilePath(); +} + Result InstanceLockFile::Status() const { auto in_use_state = CF_EXPECT(lock_file_.Status()); return in_use_state; @@ -46,6 +51,10 @@ Result InstanceLockFile::Status(InUseState state) { return {}; } +Result InstanceLockFile::OwnerUid() const { + return CF_EXPECT(lock_file_.OwnerUid()); +} + InstanceLockFileManager::InstanceLockFileManager( std::string instance_locks_path) : instance_locks_path_(std::move(instance_locks_path)) {}; @@ -59,18 +68,97 @@ Result InstanceLockFileManager::LockFilePath( return path.str(); } +Result> +InstanceLockFileManager::AcquireOwnedLockIfPresent(unsigned instance_num) { + const auto lock_file_path = CF_EXPECT(LockFilePath(instance_num)); + Result path_owner = FileOwner(lock_file_path); + if (!path_owner.has_value() || *path_owner != geteuid()) { + return std::nullopt; + } + InstanceLockFile lock = CF_EXPECT(AcquireLock(instance_num)); + uid_t fd_owner = CF_EXPECT(lock.OwnerUid()); + if (fd_owner != geteuid()) { + return std::nullopt; + } + return lock; +} + +Result InstanceLockFileManager::ReleaseHeldLock(InstanceLockFile& lock) { + uid_t fd_owner = CF_EXPECT(lock.OwnerUid()); + if (fd_owner != geteuid()) { + return {}; + } + CF_EXPECT(lock.Status(InUseState::kNotInUse)); + CF_EXPECT(RemoveFile(lock.LockFilePath()), StrError()); + return {}; +} + +Result InstanceLockFileManager::ReleaseInstanceLock( + unsigned instance_num) { + std::optional lock = + CF_EXPECT(AcquireOwnedLockIfPresent(instance_num)); + if (!lock.has_value()) { + return {}; + } + CF_EXPECT(ReleaseHeldLock(*lock)); + return {}; +} + Result InstanceLockFileManager::RemoveLockFile(unsigned instance_num) { const auto lock_file_path = CF_EXPECT(LockFilePath(instance_num)); CF_EXPECT(RemoveFile(lock_file_path), StrError()); return {}; } +Result InstanceLockFileManager::AcquireCurrentUserLock( + unsigned instance_num) { + constexpr int kMaxAttempts = 16; + for (int attempt = 0; attempt < kMaxAttempts; ++attempt) { + InstanceLockFile lock = CF_EXPECT(AcquireLock(instance_num)); + InUseState status = CF_EXPECT(lock.Status()); + uid_t owner = CF_EXPECT(lock.OwnerUid()); + if (owner == geteuid()) { + return lock; + } + if (status == InUseState::kInUse) { + return CF_ERRF( + "Instance ID {} is currently in use by another user (uid {})", + instance_num, owner); + } + // Stale `kNotInUse` file owned by another UID: unlink while holding + // `LOCK_EX` so the next iteration creates a fresh inode owned by geteuid(). + CF_EXPECTF(RemoveFile(lock.LockFilePath()), + "Failed to replace stale lock file '{}': {}", + lock.LockFilePath(), StrError()); + } + return CF_ERRF("Failed to acquire instance lock {} for current user", + instance_num); +} + Result InstanceLockFileManager::AcquireUnusedLock() { for (unsigned i = 1;; i++) { auto lock = CF_EXPECT(TryAcquireLock(i)); - if (lock && CF_EXPECT(lock->Status()) == InUseState::kNotInUse) { + if (!lock) { + continue; + } + if (CF_EXPECT(lock->Status()) != InUseState::kNotInUse) { + continue; + } + if (CF_EXPECT(lock->OwnerUid()) == geteuid()) { return *lock; } + // Replace stale `kNotInUse` inode owned by another UID so `st_uid` matches + // `geteuid()`. If unlinking fails (e.g. sticky custom dir), skip to i + 1. + if (!RemoveFile(lock->LockFilePath()).has_value()) { + continue; + } + lock.reset(); + auto fresh_lock = CF_EXPECT(TryAcquireLock(i)); + if (fresh_lock && + CF_EXPECT(fresh_lock->Status()) == InUseState::kNotInUse && + CF_EXPECT(fresh_lock->OwnerUid()) == geteuid()) { + return *fresh_lock; + } } } diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h index 721dd188050..05ab4b263cf 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock.h @@ -15,6 +15,8 @@ */ #pragma once +#include + #include #include @@ -28,8 +30,10 @@ class InstanceLockFile { public: unsigned Instance() const; + const std::string& LockFilePath() const; Result Status() const; Result Status(InUseState); + Result OwnerUid() const; private: InstanceLockFile(LockFile&& lock_file, unsigned instance_num); @@ -42,7 +46,24 @@ class InstanceLockFileManager { InstanceLockFileManager(std::string instance_locks_path); Result AcquireLock(unsigned instance_num); + // Acquires `instance_num` for the current user (`geteuid()`): fails if + // marked `kInUse` by another UID, and replaces any stale `kNotInUse` inode + // owned by another UID with a fresh inode owned by `geteuid()`. + Result AcquireCurrentUserLock(unsigned instance_num); Result AcquireUnusedLock(); + Result> TryAcquireLock(unsigned instance_num); + + // Acquires `instance_num` if its lock file exists and is owned by `geteuid()` + // so the caller can release it while holding the lock. + Result> AcquireOwnedLockIfPresent( + unsigned instance_num); + + // Resets and removes an acquired lock file while holding its exclusive flock. + Result ReleaseHeldLock(InstanceLockFile& lock); + + // Releases `instance_num` if its lock file exists and is owned by + // `geteuid()`. + Result ReleaseInstanceLock(unsigned instance_num); // TODO: This routine should be removed and replaced with allocd // The caller must check if the instance_num belongs to the user, before @@ -51,7 +72,6 @@ class InstanceLockFileManager { private: Result LockFilePath(unsigned instance_num); - Result> TryAcquireLock(unsigned instance_num); std::string instance_locks_path_; LockFileManager lock_file_manager_; diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock_test.cpp new file mode 100644 index 00000000000..0d18e9a30f5 --- /dev/null +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/instance_lock_test.cpp @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/commands/cvd/instances/lock/instance_lock.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "gmock/gmock-matchers.h" +#include "gtest/gtest.h" + +#include "cuttlefish/files/file_exists.h" +#include "cuttlefish/files/recursively_remove_directory.h" +#include "cuttlefish/host/commands/cvd/instances/lock/lock_file.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +class TempLocksDir { + public: + TempLocksDir() { + const char* tmpdir = getenv("TEST_TMPDIR"); + std::string base = + (tmpdir != nullptr && tmpdir[0] != '\0') ? tmpdir : "/tmp"; + std::string tmpl = absl::StrCat(base, "/cf_lock_test.XXXXXX"); + char* created = mkdtemp(tmpl.data()); + if (created != nullptr) { + dir_ = absl::StrCat(created, "/"); + } + } + + ~TempLocksDir() { + if (!dir_.empty()) { + (void)RecursivelyRemoveDirectory(dir_); + } + } + + const std::string& Dir() const { return dir_; } + + private: + std::string dir_; +}; + +TEST(InstanceLockTest, AcquireCurrentUserAndReleaseHeldLock) { + TempLocksDir tmp; + ASSERT_FALSE(tmp.Dir().empty()); + InstanceLockFileManager mgr(tmp.Dir()); + + Result lock = mgr.AcquireCurrentUserLock(42); + ASSERT_THAT(lock, IsOk()); + EXPECT_EQ(lock->Instance(), 42u); + ASSERT_THAT(lock->OwnerUid(), IsOkAndValue(geteuid())); + ASSERT_THAT(lock->Status(), IsOkAndValue(InUseState::kNotInUse)); + ASSERT_THAT(lock->Status(InUseState::kInUse), IsOk()); + + // Non-blocking `TryAcquireLock` from another manager fails while held. + InstanceLockFileManager other_mgr(tmp.Dir()); + Result> contended = + other_mgr.TryAcquireLock(42); + ASSERT_THAT(contended, IsOk()); + EXPECT_FALSE(contended->has_value()); + + std::string lock_path = lock->LockFilePath(); + EXPECT_TRUE(FileExists(lock_path)); + ASSERT_THAT(mgr.ReleaseHeldLock(*lock), IsOk()); + EXPECT_FALSE(FileExists(lock_path)); +} + +TEST(InstanceLockTest, ConcurrentAcquireAndReleaseNeverSplitBrain) { + TempLocksDir tmp; + ASSERT_FALSE(tmp.Dir().empty()); + + constexpr int kNumThreads = 8; + constexpr int kIterations = 40; + std::atomic active_holders{0}; + std::atomic overlap_detected{false}; + + std::vector threads; + threads.reserve(kNumThreads); + for (int t = 0; t < kNumThreads; ++t) { + threads.emplace_back([&]() { + InstanceLockFileManager mgr(tmp.Dir()); + for (int i = 0; i < kIterations; ++i) { + Result lock = mgr.AcquireCurrentUserLock(7); + ASSERT_THAT(lock, IsOk()); + int prev = active_holders.fetch_add(1, std::memory_order_seq_cst); + if (prev != 0) { + overlap_detected.store(true, std::memory_order_seq_cst); + } + std::this_thread::yield(); + active_holders.fetch_sub(1, std::memory_order_seq_cst); + if ((i % 2) == 0) { + EXPECT_THAT(mgr.ReleaseHeldLock(*lock), IsOk()); + } + } + }); + } + for (std::thread& th : threads) { + th.join(); + } + + EXPECT_FALSE(overlap_detected.load()); +} + +TEST(InstanceLockTest, AcquireUnusedLockSkipsInUseIds) { + TempLocksDir tmp; + ASSERT_FALSE(tmp.Dir().empty()); + InstanceLockFileManager mgr(tmp.Dir()); + + { + // Drop flock at scope exit while leaving `1` marked `kInUse`. + Result lock1 = mgr.AcquireCurrentUserLock(1); + ASSERT_THAT(lock1, IsOk()); + ASSERT_THAT(lock1->Status(InUseState::kInUse), IsOk()); + } + + Result unused = mgr.AcquireUnusedLock(); + ASSERT_THAT(unused, IsOk()); + EXPECT_EQ(unused->Instance(), 2u); + ASSERT_THAT(mgr.ReleaseInstanceLock(1), IsOk()); + EXPECT_FALSE(FileExists(absl::StrCat(tmp.Dir(), "local-instance-1.lock"))); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp index 4066f6425f4..32bd1d68cbe 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.cpp @@ -16,7 +16,11 @@ #include "cuttlefish/host/commands/cvd/instances/lock/lock_file.h" +#include +#include #include +#include +#include #include #include @@ -87,6 +91,23 @@ Result LockFile::Status(InUseState state) { return {}; } +Result LockFile::OwnerUid() const { + return CF_EXPECT(fd_->Fstat()).st_uid; +} + +static bool IsSameInode(const SharedFD& fd, const std::string& file_path) { + Result fd_stat = fd->Fstat(); + if (!fd_stat.has_value()) { + return false; + } + struct stat path_stat{}; + if (stat(file_path.c_str(), &path_stat) != 0) { + return false; + } + return fd_stat->st_dev == path_stat.st_dev && + fd_stat->st_ino == path_stat.st_ino; +} + bool LockFile::operator<(const LockFile& other) const { if (this == std::addressof(other)) { return false; @@ -103,35 +124,75 @@ bool LockFile::operator<(const LockFile& other) const { Result LockFileManager::OpenLockFile(const std::string& file_path) { auto parent_dir = android::base::Dirname(file_path); CF_EXPECT(EnsureDirectoryExists(parent_dir, 0777)); - int result = chmod(parent_dir.c_str(), 0777); - if (result) { - VLOG(0) << "failed: chmod 777 " << parent_dir; + if (Result dir_owner = FileOwner(parent_dir); + dir_owner.has_value() && *dir_owner == geteuid()) { + int result = chmod(parent_dir.c_str(), 0777); + if (result != 0) { + VLOG(0) << "failed: chmod 777 " << parent_dir; + } } - Fd fd = CF_EXPECT(Fd::Open(file_path.data(), O_CREAT | O_RDWR, 0666)); - result = chmod(file_path.c_str(), 0666); - if (result) { - VLOG(0) << "failed: chmod 666 " << file_path; + + // Open existing lock files WITHOUT `O_CREAT` first so Linux + // `fs.protected_regular=2` (`may_create_in_sticky`) does not reject opening a + // `0666` file owned by another UID inside the `0777` shared lock directory. + constexpr int kMaxOpenAttempts = 16; + for (int attempt = 0; attempt < kMaxOpenAttempts; ++attempt) { + Result existing_fd = + Fd::Open(file_path.data(), O_RDWR | O_CLOEXEC); + if (existing_fd.has_value()) { + return std::move(*existing_fd); + } + if (*existing_fd.error() != ENOENT) { + return CF_EXPECT(std::move(existing_fd)); + } + + Result created_fd = + Fd::Open(file_path.data(), O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0666); + if (created_fd.has_value()) { + if (!created_fd->Chmod(0666)) { + VLOG(0) << "failed: fchmod 666 " << file_path; + } + return std::move(*created_fd); + } + if (*created_fd.error() == EEXIST) { + continue; + } + return CF_EXPECT(std::move(created_fd)); } - return fd; + return CF_ERRF("Failed to open or create lock file '{}' after {} attempts", + file_path, kMaxOpenAttempts); } Result LockFileManager::AcquireLock( const std::string& lock_file_path) { - SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); - CF_EXPECT(fd->Flock(LOCK_EX)); - return LockFile(fd, lock_file_path); + while (true) { + SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); + CF_EXPECT(fd->Flock(LOCK_EX)); + if (IsSameInode(fd, lock_file_path)) { + return LockFile(fd, lock_file_path); + } + (void)fd->Flock(LOCK_UN | LOCK_NB); + } } Result> LockFileManager::TryAcquireLock( const std::string& lock_file_path) { - SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); - Result flock_result = fd->Flock(LOCK_EX | LOCK_NB); - if (flock_result) { - return std::optional(LockFile(fd, lock_file_path)); - } else if (!flock_result && *flock_result.error() == EWOULDBLOCK) { + for (int attempt = 0; attempt < 8; ++attempt) { + SharedFD fd = CF_EXPECT(OpenLockFile(lock_file_path)); + Result flock_result = fd->Flock(LOCK_EX | LOCK_NB); + if (flock_result) { + if (IsSameInode(fd, lock_file_path)) { + return std::optional(LockFile(fd, lock_file_path)); + } + (void)fd->Flock(LOCK_UN | LOCK_NB); + continue; + } else if (*flock_result.error() == EWOULDBLOCK || + *flock_result.error() == EAGAIN) { + return {}; + } + CF_EXPECT(std::move(flock_result)); return {}; } - CF_EXPECT(std::move(flock_result)); return {}; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h index c7af9e58853..ef34394f99e 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h +++ b/base/cvd/cuttlefish/host/commands/cvd/instances/lock/lock_file.h @@ -15,6 +15,8 @@ */ #pragma once +#include + #include #include #include @@ -66,6 +68,7 @@ class LockFile { const auto& LockFilePath() const { return lock_file_path_; } Result Status() const; Result Status(InUseState); + Result OwnerUid() const; // to put this into a set bool operator<(const LockFile& other) const;