From f94702e2f6f516b2f6ed2533d0b6a0f27624b527 Mon Sep 17 00:00:00 2001 From: Arjun Dhaliwal Date: Wed, 18 Nov 2026 13:33:45 -0800 Subject: [PATCH 1/4] Make the netlink driver match iproute2 --- base/cvd/allocd/alloc_netlink.cpp | 21 +++++++++++-------- base/cvd/host_tests/common/ip.go | 16 ++++++++++++++ .../cvd/host_tests/cvdalloc_test/main_test.go | 10 +++++---- .../cvdalloc_test/shared_bridge_test.go | 20 +++++++++--------- 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/base/cvd/allocd/alloc_netlink.cpp b/base/cvd/allocd/alloc_netlink.cpp index a39164358e4..509a7277809 100644 --- a/base/cvd/allocd/alloc_netlink.cpp +++ b/base/cvd/allocd/alloc_netlink.cpp @@ -125,17 +125,22 @@ Result BringUpIface(absl::string_view name) { Result AddGateway(absl::string_view name, absl::string_view gateway, absl::string_view netmask) { VLOG(0) << "AddGateway: " << name << ", " << gateway << netmask; - auto index = Index(name); - PCHECK(index.has_value()) << "Index: " << name; + const unsigned int index = CF_EXPECT(Index(name)); auto factory = NetlinkClientFactory::Default(); std::unique_ptr nl = factory->New(NETLINK_ROUTE); + const int prefix = Prefix(netmask); NetlinkRequest req(RTM_NEWADDR, NLM_F_REQUEST | NLM_F_ACK); - req.AddAddrInfo(*index, Prefix(netmask)); + req.AddAddrInfo(index, prefix); in_addr_t inaddr = inet_addr(std::string(gateway).c_str()); req.AddInAddr(IFA_LOCAL, &inaddr); // NOLINT(misc-include-cleaner): netlink req.AddInAddr(IFA_ADDRESS, &inaddr); // NOLINT(misc-include-cleaner): netlink + // Same as iproute2's "broadcast +". + if (prefix <= 30) { + in_addr_t broadcast = inaddr | htonl(UINT32_MAX >> prefix); + req.AddInAddr(IFA_BROADCAST, &broadcast); // NOLINT(misc-include-cleaner) + } CF_EXPECT(nl->Send(req), "AddGateway"); return {}; @@ -165,17 +170,15 @@ Result DestroyGateway(absl::string_view name, absl::string_view gateway, Result LinkTapToBridge(absl::string_view tap_name, absl::string_view bridge_name) { VLOG(0) << "LinkTapToBridge: " << tap_name << ", " << bridge_name; - auto tap_index = Index(tap_name); - PCHECK(tap_index.has_value()) << "Index: " << tap_name; - auto bridge_index = Index(bridge_name); - PCHECK(bridge_index.has_value()) << "Index: " << bridge_name; + const unsigned int tap_index = CF_EXPECT(Index(tap_name)); + const unsigned int bridge_index = CF_EXPECT(Index(bridge_name)); auto factory = NetlinkClientFactory::Default(); std::unique_ptr nl = factory->New(NETLINK_ROUTE); NetlinkRequest req(RTM_NEWLINK, NLM_F_REQUEST | NLM_F_ACK); - req.AddIfInfo(*tap_index, true); - req.AddInt(IFLA_MASTER, *bridge_index); + req.AddIfInfo(tap_index, true); + req.AddInt(IFLA_MASTER, bridge_index); CF_EXPECT(nl->Send(req), "LinkTapToBridge"); return {}; diff --git a/base/cvd/host_tests/common/ip.go b/base/cvd/host_tests/common/ip.go index af4118bc834..967f0ea5c7c 100644 --- a/base/cvd/host_tests/common/ip.go +++ b/base/cvd/host_tests/common/ip.go @@ -71,6 +71,7 @@ type AddrInfo struct { Family string `json:"family"` Local string `json:"local"` Prefixlen int `json:"prefixlen"` + Broadcast string `json:"broadcast"` Scope string `json:"scope"` } @@ -100,3 +101,18 @@ func (hs HostState) PrimaryIPv4(ifname string) string { } return "" } + +// the broadcast address of PrimaryIPv4, or "" +func (hs HostState) PrimaryIPv4Broadcast(ifname string) string { + for _, a := range hs.Addrs { + if a.Ifname != ifname { + continue + } + for _, ai := range a.AddrInfo { + if ai.Family == "inet" { + return ai.Broadcast + } + } + } + return "" +} diff --git a/base/cvd/host_tests/cvdalloc_test/main_test.go b/base/cvd/host_tests/cvdalloc_test/main_test.go index 609f68a1c91..b7b81cb9542 100644 --- a/base/cvd/host_tests/cvdalloc_test/main_test.go +++ b/base/cvd/host_tests/cvdalloc_test/main_test.go @@ -26,6 +26,7 @@ import ( type netIface struct { Name string Addr string // primary IPv4 CIDR, "" if none + Brd string // broadcast address of Addr, "" if none Master string // bridge it is enslaved to, "" if none Tun bool // tap opened with vnet_hdr } @@ -42,12 +43,12 @@ type phaseState struct { func instance1State() phaseState { return phaseState{ Ifaces: []netIface{ - {Name: "cvd-pi-ebr", Addr: "192.168.192.1/24"}, - {Name: "cvd-pi-wbr", Addr: "192.168.160.1/24"}, + {Name: "cvd-pi-ebr", Addr: "192.168.192.1/24", Brd: "192.168.192.255"}, + {Name: "cvd-pi-wbr", Addr: "192.168.160.1/24", Brd: "192.168.160.255"}, {Name: "cvd-pi-etap1", Master: "cvd-pi-ebr", Tun: true}, {Name: "cvd-pi-wtap1", Master: "cvd-pi-wbr", Tun: true}, - {Name: "cvd-pi-mtap1", Addr: "192.168.144.1/30", Tun: true}, - {Name: "cvd-pi-wifiap1", Addr: "192.168.176.1/30", Tun: true}, + {Name: "cvd-pi-mtap1", Addr: "192.168.144.1/30", Brd: "192.168.144.3", Tun: true}, + {Name: "cvd-pi-wifiap1", Addr: "192.168.176.1/30", Brd: "192.168.176.3", Tun: true}, }, NftTables: []common.NftTable{{Family: "ip", Name: "cvdalloc_1", Flags: []string{"owner"}}}, NftChains: []common.NftChain{ @@ -105,6 +106,7 @@ func observe(t *testing.T, s *common.Sandbox) phaseState { got.Ifaces = append(got.Ifaces, netIface{ Name: l.Ifname, Addr: hs.PrimaryIPv4(l.Ifname), + Brd: hs.PrimaryIPv4Broadcast(l.Ifname), Master: l.Master, Tun: l.Kind() == "tun" && l.VnetHdr(), }) diff --git a/base/cvd/host_tests/cvdalloc_test/shared_bridge_test.go b/base/cvd/host_tests/cvdalloc_test/shared_bridge_test.go index f83dbbbb0f4..78b6ffbd970 100644 --- a/base/cvd/host_tests/cvdalloc_test/shared_bridge_test.go +++ b/base/cvd/host_tests/cvdalloc_test/shared_bridge_test.go @@ -26,16 +26,16 @@ import ( func TestCvdallocSharedBridgeRelease(t *testing.T) { wantBoth := phaseState{ Ifaces: []netIface{ - {Name: "cvd-pi-ebr", Addr: "192.168.192.1/24"}, - {Name: "cvd-pi-wbr", Addr: "192.168.160.1/24"}, + {Name: "cvd-pi-ebr", Addr: "192.168.192.1/24", Brd: "192.168.192.255"}, + {Name: "cvd-pi-wbr", Addr: "192.168.160.1/24", Brd: "192.168.160.255"}, {Name: "cvd-pi-etap1", Master: "cvd-pi-ebr", Tun: true}, {Name: "cvd-pi-wtap1", Master: "cvd-pi-wbr", Tun: true}, - {Name: "cvd-pi-mtap1", Addr: "192.168.144.1/30", Tun: true}, - {Name: "cvd-pi-wifiap1", Addr: "192.168.176.1/30", Tun: true}, + {Name: "cvd-pi-mtap1", Addr: "192.168.144.1/30", Brd: "192.168.144.3", Tun: true}, + {Name: "cvd-pi-wifiap1", Addr: "192.168.176.1/30", Brd: "192.168.176.3", Tun: true}, {Name: "cvd-pi-etap2", Master: "cvd-pi-ebr", Tun: true}, {Name: "cvd-pi-wtap2", Master: "cvd-pi-wbr", Tun: true}, - {Name: "cvd-pi-mtap2", Addr: "192.168.144.5/30", Tun: true}, - {Name: "cvd-pi-wifiap2", Addr: "192.168.176.5/30", Tun: true}, + {Name: "cvd-pi-mtap2", Addr: "192.168.144.5/30", Brd: "192.168.144.7", Tun: true}, + {Name: "cvd-pi-wifiap2", Addr: "192.168.176.5/30", Brd: "192.168.176.7", Tun: true}, }, NftTables: []common.NftTable{ {Family: "ip", Name: "cvdalloc_1", Flags: []string{"owner"}}, @@ -54,12 +54,12 @@ func TestCvdallocSharedBridgeRelease(t *testing.T) { wantOnlyTwo := phaseState{ Ifaces: []netIface{ - {Name: "cvd-pi-ebr", Addr: "192.168.192.1/24"}, - {Name: "cvd-pi-wbr", Addr: "192.168.160.1/24"}, + {Name: "cvd-pi-ebr", Addr: "192.168.192.1/24", Brd: "192.168.192.255"}, + {Name: "cvd-pi-wbr", Addr: "192.168.160.1/24", Brd: "192.168.160.255"}, {Name: "cvd-pi-etap2", Master: "cvd-pi-ebr", Tun: true}, {Name: "cvd-pi-wtap2", Master: "cvd-pi-wbr", Tun: true}, - {Name: "cvd-pi-mtap2", Addr: "192.168.144.5/30", Tun: true}, - {Name: "cvd-pi-wifiap2", Addr: "192.168.176.5/30", Tun: true}, + {Name: "cvd-pi-mtap2", Addr: "192.168.144.5/30", Brd: "192.168.144.7", Tun: true}, + {Name: "cvd-pi-wifiap2", Addr: "192.168.176.5/30", Brd: "192.168.176.7", Tun: true}, }, NftTables: []common.NftTable{{Family: "ip", Name: "cvdalloc_2", Flags: []string{"owner"}}}, NftChains: []common.NftChain{ From dd80bdb7d39ec0d95e5d8c7eed150b8676522a6e Mon Sep 17 00:00:00 2001 From: Arjun Dhaliwal Date: Wed, 18 Nov 2026 13:33:45 -0800 Subject: [PATCH 2/4] Default allocd to the netlink driver --- base/cvd/allocd/BUILD.bazel | 36 +++++++++++----- base/cvd/allocd/alloc_netlink_dummy.cpp | 56 +++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 base/cvd/allocd/alloc_netlink_dummy.cpp diff --git a/base/cvd/allocd/BUILD.bazel b/base/cvd/allocd/BUILD.bazel index 00eb7f6ddfc..9e88b124ac7 100644 --- a/base/cvd/allocd/BUILD.bazel +++ b/base/cvd/allocd/BUILD.bazel @@ -7,14 +7,7 @@ package( string_flag( name = "driver", - build_setting_default = "iproute2", -) - -config_setting( - name = "netlink", - flag_values = { - ":driver": "netlink", - }, + build_setting_default = "netlink", ) config_setting( @@ -29,8 +22,8 @@ cf_cc_library( srcs = ["alloc_utils.cpp"], hdrs = ["alloc_utils.h"], deps = select({ - ":netlink": [":alloc_netlink"], - "//conditions:default": [":alloc_iproute2"], + ":iproute2": [":alloc_iproute2"], + "//conditions:default": [":alloc_netlink"], }) + [ "//cuttlefish/common/libs/fs:fd", "//cuttlefish/common/libs/utils:files", @@ -58,10 +51,31 @@ cf_cc_library( ], ) -cf_cc_library( +alias( name = "alloc_netlink", + actual = select({ + "@platforms//os:linux": ":alloc_netlink_linux", + "//conditions:default": ":alloc_netlink_dummy", + }), +) + +cf_cc_library( + name = "alloc_netlink_dummy", + srcs = ["alloc_netlink_dummy.cpp"], + hdrs = ["alloc_driver.h"], + deps = [ + "//cuttlefish/result", + "@abseil-cpp//absl/strings:string_view", + ], +) + +cf_cc_library( + name = "alloc_netlink_linux", srcs = ["alloc_netlink.cpp"], hdrs = ["alloc_driver.h"], + target_compatible_with = [ + "@platforms//os:linux", + ], deps = [ "//allocd/net:netlink", "//cuttlefish/common/libs/fs:fd", diff --git a/base/cvd/allocd/alloc_netlink_dummy.cpp b/base/cvd/allocd/alloc_netlink_dummy.cpp new file mode 100644 index 00000000000..be4ff445032 --- /dev/null +++ b/base/cvd/allocd/alloc_netlink_dummy.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "absl/strings/string_view.h" + +#include "allocd/alloc_driver.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { +namespace { + +constexpr char kLinuxOnly[] = "The netlink driver only supports Linux"; + +} // namespace + +Result AddTapIface(absl::string_view) { return CF_ERR(kLinuxOnly); } + +Result ShutdownIface(absl::string_view) { return CF_ERR(kLinuxOnly); } + +Result BringUpIface(absl::string_view) { return CF_ERR(kLinuxOnly); } + +Result AddGateway(absl::string_view, absl::string_view, + absl::string_view) { + return CF_ERR(kLinuxOnly); +} + +Result DestroyGateway(absl::string_view, absl::string_view, + absl::string_view) { + return CF_ERR(kLinuxOnly); +} + +Result LinkTapToBridge(absl::string_view, absl::string_view) { + return CF_ERR(kLinuxOnly); +} + +Result DeleteIface(absl::string_view) { return CF_ERR(kLinuxOnly); } + +Result BridgeInUse(absl::string_view) { return CF_ERR(kLinuxOnly); } + +Result BridgeExists(absl::string_view) { return CF_ERR(kLinuxOnly); } + +Result CreateBridge(absl::string_view) { return CF_ERR(kLinuxOnly); } + +} // namespace cuttlefish From df95482ee51a51fd9fef97fcefbd2237da611ec8 Mon Sep 17 00:00:00 2001 From: Arjun Dhaliwal Date: Wed, 18 Nov 2026 13:33:45 -0800 Subject: [PATCH 3/4] Clean up after a killed cvdalloc --- base/cvd/allocd/BUILD.bazel | 1 + base/cvd/allocd/alloc_utils.cpp | 61 ++++++--- .../host/commands/cvdalloc/BUILD.bazel | 1 + .../host/commands/cvdalloc/cvdalloc.cpp | 33 +++++ base/cvd/cuttlefish/process/proc_file_utils.h | 3 + .../process/proc_file_utils_darwin.cc | 3 + .../process/proc_file_utils_linux.cc | 16 +++ .../process/proc_file_utils_openbsd.cc | 3 + base/cvd/host_tests/common/cvdalloc.go | 53 ++++++++ base/cvd/host_tests/cvdalloc_test/BUILD.bazel | 1 + .../host_tests/cvdalloc_test/crash_test.go | 126 ++++++++++++++++++ .../cvdalloc_test/shared_bridge_test.go | 18 ++- 12 files changed, 293 insertions(+), 26 deletions(-) create mode 100644 base/cvd/host_tests/cvdalloc_test/crash_test.go diff --git a/base/cvd/allocd/BUILD.bazel b/base/cvd/allocd/BUILD.bazel index 9e88b124ac7..69237c81dd8 100644 --- a/base/cvd/allocd/BUILD.bazel +++ b/base/cvd/allocd/BUILD.bazel @@ -30,6 +30,7 @@ cf_cc_library( "//cuttlefish/common/libs/utils:network", "//cuttlefish/host/commands/cvd/utils:common", "//cuttlefish/process:execute", + "//cuttlefish/process:proc_file_utils", "//cuttlefish/result", "@abseil-cpp//absl/base:no_destructor", "@abseil-cpp//absl/log", diff --git a/base/cvd/allocd/alloc_utils.cpp b/base/cvd/allocd/alloc_utils.cpp index 919b5732dbe..1f4de254e06 100644 --- a/base/cvd/allocd/alloc_utils.cpp +++ b/base/cvd/allocd/alloc_utils.cpp @@ -19,13 +19,15 @@ #include #include #include +#include -#include +#include #include #include #include "absl/base/no_destructor.h" #include "absl/log/log.h" +#include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" @@ -36,6 +38,7 @@ #include "cuttlefish/common/libs/utils/network.h" #include "cuttlefish/host/commands/cvd/utils/common.h" #include "cuttlefish/process/execute.h" +#include "cuttlefish/process/proc_file_utils.h" #include "cuttlefish/result/result.h" #ifdef __linux__ @@ -55,6 +58,31 @@ Result SearchForDnsmasq() { return CF_EXPECT(Search({"/usr/sbin", "/sbin"}, "dnsmasq")); } +std::string DnsmasqPidFile(absl::string_view bridge_name) { + return absl::StrCat(CvdDir(), "/cuttlefish-dnsmasq-", bridge_name, ".pid"); +} + +std::optional ReadDnsmasqPid(absl::string_view bridge_name) { + const Result contents = + ReadFileContents(DnsmasqPidFile(bridge_name)); + pid_t pid = 0; + if (!contents.has_value() || !absl::SimpleAtoi(*contents, &pid) || pid <= 0) { + return std::nullopt; + } + return pid; +} + +bool DnsmasqAlive(absl::string_view bridge_name) { + const std::optional pid = ReadDnsmasqPid(bridge_name); + return pid.has_value() && ProcessAlive(*pid); +} + +bool StartBridgeDnsmasq(absl::string_view bridge_name, + absl::string_view ipaddr) { + return StartDnsmasq(bridge_name, absl::StrFormat("%s.1", ipaddr), + absl::StrFormat("%s.2,%s.255", ipaddr, ipaddr)); +} + } // namespace bool CreateEthernetIface(absl::string_view name, @@ -199,7 +227,6 @@ bool SetupBridgeGateway(absl::string_view bridge_name, GatewayConfig config{false, false}; auto gateway = absl::StrFormat("%s.1", ipaddr); auto netmask = "/24"; - auto dhcp_range = absl::StrFormat("%s.2,%s.255", ipaddr, ipaddr); if (!AddGateway(bridge_name, gateway, netmask).has_value()) { return false; @@ -207,7 +234,7 @@ bool SetupBridgeGateway(absl::string_view bridge_name, config.has_gateway = true; - if (!StartDnsmasq(bridge_name, gateway, dhcp_range)) { + if (!StartBridgeDnsmasq(bridge_name, ipaddr)) { CleanupBridgeGateway(bridge_name, ipaddr, config); return false; } @@ -250,31 +277,23 @@ bool StartDnsmasq(absl::string_view bridge_name, absl::string_view gateway, absl::StrCat("--dhcp-option=option:dns-server,", dns_servers), absl::StrCat("--dhcp-option=option6:dns-server,", dns6_servers), "--conf-file=", - absl::StrCat("--pid-file=", CvdDir(), "/cuttlefish-dnsmasq-", - bridge_name, ".pid"), + absl::StrCat("--pid-file=", DnsmasqPidFile(bridge_name)), absl::StrCat("--dhcp-leasefile=", CvdDir(), "/cuttlefish-dnsmasq-", bridge_name, ".leases"), "--dhcp-no-override"}) == 0; } bool StopDnsmasq(absl::string_view name) { - std::ifstream file; - std::string filename = - absl::StrCat(CvdDir(), "/cuttlefish-dnsmasq-", name, ".pid"); LOG(INFO) << "stopping dnsmasq for interface: " << name; - file.open(filename); - if (!file.is_open()) { - LOG(INFO) << "dnsmasq file:" << filename - << " could not be opened, assume dnsmasq has already stopped"; + const std::optional pid = ReadDnsmasqPid(name); + if (!pid.has_value()) { + LOG(INFO) << "dnsmasq file:" << DnsmasqPidFile(name) + << " is missing or empty, assume dnsmasq has already stopped"; return true; } - std::string pid; - file >> pid; - file.close(); - // TODO: Let's use kill(2) instead of subjecting ourselves to this. - bool ret = Execute({"kill", pid}) == 0; + const bool ret = Execute({"kill", std::to_string(*pid)}) == 0; if (ret) { LOG(INFO) << "dnsmasq for:" << name << "successfully stopped"; } else { @@ -287,8 +306,12 @@ bool CreateEthernetBridgeIface(absl::string_view name, absl::string_view ipaddr) { auto exists = BridgeExists(name); if (exists.has_value() && *exists) { - LOG(INFO) << "Bridge " << name << " exists already, doing nothing."; - return true; + LOG(INFO) << "Bridge " << name << " exists already, reusing it."; + if (DnsmasqAlive(name)) { + return true; + } + LOG(INFO) << "Restarting dnsmasq for bridge " << name; + return StartBridgeDnsmasq(name, ipaddr); } if (!CreateBridge(name).has_value()) { diff --git a/base/cvd/cuttlefish/host/commands/cvdalloc/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvdalloc/BUILD.bazel index e8dac90632b..af939385ed5 100644 --- a/base/cvd/cuttlefish/host/commands/cvdalloc/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvdalloc/BUILD.bazel @@ -57,6 +57,7 @@ cf_cc_binary( "//allocd/net:nft_nat_table", "//cuttlefish/common/libs/fs", "//cuttlefish/common/libs/fs:fd", + "//cuttlefish/common/libs/utils:network", "//cuttlefish/posix:strerror", "//cuttlefish/posix:temp_failure_retry", "//cuttlefish/result:expect", diff --git a/base/cvd/cuttlefish/host/commands/cvdalloc/cvdalloc.cpp b/base/cvd/cuttlefish/host/commands/cvdalloc/cvdalloc.cpp index d1fde872d90..7249cfcd100 100644 --- a/base/cvd/cuttlefish/host/commands/cvdalloc/cvdalloc.cpp +++ b/base/cvd/cuttlefish/host/commands/cvdalloc/cvdalloc.cpp @@ -32,6 +32,7 @@ #include "allocd/net/nft_nat_table.h" #include "cuttlefish/common/libs/fs/fd.h" #include "cuttlefish/common/libs/fs/shared_fd.h" +#include "cuttlefish/common/libs/utils/network.h" #include "cuttlefish/host/commands/cvdalloc/interface.h" #include "cuttlefish/host/commands/cvdalloc/privilege.h" #include "cuttlefish/host/commands/cvdalloc/sem.h" @@ -102,6 +103,37 @@ Result CreateNatTable(int id) { }); } +constexpr const char* kTapKinds[] = {"mtap", "wtap", "wifiap", "etap"}; + +bool OwnerAlive(int id) { + const Result alive = NftNatTable::Exists(NatTableName(id)); + if (!alive.has_value()) { + LOG(WARNING) << "cvdalloc: assuming id " << id + << " is alive, cannot check: " << alive.error().Message(); + return true; + } + return *alive; +} + +void RemoveStaleTaps() { + for (int id = 1; id <= static_cast(kMaxIfaceNameId); ++id) { + std::vector taps; + for (const char* kind : kTapKinds) { + std::string name = CvdallocInterfaceName(kind, id); + if (NetworkInterfaceExists(name)) { + taps.push_back(std::move(name)); + } + } + if (taps.empty() || OwnerAlive(id)) { + continue; + } + for (const std::string& tap : taps) { + LOG(INFO) << "cvdalloc: removing stale interface " << tap; + DestroyIface(tap); + } + } +} + } // namespace Result CvdallocMain(int argc, char* argv[]) { @@ -128,6 +160,7 @@ Result CvdallocMain(int argc, char* argv[]) { // Elevate before creating the table and taps, so they're cleaned up before // privileges are dropped. const ScopedPrivileges privileges = CF_EXPECT(ScopedPrivileges::Elevate()); + RemoveStaleTaps(); // Create the table before the taps and delete it after them. If another // cvdalloc is using this id, we stop here before touching its taps. const NftNatTable nat = CF_EXPECT(CreateNatTable(id)); diff --git a/base/cvd/cuttlefish/process/proc_file_utils.h b/base/cvd/cuttlefish/process/proc_file_utils.h index 07aec4ad44c..b08fbc3a03a 100644 --- a/base/cvd/cuttlefish/process/proc_file_utils.h +++ b/base/cvd/cuttlefish/process/proc_file_utils.h @@ -86,4 +86,7 @@ Result> GetEnvs(pid_t pid); Result Ppid(pid_t pid); +// On Linux a zombie counts as dead, unlike with kill(pid, 0). +bool ProcessAlive(pid_t pid); + } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/process/proc_file_utils_darwin.cc b/base/cvd/cuttlefish/process/proc_file_utils_darwin.cc index acca6d4f1b8..dfac9149b36 100644 --- a/base/cvd/cuttlefish/process/proc_file_utils_darwin.cc +++ b/base/cvd/cuttlefish/process/proc_file_utils_darwin.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -170,4 +171,6 @@ Result Ppid(const pid_t pid) { return CF_EXPECT(KinfoProc(pid)).kp_eproc.e_ppid; } +bool ProcessAlive(const pid_t pid) { return kill(pid, 0) == 0; } + } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/process/proc_file_utils_linux.cc b/base/cvd/cuttlefish/process/proc_file_utils_linux.cc index f1ae95d1099..4d2c1de5ce0 100644 --- a/base/cvd/cuttlefish/process/proc_file_utils_linux.cc +++ b/base/cvd/cuttlefish/process/proc_file_utils_linux.cc @@ -15,6 +15,7 @@ */ #include +#include #include // IWYU pragma: keep #include #include @@ -253,4 +254,19 @@ Result Ppid(const pid_t pid) { return CF_ERR("Status file does not have PPid: line in the right format"); } +bool ProcessAlive(const pid_t pid) { + const Result stat = + ReadFileContents(fmt::format("/proc/{}/stat", pid)); + if (!stat.has_value()) { + return false; + } + // The command name can contain ')', so search from the end. + const size_t comm_end = stat->rfind(')'); + if (comm_end == std::string::npos || comm_end + 2 >= stat->size()) { + return false; + } + const char state = (*stat)[comm_end + 2]; + return state != 'Z' && state != 'X'; +} + } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/process/proc_file_utils_openbsd.cc b/base/cvd/cuttlefish/process/proc_file_utils_openbsd.cc index bb7dc6ce3ab..db71cef08ca 100644 --- a/base/cvd/cuttlefish/process/proc_file_utils_openbsd.cc +++ b/base/cvd/cuttlefish/process/proc_file_utils_openbsd.cc @@ -15,6 +15,7 @@ */ #include +#include #include #include #include @@ -203,4 +204,6 @@ Result ExtractProcInfo(const pid_t pid) { Result Ppid(const pid_t pid) { return CF_EXPECT(KinfoProc(pid)).p_ppid; } +bool ProcessAlive(const pid_t pid) { return kill(pid, 0) == 0; } + } // namespace cuttlefish diff --git a/base/cvd/host_tests/common/cvdalloc.go b/base/cvd/host_tests/common/cvdalloc.go index e39f4b74893..c25ee63d130 100644 --- a/base/cvd/host_tests/common/cvdalloc.go +++ b/base/cvd/host_tests/common/cvdalloc.go @@ -167,6 +167,30 @@ func (c *Cvdalloc) StopInstance(inst *Instance) error { return nil } +// KillInstance SIGKILLs cvdalloc, as a crash would. +func (c *Cvdalloc) KillInstance(inst *Instance) error { + pid, err := inst.cvdallocPid() + if err != nil { + return err + } + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + return fmt.Errorf("killing cvdalloc id=%d: %w", inst.ID, err) + } + inst.conn.Close() + // nsenter exits with the signal that killed cvdalloc + inst.cmd.Wait() + return nil +} + +func (i *Instance) cvdallocPid() (int, error) { + // nsenter forks to enter the pid namespace, so cvdalloc is its child + pid, err := childPid(i.cmd.Process.Pid) + if err != nil { + return 0, fmt.Errorf("finding cvdalloc id=%d: %w", i.ID, err) + } + return pid, nil +} + func (i *Instance) kill() { i.conn.Close() if i.cmd.Process != nil { @@ -217,3 +241,32 @@ func CvdallocDnsmasqIfaces(s *Sandbox) []string { sort.Strings(ifaces) return ifaces } + +func cvdallocDnsmasqPidFile(iface string) string { + return fmt.Sprintf("%s/cuttlefish-dnsmasq-%s.pid", cvdallocRunDir, iface) +} + +// KillCvdallocDnsmasq SIGKILLs iface's dnsmasq, leaving its pidfile behind. +func KillCvdallocDnsmasq(s *Sandbox, iface string) error { + _, err := s.Run("sh", "-c", fmt.Sprintf("kill -9 $(cat %s)", cvdallocDnsmasqPidFile(iface))) + if err != nil { + return fmt.Errorf("killing dnsmasq for %s: %w", iface, err) + } + return nil +} + +// CvdallocDnsmasqAlive reports whether the dnsmasq for iface is running. +// Zombies count as dead, since the sandbox doesn't reap orphans. +func CvdallocDnsmasqAlive(s *Sandbox, iface string) bool { + out, err := s.Run("sh", "-c", fmt.Sprintf("cat /proc/$(cat %s)/stat", cvdallocDnsmasqPidFile(iface))) + if err != nil { + return false + } + // the command name can contain ')', so search from the end + commEnd := strings.LastIndex(out.Stdout, ")") + if commEnd < 0 || commEnd+2 >= len(out.Stdout) { + return false + } + state := out.Stdout[commEnd+2] + return state != 'Z' && state != 'X' +} diff --git a/base/cvd/host_tests/cvdalloc_test/BUILD.bazel b/base/cvd/host_tests/cvdalloc_test/BUILD.bazel index 4a5e3b82429..fc8041d6d83 100644 --- a/base/cvd/host_tests/cvdalloc_test/BUILD.bazel +++ b/base/cvd/host_tests/cvdalloc_test/BUILD.bazel @@ -18,6 +18,7 @@ go_test( name = "cvdalloc_test", size = "large", srcs = [ + "crash_test.go", "main_test.go", "shared_bridge_test.go", ], diff --git a/base/cvd/host_tests/cvdalloc_test/crash_test.go b/base/cvd/host_tests/cvdalloc_test/crash_test.go new file mode 100644 index 00000000000..70cf4cdd10f --- /dev/null +++ b/base/cvd/host_tests/cvdalloc_test/crash_test.go @@ -0,0 +1,126 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/google/android-cuttlefish/base/cvd/host_tests/common" + "github.com/google/go-cmp/cmp" +) + +func killedInstance1State() phaseState { + state := instance1State() + state.NftTables = nil + state.NftChains = nil + state.Masquerades = nil + return state +} + +// Restarting a killed id must clean up its leftover taps instead of failing. +func TestCvdallocRestartAfterKill(t *testing.T) { + s := common.NewSandbox(t) + c := common.NewCvdalloc(t, s) + base := snapshot(t, s) + + inst, err := c.StartInstance(1) + if err != nil { + t.Fatalf("start instance 1: %v", err) + } + if err := c.KillInstance(inst); err != nil { + t.Fatalf("kill instance 1: %v", err) + } + if diff := cmp.Diff(killedInstance1State(), observe(t, s), cmpOptions()); diff != "" { + t.Fatalf("host state after kill (-want +got):\n%s", diff) + } + + inst, err = c.StartInstance(1) + if err != nil { + t.Fatalf("restart instance 1: %v", err) + } + if diff := cmp.Diff(instance1State(), observe(t, s), cmpOptions()); diff != "" { + t.Errorf("host state after restart (-want +got):\n%s", diff) + } + + if err := c.StopInstance(inst); err != nil { + t.Fatalf("stop instance 1: %v", err) + } + checkReleased(t, s, base) +} + +// A killed id that never comes back must not leak its taps. +func TestCvdallocSweepsTapsOfKilledId(t *testing.T) { + s := common.NewSandbox(t) + c := common.NewCvdalloc(t, s) + base := snapshot(t, s) + + inst1, err := c.StartInstance(1) + if err != nil { + t.Fatalf("start instance 1: %v", err) + } + if err := c.KillInstance(inst1); err != nil { + t.Fatalf("kill instance 1: %v", err) + } + + inst2, err := c.StartInstance(2) + if err != nil { + t.Fatalf("start instance 2: %v", err) + } + if diff := cmp.Diff(onlyInstance2State(), observe(t, s), cmpOptions()); diff != "" { + t.Errorf("host state after instance 2 swept instance 1 (-want +got):\n%s", diff) + } + + if err := c.StopInstance(inst2); err != nil { + t.Fatalf("stop instance 2: %v", err) + } + checkReleased(t, s, base) +} + +func TestCvdallocRestartsDeadDnsmasqOnReuse(t *testing.T) { + const bridge = "cvd-pi-ebr" + s := common.NewSandbox(t) + c := common.NewCvdalloc(t, s) + base := snapshot(t, s) + + inst1, err := c.StartInstance(1) + if err != nil { + t.Fatalf("start instance 1: %v", err) + } + if err := common.KillCvdallocDnsmasq(s, bridge); err != nil { + t.Fatal(err) + } + if common.CvdallocDnsmasqAlive(s, bridge) { + t.Fatalf("dnsmasq for %s survived SIGKILL", bridge) + } + + inst2, err := c.StartInstance(2) + if err != nil { + t.Fatalf("start instance 2: %v", err) + } + if !common.CvdallocDnsmasqAlive(s, bridge) { + t.Errorf("dnsmasq for %s was not restarted", bridge) + } + if diff := cmp.Diff(bothInstancesState(), observe(t, s), cmpOptions()); diff != "" { + t.Errorf("host state with both instances (-want +got):\n%s", diff) + } + + if err := c.StopInstance(inst2); err != nil { + t.Fatalf("stop instance 2: %v", err) + } + if err := c.StopInstance(inst1); err != nil { + t.Fatalf("stop instance 1: %v", err) + } + checkReleased(t, s, base) +} diff --git a/base/cvd/host_tests/cvdalloc_test/shared_bridge_test.go b/base/cvd/host_tests/cvdalloc_test/shared_bridge_test.go index 78b6ffbd970..ac2f2f4c1eb 100644 --- a/base/cvd/host_tests/cvdalloc_test/shared_bridge_test.go +++ b/base/cvd/host_tests/cvdalloc_test/shared_bridge_test.go @@ -21,10 +21,8 @@ import ( "github.com/google/go-cmp/cmp" ) -// Start two instances, stop one, and check that the shared bridges, gateways -// and dnsmasq are untouched. -func TestCvdallocSharedBridgeRelease(t *testing.T) { - wantBoth := phaseState{ +func bothInstancesState() phaseState { + return phaseState{ Ifaces: []netIface{ {Name: "cvd-pi-ebr", Addr: "192.168.192.1/24", Brd: "192.168.192.255"}, {Name: "cvd-pi-wbr", Addr: "192.168.160.1/24", Brd: "192.168.160.255"}, @@ -51,8 +49,10 @@ func TestCvdallocSharedBridgeRelease(t *testing.T) { }, DnsmasqIfaces: []string{"cvd-pi-ebr", "cvd-pi-wbr"}, } +} - wantOnlyTwo := phaseState{ +func onlyInstance2State() phaseState { + return phaseState{ Ifaces: []netIface{ {Name: "cvd-pi-ebr", Addr: "192.168.192.1/24", Brd: "192.168.192.255"}, {Name: "cvd-pi-wbr", Addr: "192.168.160.1/24", Brd: "192.168.160.255"}, @@ -70,7 +70,11 @@ func TestCvdallocSharedBridgeRelease(t *testing.T) { }, DnsmasqIfaces: []string{"cvd-pi-ebr", "cvd-pi-wbr"}, } +} +// Start two instances, stop one, and check that the shared bridges, gateways +// and dnsmasq are untouched. +func TestCvdallocSharedBridgeRelease(t *testing.T) { opts := cmpOptions() s := common.NewSandbox(t) @@ -85,14 +89,14 @@ func TestCvdallocSharedBridgeRelease(t *testing.T) { if err != nil { t.Fatalf("start instance 2: %v", err) } - if diff := cmp.Diff(wantBoth, observe(t, s), opts); diff != "" { + if diff := cmp.Diff(bothInstancesState(), observe(t, s), opts); diff != "" { t.Errorf("host state with both instances (-want +got):\n%s", diff) } if err := c.StopInstance(inst1); err != nil { t.Fatalf("stop instance 1: %v", err) } - if diff := cmp.Diff(wantOnlyTwo, observe(t, s), opts); diff != "" { + if diff := cmp.Diff(onlyInstance2State(), observe(t, s), opts); diff != "" { t.Errorf("host state after releasing instance 1 (-want +got):\n%s", diff) } From ac2623e6ec644fadb877a7e17b9f5e5ec98cbaa2 Mon Sep 17 00:00:00 2001 From: Arjun Dhaliwal Date: Wed, 18 Nov 2026 13:33:45 -0800 Subject: [PATCH 4/4] Tear down cvdalloc on SIGTERM, SIGINT and SIGHUP --- .../host/commands/cvdalloc/BUILD.bazel | 1 + .../host/commands/cvdalloc/cvdalloc.cpp | 48 ++++++++++++++++++- .../cuttlefish/host/commands/cvdalloc/sem.cpp | 36 ++++++++++++++ .../cuttlefish/host/commands/cvdalloc/sem.h | 9 ++++ base/cvd/host_tests/common/cvdalloc.go | 25 ++++++++++ .../host_tests/cvdalloc_test/crash_test.go | 20 ++++++++ 6 files changed, 138 insertions(+), 1 deletion(-) diff --git a/base/cvd/cuttlefish/host/commands/cvdalloc/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvdalloc/BUILD.bazel index af939385ed5..c02710ed57f 100644 --- a/base/cvd/cuttlefish/host/commands/cvdalloc/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvdalloc/BUILD.bazel @@ -27,6 +27,7 @@ cf_cc_library( deps = [ "//cuttlefish/common/libs/fs", "//cuttlefish/host/libs/command_util", + "//cuttlefish/posix:strerror", "//cuttlefish/result:expect", "//cuttlefish/result:result_type", ], diff --git a/base/cvd/cuttlefish/host/commands/cvdalloc/cvdalloc.cpp b/base/cvd/cuttlefish/host/commands/cvdalloc/cvdalloc.cpp index 7249cfcd100..91d4ede6776 100644 --- a/base/cvd/cuttlefish/host/commands/cvdalloc/cvdalloc.cpp +++ b/base/cvd/cuttlefish/host/commands/cvdalloc/cvdalloc.cpp @@ -14,9 +14,13 @@ * limitations under the License. */ +#include +#include +#include #include #include +#include #include #include #include @@ -134,6 +138,41 @@ void RemoveStaleTaps() { } } +// Raw, since SharedFD isn't async-signal-safe. +int signal_pipe_write_fd = -1; + +void ForwardSignal(int signum) { + const int saved_errno = errno; + const char byte = static_cast(signum); + // A full pipe already has a signal pending. + const ssize_t ignored = write(signal_pipe_write_fd, &byte, sizeof(byte)); + (void)ignored; + errno = saved_errno; +} + +constexpr int kForwardedSignals[] = {SIGTERM, SIGINT, SIGHUP}; + +Result ForwardSignalsToPipe() { + int pipe_fds[2]; + CF_EXPECTF(pipe2(pipe_fds, O_CLOEXEC | O_NONBLOCK) == 0, "pipe2: {}", + StrError()); + signal_pipe_write_fd = pipe_fds[1]; + const SharedFD read_end = CF_EXPECT(Fd::Dup(pipe_fds[0])); + close(pipe_fds[0]); + + struct sigaction action = {}; + action.sa_handler = ForwardSignal; + // Subprocess::Wait doesn't retry waitpid on EINTR, and a failed Execute makes + // BridgeInUse report a bridge as free. + action.sa_flags = SA_RESTART; + sigemptyset(&action.sa_mask); + for (const int signum : kForwardedSignals) { + CF_EXPECTF(sigaction(signum, &action, nullptr) == 0, "sigaction({}): {}", + signum, StrError()); + } + return read_end; +} + } // namespace Result CvdallocMain(int argc, char* argv[]) { @@ -157,6 +196,8 @@ Result CvdallocMain(int argc, char* argv[]) { absl::Cleanup shutdown = [sock]() { sock->Shutdown(SHUT_RDWR); }; + const SharedFD signals = CF_EXPECT(ForwardSignalsToPipe()); + // Elevate before creating the table and taps, so they're cleaned up before // privileges are dropped. const ScopedPrivileges privileges = CF_EXPECT(ScopedPrivileges::Elevate()); @@ -178,8 +219,13 @@ Result CvdallocMain(int argc, char* argv[]) { LOG(INFO) << "cvdalloc: waiting to teardown"; - CF_EXPECT(cvdalloc::Wait(sock, cvdalloc::kSemNoTimeout)); + const std::optional signum = + CF_EXPECT(cvdalloc::Wait(sock, cvdalloc::kSemNoTimeout, signals)); std::move(teardown).Invoke(); + if (signum.has_value()) { + LOG(ERROR) << "cvdalloc: terminated by signal " << *signum; + return 1; + } CF_EXPECT(cvdalloc::Post(sock)); return 0; diff --git a/base/cvd/cuttlefish/host/commands/cvdalloc/sem.cpp b/base/cvd/cuttlefish/host/commands/cvdalloc/sem.cpp index a5a1b8986af..e6ba8e8f7fe 100644 --- a/base/cvd/cuttlefish/host/commands/cvdalloc/sem.cpp +++ b/base/cvd/cuttlefish/host/commands/cvdalloc/sem.cpp @@ -15,12 +15,18 @@ */ #include "cuttlefish/host/commands/cvdalloc/sem.h" +#include +#include +#include #include #include +#include +#include #include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/host/libs/command_util/util.h" +#include "cuttlefish/posix/strerror.h" #include "cuttlefish/result/expect.h" #include "cuttlefish/result/result_type.h" @@ -42,4 +48,34 @@ Result Wait(const SharedFD socket, std::chrono::seconds timeout) { return {}; } +Result> Wait(const SharedFD socket, + std::chrono::seconds timeout, + const SharedFD signals) { + std::vector fds = { + {socket, POLLIN, 0}, // NOLINT(misc-include-cleaner) + {signals, POLLIN, 0}, // NOLINT(misc-include-cleaner) + }; + constexpr size_t kSignalsIndex = 1; + const int timeout_ms = + timeout <= kSemNoTimeout + ? -1 + : static_cast(std::chrono::milliseconds(timeout).count()); + while (true) { + const int ready = SharedFD::Poll(fds, timeout_ms); + if (ready == -1 && errno == EINTR) { + continue; + } + CF_EXPECT(ready != 0, "Timed out"); + CF_EXPECTF(ready > 0, "poll: {}", StrError()); + if (fds[kSignalsIndex].revents != 0) { + char signum = 0; + const size_t read = CF_EXPECT(signals->Read(&signum, sizeof(signum))); + CF_EXPECT_EQ(read, sizeof(signum)); + return static_cast(signum); + } + CF_EXPECT(Wait(socket, timeout)); + return std::nullopt; + } +} + } // namespace cuttlefish::cvdalloc diff --git a/base/cvd/cuttlefish/host/commands/cvdalloc/sem.h b/base/cvd/cuttlefish/host/commands/cvdalloc/sem.h index de03fc020ad..a8c1a94fb91 100644 --- a/base/cvd/cuttlefish/host/commands/cvdalloc/sem.h +++ b/base/cvd/cuttlefish/host/commands/cvdalloc/sem.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/result/result_type.h" @@ -40,4 +41,12 @@ Result Post(SharedFD socket); */ Result Wait(SharedFD socket, std::chrono::seconds timeout); +/* + * Like Wait, but also returns when a signal number can be read from signals. + * + * Returns that signal number, or nullopt if the socket was read instead. + */ +Result> Wait(SharedFD socket, std::chrono::seconds timeout, + SharedFD signals); + } // namespace cuttlefish::cvdalloc diff --git a/base/cvd/host_tests/common/cvdalloc.go b/base/cvd/host_tests/common/cvdalloc.go index c25ee63d130..e0532042dbc 100644 --- a/base/cvd/host_tests/common/cvdalloc.go +++ b/base/cvd/host_tests/common/cvdalloc.go @@ -182,6 +182,31 @@ func (c *Cvdalloc) KillInstance(inst *Instance) error { return nil } +// TerminateInstance SIGTERMs cvdalloc and returns its exit code. +func (c *Cvdalloc) TerminateInstance(inst *Instance) (int, error) { + pid, err := inst.cvdallocPid() + if err != nil { + return 0, err + } + if err := syscall.Kill(pid, syscall.SIGTERM); err != nil { + return 0, fmt.Errorf("terminating cvdalloc id=%d: %w", inst.ID, err) + } + done := make(chan struct{}) + go func() { + inst.cmd.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(cvdallocTeardownTimeout): + inst.cmd.Process.Kill() + <-done + return 0, fmt.Errorf("cvdalloc id=%d did not exit within %v of SIGTERM%s", inst.ID, cvdallocTeardownTimeout, inst.stderrSuffix()) + } + inst.conn.Close() + return inst.cmd.ProcessState.ExitCode(), nil +} + func (i *Instance) cvdallocPid() (int, error) { // nsenter forks to enter the pid namespace, so cvdalloc is its child pid, err := childPid(i.cmd.Process.Pid) diff --git a/base/cvd/host_tests/cvdalloc_test/crash_test.go b/base/cvd/host_tests/cvdalloc_test/crash_test.go index 70cf4cdd10f..274eca544cc 100644 --- a/base/cvd/host_tests/cvdalloc_test/crash_test.go +++ b/base/cvd/host_tests/cvdalloc_test/crash_test.go @@ -124,3 +124,23 @@ func TestCvdallocRestartsDeadDnsmasqOnReuse(t *testing.T) { } checkReleased(t, s, base) } + +// SIGTERM must tear down, but exit nonzero so it isn't taken for a clean stop. +func TestCvdallocTerminatesOnSignal(t *testing.T) { + s := common.NewSandbox(t) + c := common.NewCvdalloc(t, s) + base := snapshot(t, s) + + inst, err := c.StartInstance(1) + if err != nil { + t.Fatalf("start instance 1: %v", err) + } + code, err := c.TerminateInstance(inst) + if err != nil { + t.Fatal(err) + } + if code != 1 { + t.Errorf("exit code after SIGTERM = %d, want 1", code) + } + checkReleased(t, s, base) +}