Files
Michael Crosby b438e97b93 Add cloud-hypervisor VMM backend for Linux hosts (#782)
apple/containerization currently runs containers in per-container VMs on
macOS hosts via Virtualization.framework. This adds a second VMM backend
so the same Swift orchestration layer (LinuxContainer / LinuxPod /
Vminitd gRPC contract) runs on Linux hosts via cloud-hypervisor + KVM.

**CloudHypervisor Swift package** (`Sources/CloudHypervisor/`) — a thin
client for cloud-hypervisor's REST-over-UDS API, layered on
AsyncHTTPClient. Endpoints cover VMM / VM lifecycle / hotplug (disk, fs,
net, vsock, remove-device). Cross-platform (compiles on macOS for unit
tests; consumed at runtime only by the Linux side of Containerization).

**CH backend in Containerization** — one cloud-hypervisor subprocess per
VM, gated behind `#if os(Linux)`. CHVirtualMachineManager /
CHVirtualMachineInstance mirror the VZ shape behind the existing
VirtualMachineManager / VirtualMachineInstance protocol. CHProcess and
VirtiofsdProcess manage the binaries; CHHotplugProvider handles
virtio-blk and virtio-fs runtime hotplug (with one virtiofsd per unique
source-hash tag, refcounted across containers).

**Linux host networking** — BridgeManager brings up a Linux bridge with
an IPv4 subnet and (opt-in via `--enable-nat`) iptables MASQUERADE +
scoped FORWARD rules. LinuxBridgedNetwork enslaves a fresh TAP per
container to the bridge. State is recorded under `/run/containerization`
so `cctl bridge delete` reverses exactly what create did. Bridge
teardown verifies the link kind via sysfs to refuse deleting non-bridge
interfaces.

**cctl run / bridge** — end-to-end Linux container run path (image pull,
ext4 rootfs assembly, VM boot, container exec) plus `cctl bridge
create|delete` for the host network plumbing.

**Build & dist** — `make linux-build` / `make linux-integration` build
and exercise the host side inside an apple/container `--virtualization`
dev container. `make dist-x86_64` produces a deployment tarball (cctl +
cloud-hypervisor + virtiofsd + initfs + kernel) cross-compiled from the
aarch64 dev container; pipeline documented in `docs/x86_64-build.md`.
Static-musl C deps and the Zig cross compiler are pinned by SHA256.

The host orchestrator runs as root. Per-VM runtime state lives under
`/run/containerization/ch/<UUID>` with mode 0700; UDS sockets inside are
bound with mode 0600. Vminitd's gRPC channel inherits that trust
boundary — socket-file perms are the auth.

Sandbox flags are upstream-secure by default. Two per-component opt-outs
exist for the apple/container dev-container case (where the host seccomp
profile SIGSYS-kills CH and virtiofsd):
- `CONTAINERIZATION_NO_CH_SECCOMP=1` — `cloud-hypervisor --seccomp
false`.
- `CONTAINERIZATION_NO_VIRTIOFSD_SANDBOX=1` — `virtiofsd --sandbox
none`. Each emits a one-shot `logger.warning` at process start. Legacy
alias `CONTAINERIZATION_RELAXED_SANDBOX=1` flips both. cctl spawns both
binaries with `setsid` and a minimal env allowlist (PATH / HOME /
RUST_LOG / RUST_BACKTRACE) so the parent's secrets don't leak to
children.

`make linux-integration` runs the cross-platform integration suite
against a real cloud-hypervisor VM inside the dev container. Linux runs
the cross-platform subset (`process true`/`false`/`echo hi`, virtiofs
round-trip, hotplug); the macOS suite is unchanged.

Signed-off-by: michael_crosby <michael_crosby@apple.com>
2026-07-02 11:20:22 -04:00

212 lines
7.4 KiB
Swift

//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
#if os(Linux)
import ContainerizationError
import ContainerizationExtras
import ContainerizationOS
import Foundation
import Logging
import Synchronization
#if canImport(Musl)
import Musl
#elseif canImport(Glibc)
import Glibc
#endif
/// A managed `cloud-hypervisor` subprocess.
///
/// Owns spawning the binary with `--api-socket <path>`, attaching stdout/stderr
/// per the supplied `BootLog`, and tearing it down with a SIGTERM/SIGKILL ladder.
/// One `CHProcess` per VM. Not safe to call `start()` more than once.
final class CHProcess: Sendable {
struct Config: Sendable {
let binary: URL
let apiSocketPath: URL
let bootLog: BootLog?
}
enum ExitReason: Sendable, Equatable {
case exited(Int32)
case signalled(Int32)
case unknown
}
private struct State {
var command: Command?
var bootLogHandle: FileHandle?
var exitTask: Task<ExitReason, Never>?
}
private let config: Config
private let logger: Logger?
private let state: Mutex<State>
init(config: Config, logger: Logger?) {
self.config = config
self.logger = logger
self.state = Mutex(State(command: nil, bootLogHandle: nil, exitTask: nil))
}
/// Spawn the cloud-hypervisor binary and wait for its API socket to accept
/// connections. Throws `ContainerizationError(.timeout, ...)` if the socket
/// is not connectable within the bounded poll deadline.
func start() async throws {
let logHandle = try Self.openBootLogHandle(config.bootLog)
var arguments = ["--api-socket", config.apiSocketPath.path]
if SandboxOverrides.chSeccompDisabled {
// `--seccomp false`: cloud-hypervisor's default seccomp profile
// SIGSYS-kills the VMM on syscalls it didn't anticipate. Inside
// apple/container's --virtualization dev container the unix-vsock
// muxer's accept(2)/connect(2) interactions on per-port UDS files
// trip the filter and CH dies mid-process-start, surfacing on the
// host as "Stream unexpectedly closed" on the vminitd gRPC channel.
// Opt-in via CONTAINERIZATION_NO_CH_SECCOMP=1; default = secure.
logger?.warning(
"cloud-hypervisor launching with --seccomp false (CONTAINERIZATION_NO_CH_SECCOMP=1) — VMM seccomp filter disabled"
)
arguments.append(contentsOf: ["--seccomp", "false"])
}
var command = Command(
config.binary.path,
arguments: arguments,
environment: ChildEnvironment.minimal()
)
command.stdout = logHandle
command.stderr = logHandle
// Run cloud-hypervisor in its own session. Without setsid, the VMM
// shares the parent process group and inherits SIGINT/SIGQUIT from
// the controlling TTY (e.g. Ctrl-C in `cctl run`), dying alongside
// the parent before our own teardown ladder (terminate → wait) gets
// a chance to run an orderly shutdown.
command.attrs.setsid = true
do {
try command.start()
} catch {
try? logHandle?.close()
throw error
}
let exitTask = Task<ExitReason, Never>.detached { [command, logger] in
do {
let status = try command.wait()
if status >= 128 {
return .signalled(status - 128)
}
return .exited(status)
} catch {
logger?.error("cloud-hypervisor wait failed: \(error)")
return .unknown
}
}
state.withLock {
$0.command = command
$0.bootLogHandle = logHandle
$0.exitTask = exitTask
}
try await waitForAPISocket()
}
/// Wait for the subprocess to exit. Resolves with the cached `ExitReason`
/// once `wait4` has returned. Safe to call any number of times.
func wait() async -> ExitReason {
guard let task = state.withLock({ $0.exitTask }) else {
return .unknown
}
return await task.value
}
/// Send SIGTERM, then SIGKILL after `graceSeconds` if the process is still
/// running. Returns once the process has been reaped.
func terminate(graceSeconds: UInt32) async {
guard let command = state.withLock({ $0.command }) else { return }
_ = command.kill(SIGTERM)
do {
try await Timeout.run(for: .seconds(Int(graceSeconds))) {
_ = await self.wait()
}
} catch {
logger?.warning("cloud-hypervisor did not exit within \(graceSeconds)s, sending SIGKILL")
_ = command.kill(SIGKILL)
_ = await wait()
}
state.withLock {
try? $0.bootLogHandle?.close()
$0.bootLogHandle = nil
}
}
// MARK: - Private helpers
private static let socketDeadline: Duration = .seconds(2)
private static let socketPollInterval: Duration = .milliseconds(50)
private func waitForAPISocket() async throws {
let clock = ContinuousClock()
let deadline = clock.now.advanced(by: Self.socketDeadline)
while clock.now < deadline {
if Self.isAPISocketReady(at: config.apiSocketPath) {
return
}
try? await Task.sleep(for: Self.socketPollInterval)
}
await terminate(graceSeconds: 5)
throw ContainerizationError(
.timeout,
message: "cloud-hypervisor API socket not connectable at \(config.apiSocketPath.path) within \(Self.socketDeadline)"
)
}
private static func isAPISocketReady(at url: URL) -> Bool {
guard let unix = try? UnixType(path: url.path) else { return false }
guard let socket = try? Socket(type: unix) else { return false }
defer { try? socket.close() }
do {
try socket.connect()
return true
} catch {
return false
}
}
private static func openBootLogHandle(_ bootLog: BootLog?) throws -> FileHandle? {
guard let bootLog else { return nil }
switch bootLog.base {
case .file(let path, let append):
var flags = O_WRONLY | O_CREAT
flags |= append ? O_APPEND : O_TRUNC
let fd = open(path.path, flags, 0o644)
guard fd >= 0 else {
throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
}
return FileHandle(fileDescriptor: fd, closeOnDealloc: true)
case .fileHandle(let handle):
return handle
}
}
}
#endif