b438e97b93
apple/containerization currently runs containers in per-container VMs on macOS hosts via Virtualization.framework. This adds a second VMM backend so the same Swift orchestration layer (LinuxContainer / LinuxPod / Vminitd gRPC contract) runs on Linux hosts via cloud-hypervisor + KVM. **CloudHypervisor Swift package** (`Sources/CloudHypervisor/`) — a thin client for cloud-hypervisor's REST-over-UDS API, layered on AsyncHTTPClient. Endpoints cover VMM / VM lifecycle / hotplug (disk, fs, net, vsock, remove-device). Cross-platform (compiles on macOS for unit tests; consumed at runtime only by the Linux side of Containerization). **CH backend in Containerization** — one cloud-hypervisor subprocess per VM, gated behind `#if os(Linux)`. CHVirtualMachineManager / CHVirtualMachineInstance mirror the VZ shape behind the existing VirtualMachineManager / VirtualMachineInstance protocol. CHProcess and VirtiofsdProcess manage the binaries; CHHotplugProvider handles virtio-blk and virtio-fs runtime hotplug (with one virtiofsd per unique source-hash tag, refcounted across containers). **Linux host networking** — BridgeManager brings up a Linux bridge with an IPv4 subnet and (opt-in via `--enable-nat`) iptables MASQUERADE + scoped FORWARD rules. LinuxBridgedNetwork enslaves a fresh TAP per container to the bridge. State is recorded under `/run/containerization` so `cctl bridge delete` reverses exactly what create did. Bridge teardown verifies the link kind via sysfs to refuse deleting non-bridge interfaces. **cctl run / bridge** — end-to-end Linux container run path (image pull, ext4 rootfs assembly, VM boot, container exec) plus `cctl bridge create|delete` for the host network plumbing. **Build & dist** — `make linux-build` / `make linux-integration` build and exercise the host side inside an apple/container `--virtualization` dev container. `make dist-x86_64` produces a deployment tarball (cctl + cloud-hypervisor + virtiofsd + initfs + kernel) cross-compiled from the aarch64 dev container; pipeline documented in `docs/x86_64-build.md`. Static-musl C deps and the Zig cross compiler are pinned by SHA256. The host orchestrator runs as root. Per-VM runtime state lives under `/run/containerization/ch/<UUID>` with mode 0700; UDS sockets inside are bound with mode 0600. Vminitd's gRPC channel inherits that trust boundary — socket-file perms are the auth. Sandbox flags are upstream-secure by default. Two per-component opt-outs exist for the apple/container dev-container case (where the host seccomp profile SIGSYS-kills CH and virtiofsd): - `CONTAINERIZATION_NO_CH_SECCOMP=1` — `cloud-hypervisor --seccomp false`. - `CONTAINERIZATION_NO_VIRTIOFSD_SANDBOX=1` — `virtiofsd --sandbox none`. Each emits a one-shot `logger.warning` at process start. Legacy alias `CONTAINERIZATION_RELAXED_SANDBOX=1` flips both. cctl spawns both binaries with `setsid` and a minimal env allowlist (PATH / HOME / RUST_LOG / RUST_BACKTRACE) so the parent's secrets don't leak to children. `make linux-integration` runs the cross-platform integration suite against a real cloud-hypervisor VM inside the dev container. Linux runs the cross-platform subset (`process true`/`false`/`echo hi`, virtiofs round-trip, hotplug); the macOS suite is unchanged. Signed-off-by: michael_crosby <michael_crosby@apple.com>
453 lines
16 KiB
Swift
453 lines
16 KiB
Swift
//===----------------------------------------------------------------------===//
|
|
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// https://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
import ContainerizationError
|
|
import Foundation
|
|
|
|
#if os(macOS)
|
|
import Virtualization
|
|
#endif
|
|
|
|
#if os(Linux)
|
|
#if canImport(Musl)
|
|
import Musl
|
|
#elseif canImport(Glibc)
|
|
import Glibc
|
|
#endif
|
|
#endif
|
|
|
|
/// A filesystem mount exposed to a container.
|
|
public struct Mount: Sendable {
|
|
/// The filesystem or mount type. This is the string
|
|
/// that will be used for the mount syscall itself.
|
|
public var type: String
|
|
/// The source path of the mount.
|
|
public var source: String
|
|
/// The destination path of the mount.
|
|
public var destination: String
|
|
/// Filesystem or mount specific options.
|
|
public var options: [String]
|
|
/// Runtime specific options. This can be used
|
|
/// as a way to discern what kind of device a vmm
|
|
/// should create for this specific mount (virtioblock
|
|
/// virtiofs etc.).
|
|
public let runtimeOptions: RuntimeOptions
|
|
|
|
/// A type representing a "hint" of what type
|
|
/// of mount this really is (block, directory, purely
|
|
/// guest mount) and a set of type specific options, if any.
|
|
public enum RuntimeOptions: Sendable {
|
|
case virtioblk([String])
|
|
case virtiofs([String])
|
|
case shared
|
|
case any([String])
|
|
}
|
|
|
|
public init(
|
|
type: String,
|
|
source: String,
|
|
destination: String,
|
|
options: [String],
|
|
runtimeOptions: RuntimeOptions
|
|
) {
|
|
self.type = type
|
|
self.source = source
|
|
self.destination = destination
|
|
self.options = options
|
|
self.runtimeOptions = runtimeOptions
|
|
}
|
|
|
|
/// Mount representing a virtio block device.
|
|
public static func block(
|
|
format: String,
|
|
source: String,
|
|
destination: String,
|
|
options: [String] = [],
|
|
runtimeOptions: [String] = []
|
|
) -> Self {
|
|
.init(
|
|
type: format,
|
|
source: source,
|
|
destination: destination,
|
|
options: options,
|
|
runtimeOptions: .virtioblk(runtimeOptions)
|
|
)
|
|
}
|
|
|
|
/// Mount representing a virtiofs share.
|
|
public static func share(
|
|
source: String,
|
|
destination: String,
|
|
options: [String] = [],
|
|
runtimeOptions: [String] = []
|
|
) -> Self {
|
|
.init(
|
|
type: "virtiofs",
|
|
source: source,
|
|
destination: destination,
|
|
options: options,
|
|
runtimeOptions: .virtiofs(runtimeOptions)
|
|
)
|
|
}
|
|
|
|
/// A generic mount.
|
|
public static func any(
|
|
type: String,
|
|
source: String,
|
|
destination: String,
|
|
options: [String] = [],
|
|
runtimeOptions: [String] = []
|
|
) -> Self {
|
|
.init(
|
|
type: type,
|
|
source: source,
|
|
destination: destination,
|
|
options: options,
|
|
runtimeOptions: .any(runtimeOptions)
|
|
)
|
|
}
|
|
|
|
/// A mount referencing a shared pod volume by name.
|
|
public static func sharedMount(
|
|
name: String,
|
|
destination: String,
|
|
options: [String] = []
|
|
) -> Self {
|
|
.init(
|
|
type: "none",
|
|
source: name,
|
|
destination: destination,
|
|
options: options,
|
|
runtimeOptions: .shared
|
|
)
|
|
}
|
|
|
|
/// Clone the Mount to the provided path.
|
|
///
|
|
/// On macOS this uses `clonefile` (via `FileManager.copyItem`) for a
|
|
/// copy-on-write copy when the underlying filesystem supports it. On
|
|
/// Linux it tries `ioctl(FICLONE)` first (CoW on btrfs / xfs / bcachefs)
|
|
/// and falls back to a `SEEK_DATA`/`SEEK_HOLE` sparse copy that copies
|
|
/// only data ranges. This matters for EXT4 images produced by
|
|
/// `EXT4+Formatter`, which sparse-allocate via `lseek + 1-byte write` —
|
|
/// a non-sparse copy would inflate a ~50 MB alpine rootfs into a
|
|
/// fully-allocated 2 GiB clone and exhaust the integration suite's
|
|
/// writable layer in ~30 tests.
|
|
public func clone(to: String) throws -> Self {
|
|
#if os(Linux)
|
|
try Self.linuxSparseCopy(from: self.source, to: to)
|
|
#else
|
|
try FileManager.default.copyItem(atPath: self.source, toPath: to)
|
|
#endif
|
|
|
|
return .init(
|
|
type: self.type,
|
|
source: to,
|
|
destination: self.destination,
|
|
options: self.options,
|
|
runtimeOptions: self.runtimeOptions
|
|
)
|
|
}
|
|
|
|
#if os(Linux)
|
|
/// Copy `src` to `dst`, preferring a CoW reflink (`ioctl(FICLONE)`) and
|
|
/// falling back to a SEEK_DATA/SEEK_HOLE sparse copy. The reflink path
|
|
/// succeeds on btrfs / xfs (`reflink=1`) / bcachefs; on ext4 / tmpfs /
|
|
/// overlayfs it fails fast with EOPNOTSUPP/EXDEV/EINVAL and we walk
|
|
/// the hole map instead. The sparse-copy path also handles
|
|
/// filesystems that don't support hole-seeking (the very first
|
|
/// SEEK_DATA returns EINVAL) by copying the remainder verbatim. Mode
|
|
/// bits are preserved from the source.
|
|
private static func linuxSparseCopy(from src: String, to dst: String) throws {
|
|
// Stable Linux ABI since 3.1 (ext4, tmpfs, overlayfs all support it).
|
|
// Re-declared here so the build doesn't depend on whether the
|
|
// Glibc/Musl Swift overlay re-exports them.
|
|
let SEEK_DATA: Int32 = 3
|
|
let SEEK_HOLE: Int32 = 4
|
|
// _IOW(0x94, 9, int) on every Linux arch we target (x86_64, aarch64).
|
|
let FICLONE: CUnsignedLong = 0x4004_9409
|
|
|
|
let srcFd = open(src, O_RDONLY | O_CLOEXEC)
|
|
guard srcFd >= 0 else {
|
|
throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
|
|
}
|
|
defer { _ = close(srcFd) }
|
|
|
|
var st = stat()
|
|
guard fstat(srcFd, &st) == 0 else {
|
|
throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
|
|
}
|
|
let size = off_t(st.st_size)
|
|
let mode = mode_t(st.st_mode & 0o7777)
|
|
|
|
let dstFd = open(dst, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, mode)
|
|
guard dstFd >= 0 else {
|
|
throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
|
|
}
|
|
defer { _ = close(dstFd) }
|
|
|
|
// FICLONE atomically replaces dst's contents with a CoW clone of
|
|
// src — sets size and contents in one shot, no ftruncate needed
|
|
// afterwards. On failure FICLONE guarantees dst is untouched, so
|
|
// we can safely fall through to the sparse-copy path. ioctl(2) is
|
|
// variadic; type-pun via a fixed-arity function pointer (same
|
|
// pattern as ContainerizationOS.Socket).
|
|
let ioctlFICLONE: @convention(c) (CInt, CUnsignedLong, CInt) -> CInt = ioctl
|
|
if ioctlFICLONE(dstFd, FICLONE, srcFd) == 0 {
|
|
return
|
|
}
|
|
|
|
// Set the destination size up front so any trailing hole survives —
|
|
// we only ever pwrite data ranges, never zero-fill.
|
|
guard ftruncate(dstFd, size) == 0 else {
|
|
throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
|
|
}
|
|
|
|
let bufSize = 1 << 20 // 1 MiB
|
|
let buf = UnsafeMutableRawPointer.allocate(byteCount: bufSize, alignment: 16)
|
|
defer { buf.deallocate() }
|
|
|
|
var pos: off_t = 0
|
|
while pos < size {
|
|
let dataStart = lseek(srcFd, pos, SEEK_DATA)
|
|
if dataStart < 0 {
|
|
// ENXIO: no more data — rest is hole, already covered by ftruncate.
|
|
if errno == ENXIO {
|
|
break
|
|
}
|
|
// EINVAL/ENOTSUP: filesystem doesn't support SEEK_DATA. Treat
|
|
// the remainder as one big data range and copy it verbatim.
|
|
try Self.copyRange(srcFd: srcFd, dstFd: dstFd, start: pos, end: size, buf: buf, bufSize: bufSize)
|
|
break
|
|
}
|
|
|
|
// SEEK_HOLE returns end-of-file when there's no trailing hole.
|
|
let dataEnd = lseek(srcFd, dataStart, SEEK_HOLE)
|
|
let endOff: off_t = dataEnd < 0 ? size : dataEnd
|
|
|
|
try Self.copyRange(srcFd: srcFd, dstFd: dstFd, start: dataStart, end: endOff, buf: buf, bufSize: bufSize)
|
|
pos = endOff
|
|
}
|
|
}
|
|
|
|
private static func copyRange(
|
|
srcFd: Int32,
|
|
dstFd: Int32,
|
|
start: off_t,
|
|
end: off_t,
|
|
buf: UnsafeMutableRawPointer,
|
|
bufSize: Int
|
|
) throws {
|
|
var off = start
|
|
while off < end {
|
|
let want = Int(min(off_t(bufSize), end - off))
|
|
let nread = pread(srcFd, buf, want, off)
|
|
if nread < 0 {
|
|
if errno == EINTR { continue }
|
|
throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
|
|
}
|
|
if nread == 0 {
|
|
// Source shorter than fstat reported — shouldn't happen, but
|
|
// bail rather than spin.
|
|
return
|
|
}
|
|
var written = 0
|
|
while written < nread {
|
|
let nwrite = pwrite(dstFd, buf.advanced(by: written), nread - written, off + off_t(written))
|
|
if nwrite < 0 {
|
|
if errno == EINTR { continue }
|
|
throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
|
|
}
|
|
written += nwrite
|
|
}
|
|
off += off_t(nread)
|
|
}
|
|
}
|
|
#endif
|
|
}
|
|
|
|
#if os(macOS)
|
|
|
|
extension Mount {
|
|
private enum StorageAttachmentType {
|
|
case diskImage
|
|
case networkBlockDevice
|
|
}
|
|
|
|
private var storageAttachmentType: StorageAttachmentType {
|
|
let nbdSchemes = ["nbd://", "nbds://", "nbd+unix://", "nbds+unix://"]
|
|
if nbdSchemes.contains(where: { self.source.hasPrefix($0) }) {
|
|
return .networkBlockDevice
|
|
}
|
|
return .diskImage
|
|
}
|
|
|
|
func configure(config: inout VZVirtualMachineConfiguration) throws {
|
|
switch self.runtimeOptions {
|
|
case .virtioblk(let options):
|
|
let device: VZStorageDeviceAttachment
|
|
switch self.storageAttachmentType {
|
|
case .networkBlockDevice:
|
|
device = try VZNetworkBlockDeviceStorageDeviceAttachment.mountToVZAttachment(mount: self, options: options)
|
|
case .diskImage:
|
|
device = try VZDiskImageStorageDeviceAttachment.mountToVZAttachment(mount: self, options: options)
|
|
}
|
|
let attachment = VZVirtioBlockDeviceConfiguration(attachment: device)
|
|
config.storageDevices.append(attachment)
|
|
case .virtiofs(_):
|
|
// VirtioFS mounts are handled centrally via VZMultipleDirectoryShare in VZVirtualMachineInstance
|
|
// No per-mount device configuration needed
|
|
break
|
|
case .shared, .any:
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
extension VZDiskImageStorageDeviceAttachment {
|
|
static func mountToVZAttachment(mount: Mount, options: [String]) throws -> VZDiskImageStorageDeviceAttachment {
|
|
var synchronizationMode: VZDiskImageSynchronizationMode = .fsync
|
|
var cachingMode: VZDiskImageCachingMode = .cached
|
|
|
|
for option in options {
|
|
let split = option.split(separator: "=")
|
|
if split.count != 2 {
|
|
continue
|
|
}
|
|
|
|
let key = String(split[0])
|
|
let value = String(split[1])
|
|
|
|
switch key {
|
|
case "vzDiskImageCachingMode":
|
|
switch value {
|
|
case "automatic":
|
|
cachingMode = .automatic
|
|
case "cached":
|
|
cachingMode = .cached
|
|
case "uncached":
|
|
cachingMode = .uncached
|
|
default:
|
|
throw ContainerizationError(
|
|
.invalidArgument,
|
|
message: "unknown vzDiskImageCachingMode value for virtio block device: \(value)"
|
|
)
|
|
}
|
|
case "vzDiskImageSynchronizationMode":
|
|
switch value {
|
|
case "full":
|
|
synchronizationMode = .full
|
|
case "fsync":
|
|
synchronizationMode = .fsync
|
|
case "none":
|
|
synchronizationMode = .none
|
|
default:
|
|
throw ContainerizationError(
|
|
.invalidArgument,
|
|
message: "unknown vzDiskImageSynchronizationMode value for virtio block device: \(value)"
|
|
)
|
|
}
|
|
default:
|
|
throw ContainerizationError(
|
|
.invalidArgument,
|
|
message: "unknown vmm option encountered: \(key)"
|
|
)
|
|
}
|
|
}
|
|
return try VZDiskImageStorageDeviceAttachment(
|
|
url: URL(filePath: mount.source),
|
|
readOnly: mount.readonly,
|
|
cachingMode: cachingMode,
|
|
synchronizationMode: synchronizationMode
|
|
)
|
|
}
|
|
}
|
|
|
|
extension VZNetworkBlockDeviceStorageDeviceAttachment {
|
|
static func mountToVZAttachment(mount: Mount, options: [String]) throws -> VZNetworkBlockDeviceStorageDeviceAttachment {
|
|
guard let url = URL(string: mount.source) else {
|
|
throw ContainerizationError(
|
|
.invalidArgument,
|
|
message: "invalid NBD URL: \(mount.source)"
|
|
)
|
|
}
|
|
|
|
var timeout: TimeInterval = 5
|
|
var synchronizationMode: VZDiskSynchronizationMode = .full
|
|
|
|
for option in options {
|
|
let split = option.split(separator: "=")
|
|
if split.count != 2 {
|
|
continue
|
|
}
|
|
|
|
let key = String(split[0])
|
|
let value = String(split[1])
|
|
|
|
switch key {
|
|
case "vzTimeout":
|
|
guard let t = TimeInterval(value) else {
|
|
throw ContainerizationError(
|
|
.invalidArgument,
|
|
message: "invalid vzTimeout value for NBD device: \(value)"
|
|
)
|
|
}
|
|
timeout = t
|
|
case "vzSynchronizationMode":
|
|
switch value {
|
|
case "full":
|
|
synchronizationMode = .full
|
|
case "none":
|
|
synchronizationMode = .none
|
|
default:
|
|
throw ContainerizationError(
|
|
.invalidArgument,
|
|
message: "unknown vzSynchronizationMode value for NBD device: \(value)"
|
|
)
|
|
}
|
|
default:
|
|
throw ContainerizationError(
|
|
.invalidArgument,
|
|
message: "unknown vmm option encountered: \(key)"
|
|
)
|
|
}
|
|
}
|
|
|
|
return try VZNetworkBlockDeviceStorageDeviceAttachment(
|
|
url: url,
|
|
timeout: timeout,
|
|
isForcedReadOnly: mount.readonly,
|
|
synchronizationMode: synchronizationMode
|
|
)
|
|
}
|
|
}
|
|
|
|
#endif
|
|
|
|
extension Mount {
|
|
fileprivate var readonly: Bool {
|
|
self.options.contains("ro")
|
|
}
|
|
|
|
/// Returns true if this mount is a virtio block device.
|
|
public var isBlock: Bool {
|
|
if case .virtioblk = self.runtimeOptions {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
}
|