tstest/tailmac: add customized macOS virtualization tooling (#13146)

updates tailcale/corp#22371

Adds custom macOS vm tooling.  See the README for
the general gist, but this will spin up VMs with unixgram
capable network interfaces listening to a named socket,
and with a virtio socket device for host-guest communication.

We can add other devices like consoles, serial, etc as needed.

The whole things is buildable with a single make command, and
everything is controllable via the command line using the TailMac
utility.

This should all be generally functional but takes a few shortcuts
with error handling and the like.  The virtio socket device support
has not been tested and may require some refinement.

Signed-off-by: Jonathan Nobels <jonathan@tailscale.com>

tstest/tailmac/Swift/Common/Config.swift

@@ -0,0 +1,125 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+import Foundation
+
+let kDefaultDiskSizeGb: Int64 = 72
+let kDefaultMemSizeGb: UInt64 = 72
+
+
+/// Represents a configuration for a virtual machine
+class Config: Codable {
+    var serverSocket = "/tmp/qemu-dgram.sock"
+    var memorySize = (kDefaultMemSizeGb * 1024 * 1024 * 1024) as UInt64
+    var mac = "52:cc:cc:cc:cc:01"
+    var ethermac = "52:cc:cc:cc:ce:01"
+    var port: UInt32 = 51009
+
+    // The virtual machines ID.  Also double as the directory name under which
+    // we will store configuration, block device, etc.
+    let vmID: String
+
+    required init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+        if let ethermac = try container.decodeIfPresent(String.self, forKey: .ethermac) {
+            self.ethermac = ethermac
+        }
+        if let serverSocket = try container.decodeIfPresent(String.self, forKey: .serverSocket) {
+            self.serverSocket = serverSocket
+        }
+        if let memorySize = try container.decodeIfPresent(UInt64.self, forKey: .memorySize) {
+            self.memorySize = memorySize
+        }
+        if let port = try container.decodeIfPresent(UInt32.self, forKey: .port) {
+            self.port = port
+        }
+        if let mac = try container.decodeIfPresent(String.self, forKey: .mac) {
+            self.mac = mac
+        }
+        if let vmID = try container.decodeIfPresent(String.self, forKey: .vmID) {
+            self.vmID = vmID
+        } else {
+            self.vmID = "default"
+        }
+    }
+
+    init(_ vmID: String = "default") {
+        self.vmID = vmID
+        let configFile = vmDataURL.appendingPathComponent("config.json")
+        if FileManager.default.fileExists(atPath: configFile.path()) {
+            print("Using config file at path \(configFile)")
+            if let jsonData = try? Data(contentsOf: configFile) {
+                let config = try! JSONDecoder().decode(Config.self, from: jsonData)
+                self.serverSocket = config.serverSocket
+                self.memorySize = config.memorySize
+                self.mac = config.mac
+                self.port = config.port
+                self.ethermac = config.ethermac
+            }
+        }
+    }
+
+    func persist() {
+        let configFile = vmDataURL.appendingPathComponent("config.json")
+        let data = try! JSONEncoder().encode(self)
+        try! data.write(to: configFile)
+    }
+
+    lazy var restoreImageURL: URL = {
+        vmBundleURL.appendingPathComponent("RestoreImage.ipsw")
+    }()
+
+    // The VM Data URL holds the specific files composing a unique VM guest instance
+    // By default, VM's are persisted at ~/VM.bundle/<vmID>
+    lazy var vmDataURL =  {
+        let dataURL = vmBundleURL.appendingPathComponent(vmID)
+        return dataURL
+    }()
+
+    lazy var auxiliaryStorageURL = {
+        vmDataURL.appendingPathComponent("AuxiliaryStorage")
+    }()
+
+    lazy var diskImageURL = {
+        vmDataURL.appendingPathComponent("Disk.img")
+    }()
+
+    lazy var diskSize: Int64 = {
+        kDefaultDiskSizeGb * 1024 * 1024 * 1024
+    }()
+
+    lazy var hardwareModelURL = {
+        vmDataURL.appendingPathComponent("HardwareModel")
+    }()
+
+    lazy var machineIdentifierURL = {
+        vmDataURL.appendingPathComponent("MachineIdentifier")
+    }()
+
+    lazy var saveFileURL = {
+        vmDataURL.appendingPathComponent("SaveFile.vzvmsave")
+    }()
+
+}
+
+// The VM Bundle URL holds the restore image and a set of VM images
+// By default, VM's are persisted at ~/VM.bundle
+var vmBundleURL: URL = {
+    let vmBundlePath = NSHomeDirectory() + "/VM.bundle/"
+    createDir(vmBundlePath)
+    let bundleURL = URL(fileURLWithPath: vmBundlePath)
+    return bundleURL
+}()
+
+
+func createDir(_ path: String) {
+    do {
+        try FileManager.default.createDirectory(atPath: path, withIntermediateDirectories: true)
+    } catch {
+        fatalError("Unable to create dir at \(path) \(error)")
+    }
+}
+
+
+
+

tstest/tailmac/Swift/Common/Notifications.swift

@@ -0,0 +1,12 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+import Foundation
+
+struct Notifications {
+    // Stops the virtual machine and saves its state
+    static var stop = Notification.Name("io.tailscale.macvmhost.stop")
+
+    // Pauses the virtual machine and exits without saving its state
+    static var halt = Notification.Name("io.tailscale.macvmhost.halt")
+}

tstest/tailmac/Swift/Common/TailMacConfigHelper.swift

@@ -0,0 +1,145 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+import Foundation
+import Virtualization
+
+struct TailMacConfigHelper {
+    let config: Config
+
+    func computeCPUCount() -> Int {
+        let totalAvailableCPUs = ProcessInfo.processInfo.processorCount
+
+        var virtualCPUCount = totalAvailableCPUs <= 1 ? 1 : totalAvailableCPUs - 1
+        virtualCPUCount = max(virtualCPUCount, VZVirtualMachineConfiguration.minimumAllowedCPUCount)
+        virtualCPUCount = min(virtualCPUCount, VZVirtualMachineConfiguration.maximumAllowedCPUCount)
+
+        return virtualCPUCount
+    }
+
+    func computeMemorySize() -> UInt64 {
+        // Set the amount of system memory to 4 GB; this is a baseline value
+        // that you can change depending on your use case.
+        var memorySize = config.memorySize
+        memorySize = max(memorySize, VZVirtualMachineConfiguration.minimumAllowedMemorySize)
+        memorySize = min(memorySize, VZVirtualMachineConfiguration.maximumAllowedMemorySize)
+
+        return memorySize
+    }
+
+    func createBootLoader() -> VZMacOSBootLoader {
+        return VZMacOSBootLoader()
+    }
+
+    func createGraphicsDeviceConfiguration() -> VZMacGraphicsDeviceConfiguration {
+        let graphicsConfiguration = VZMacGraphicsDeviceConfiguration()
+        graphicsConfiguration.displays = [
+            // The system arbitrarily chooses the resolution of the display to be 1920 x 1200.
+            VZMacGraphicsDisplayConfiguration(widthInPixels: 1920, heightInPixels: 1200, pixelsPerInch: 80)
+        ]
+
+        return graphicsConfiguration
+    }
+
+    func createBlockDeviceConfiguration() -> VZVirtioBlockDeviceConfiguration {
+        do {
+            let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(url: config.diskImageURL, readOnly: false)
+            let disk = VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
+            return disk
+        } catch {
+            fatalError("Failed to create Disk image. \(error)")
+        }
+    }
+
+    func createSocketDeviceConfiguration() -> VZVirtioSocketDeviceConfiguration {
+       return VZVirtioSocketDeviceConfiguration()
+    }
+
+    func createNetworkDeviceConfiguration() -> VZVirtioNetworkDeviceConfiguration {
+        let networkDevice = VZVirtioNetworkDeviceConfiguration()
+        networkDevice.macAddress = VZMACAddress(string: config.ethermac)!
+
+        /* Bridged networking requires special entitlements from Apple
+         if let interface = VZBridgedNetworkInterface.networkInterfaces.first(where: { $0.identifier == "en0" }) {
+            let networkAttachment = VZBridgedNetworkDeviceAttachment(interface: interface)
+            networkDevice.attachment = networkAttachment
+         } else {
+            print("Assuming en0 for bridged ethernet.  Could not findd adapter")
+         }*/
+
+        /// But we can do NAT without Tim Apple's approval
+        let networkAttachment = VZNATNetworkDeviceAttachment()
+        networkDevice.attachment = networkAttachment
+
+        return networkDevice
+    }
+
+    func createSocketNetworkDeviceConfiguration() -> VZVirtioNetworkDeviceConfiguration {
+        let networkDevice = VZVirtioNetworkDeviceConfiguration()
+        networkDevice.macAddress = VZMACAddress(string: config.mac)!
+
+        let socket = Darwin.socket(AF_UNIX, SOCK_DGRAM, 0)
+
+        // Outbound network packets
+        let serverSocket = config.serverSocket
+
+        // Inbound network packets
+        let clientSockId = config.vmID
+        let clientSocket = "/tmp/qemu-dgram-\(clientSockId).sock"
+
+        unlink(clientSocket)
+        var clientAddr = sockaddr_un()
+        clientAddr.sun_family = sa_family_t(AF_UNIX)
+        clientSocket.withCString { ptr in
+            withUnsafeMutablePointer(to: &clientAddr.sun_path.0) { dest in
+                _ = strcpy(dest, ptr)
+            }
+        }
+
+        let bindRes = Darwin.bind(socket,
+                                  withUnsafePointer(to: &clientAddr, { $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { $0 } }),
+                                  socklen_t(MemoryLayout<sockaddr_un>.size))
+
+        if bindRes == -1 {
+            print("Error binding virtual network client socket - \(String(cString: strerror(errno)))")
+            return networkDevice
+        }
+
+        var serverAddr = sockaddr_un()
+        serverAddr.sun_family = sa_family_t(AF_UNIX)
+        serverSocket.withCString { ptr in
+            withUnsafeMutablePointer(to: &serverAddr.sun_path.0) { dest in
+                _ = strcpy(dest, ptr)
+            }
+        }
+
+        let connectRes = Darwin.connect(socket,
+                                        withUnsafePointer(to: &serverAddr, { $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { $0 } }),
+                                        socklen_t(MemoryLayout<sockaddr_un>.size))
+
+        if connectRes == -1 {
+            print("Error binding virtual network server socket - \(String(cString: strerror(errno)))")
+            return networkDevice
+        }
+
+        print("Virtual if mac address is \(config.mac)")
+        print("Client bound to \(clientSocket)")
+        print("Connected to server at \(serverSocket)")
+        print("Socket fd is \(socket)")
+
+
+        let handle = FileHandle(fileDescriptor: socket)
+        let device = VZFileHandleNetworkDeviceAttachment(fileHandle: handle)
+        networkDevice.attachment = device
+        return networkDevice
+    }
+
+    func createPointingDeviceConfiguration() -> VZPointingDeviceConfiguration {
+        return VZMacTrackpadConfiguration()
+    }
+
+    func createKeyboardConfiguration() -> VZKeyboardConfiguration {
+        return VZMacKeyboardConfiguration()
+    }
+}
+