From b77bbb60d01beddee6127dbc7fbe567da482d1e9 Mon Sep 17 00:00:00 2001 From: "Trevor L. McDonell" Date: Sun, 21 Dec 2025 12:43:55 +0100 Subject: [PATCH 1/7] fix: restrict SIMD extension to types where Scalar is also Generic This is implicitly true becaues SIMDScalar is limited to scalar types, but this is more precise and resistent to future changes. --- Sources/MultiArray/ArrayData.swift | 14 +++++++------- Sources/MultiArray/Generic.swift | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Sources/MultiArray/ArrayData.swift b/Sources/MultiArray/ArrayData.swift index e55f319..331411e 100644 --- a/Sources/MultiArray/ArrayData.swift +++ b/Sources/MultiArray/ArrayData.swift @@ -90,13 +90,13 @@ extension Float16: ArrayData {} extension Float32: ArrayData {} extension Float64: ArrayData {} -extension SIMD2: ArrayData {} -extension SIMD3: ArrayData {} -extension SIMD4: ArrayData {} -extension SIMD8: ArrayData {} -extension SIMD16: ArrayData {} -extension SIMD32: ArrayData {} -extension SIMD64: ArrayData {} +extension SIMD2: ArrayData where Scalar: Generic, Scalar.RawRepresentation: ArrayData {} +extension SIMD3: ArrayData where Scalar: Generic, Scalar.RawRepresentation: ArrayData {} +extension SIMD4: ArrayData where Scalar: Generic, Scalar.RawRepresentation: ArrayData {} +extension SIMD8: ArrayData where Scalar: Generic, Scalar.RawRepresentation: ArrayData {} +extension SIMD16: ArrayData where Scalar: Generic, Scalar.RawRepresentation: ArrayData {} +extension SIMD32: ArrayData where Scalar: Generic, Scalar.RawRepresentation: ArrayData {} +extension SIMD64: ArrayData where Scalar: Generic, Scalar.RawRepresentation: ArrayData {} public extension FixedWidthInteger { typealias Buffer = UnsafeMutablePointer diff --git a/Sources/MultiArray/Generic.swift b/Sources/MultiArray/Generic.swift index 895ad25..09ae0f0 100644 --- a/Sources/MultiArray/Generic.swift +++ b/Sources/MultiArray/Generic.swift @@ -103,7 +103,7 @@ public extension BinaryFloatingPoint { typealias RawRepresentation = Self } -public extension SIMD { +public extension SIMD where Scalar: Generic { typealias RawRepresentation = Self } From 944c023a1237605efeae454e9586dd9ae246e8f3 Mon Sep 17 00:00:00 2001 From: "Trevor L. McDonell" Date: Sun, 21 Dec 2025 15:09:17 +0100 Subject: [PATCH 2/7] fix: restrict ArrayData to only fixed-sized types This moves the platform dependent sized Int and UInt to the surface layer --- Benchmarks/Benchmarks/Benchmarks.swift | 2 +- Sources/MultiArray/ArrayData.swift | 2 -- Sources/MultiArray/Generic.swift | 45 ++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/Benchmarks/Benchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/Benchmarks.swift index f990341..81030bb 100644 --- a/Benchmarks/Benchmarks/Benchmarks.swift +++ b/Benchmarks/Benchmarks/Benchmarks.swift @@ -160,7 +160,7 @@ struct Zone: Randomizable & Generic { @inlinable init(from rep: RawRepresentation) { - self.id = rep._0 + self.id = Int(from: rep._0) self.position = Vec3(from: rep._1) } } diff --git a/Sources/MultiArray/ArrayData.swift b/Sources/MultiArray/ArrayData.swift index 331411e..c246091 100644 --- a/Sources/MultiArray/ArrayData.swift +++ b/Sources/MultiArray/ArrayData.swift @@ -69,14 +69,12 @@ extension ArrayData where Buffer == UnsafeMutablePointer { } // Primal types -extension Int: ArrayData {} extension Int8: ArrayData {} extension Int16: ArrayData {} extension Int32: ArrayData {} extension Int64: ArrayData {} @available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, *) extension Int128: ArrayData {} -extension UInt: ArrayData {} extension UInt8: ArrayData {} extension UInt16: ArrayData {} extension UInt32: ArrayData {} diff --git a/Sources/MultiArray/Generic.swift b/Sources/MultiArray/Generic.swift index 09ae0f0..1782a0b 100644 --- a/Sources/MultiArray/Generic.swift +++ b/Sources/MultiArray/Generic.swift @@ -51,15 +51,13 @@ extension Generic where RawRepresentation == Self { @attached(extension, conformances: Generic, names: arbitrary) public macro Generic() = #externalMacro(module: "MultiArrayMacros", type: "GenericExtensionMacro") -// Primal types -extension Int: Generic {} +// Primal, fixed size types extension Int8: Generic {} extension Int16: Generic {} extension Int32: Generic {} extension Int64: Generic {} @available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, *) extension Int128: Generic {} -extension UInt: Generic {} extension UInt8: Generic {} extension UInt16: Generic {} extension UInt32: Generic {} @@ -81,6 +79,47 @@ extension SIMD16: Generic {} extension SIMD32: Generic {} extension SIMD64: Generic {} +// Primal, platform dependent sized types +extension Int: Generic { + #if arch(x86_64) || arch(arm64) + public typealias RawRepresentation = Int64 + #else + public typealias RawRepresentation = Int32 + #endif + + @inlinable + @_alwaysEmitIntoClient + public var rawRepresentation: RawRepresentation { RawRepresentation(self) } + + @inlinable + @_alwaysEmitIntoClient + public init(from rep: RawRepresentation) { + assert(MemoryLayout.size == MemoryLayout.size) + assert(MemoryLayout.stride == MemoryLayout.stride) + self = Int(rep) + } +} + +extension UInt: Generic { + #if arch(x86_64) || arch(arm64) + public typealias RawRepresentation = UInt64 + #else + public typealias RawRepresentation = UInt32 + #endif + + @inlinable + @_alwaysEmitIntoClient + public var rawRepresentation: RawRepresentation { RawRepresentation(self) } + + @inlinable + @_alwaysEmitIntoClient + public init(from rep: RawRepresentation) { + assert(MemoryLayout.size == MemoryLayout.size) + assert(MemoryLayout.stride == MemoryLayout.stride) + self = UInt(rep) + } +} + extension Bool: Generic { public typealias RawRepresentation = UInt8 From 12adcc9eb206bf685847477a703c0f04efd7b6cd Mon Sep 17 00:00:00 2001 From: "Trevor L. McDonell" Date: Thu, 18 Dec 2025 16:39:29 +0100 Subject: [PATCH 3/7] feat: initialise padding regions between chunks --- Sources/MultiArray/ArrayData.swift | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Sources/MultiArray/ArrayData.swift b/Sources/MultiArray/ArrayData.swift index c246091..d683d6f 100644 --- a/Sources/MultiArray/ArrayData.swift +++ b/Sources/MultiArray/ArrayData.swift @@ -267,6 +267,13 @@ internal func getRawSize(for _: T.Type, count: Int, from offset: Int) -> Int internal func reserveCapacity(for type: T.Type, count: Int, from context: inout UnsafeMutableRawPointer) -> UnsafeMutablePointer { let begin = context.alignedUp(for: type) let end = begin + count * MemoryLayout.stride + let pad = begin - context + + // Initialise any gaps between the struct-of-array chunks + if pad > 0 { + context.initializeMemory(as: UInt8.self, repeating: 0x00, count: pad) + } + context = end return begin.bindMemory(to: type, capacity: count) } From 0603c8dac090926b9b141a30d51ea8f2d4fb30ab Mon Sep 17 00:00:00 2001 From: "Trevor L. McDonell" Date: Fri, 19 Dec 2025 11:42:44 +0100 Subject: [PATCH 4/7] feat: add binary encoding (memory dump) for unboxed types This explicitly excludes Box, so we are statically restricted to types we can fully encode in our struct-of-array representation. --- Sources/MultiArray/Extensions/Data.swift | 531 +++++++++++++++++++++++ 1 file changed, 531 insertions(+) create mode 100644 Sources/MultiArray/Extensions/Data.swift diff --git a/Sources/MultiArray/Extensions/Data.swift b/Sources/MultiArray/Extensions/Data.swift new file mode 100644 index 0000000..b04e774 --- /dev/null +++ b/Sources/MultiArray/Extensions/Data.swift @@ -0,0 +1,531 @@ +// Copyright (c) 2026 The swift-multiarray authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation + +extension MultiArray where Element.RawRepresentation: BinaryArrayData { + private static var magic: UInt32 { 0x4D_41_52_52 } // MARR swiftformat:disable:this numberFormatting + + /// Dump the given array into a Data buffer. + /// + /// This is statically restricted to types which we can encode fully in the + /// struct-of-array representation (i.e. no internal pointers, no internal + /// padding, etc.) and so serialisation and deserialisation are able to + /// efficiently copy the underlying buffer in one go. Note that this means + /// we do not do any endian conversion: you will get an error if you try to + /// decode the buffer on a machine with a different endianess than which it + /// was encoded. Thus, this is more a "memory snapshot" rather than a + /// serialised encoding, because the layout of the data in the buffer is + /// dependent on the host machine, library internals, etc. etc.. Still, + /// that's what enables it to be fast, for when you know what you are doing. + /// Caveat emptor. If this makes you anxious or you need a stable wire + /// format, consider using the Codable instances instead, e.g.: + /// + /// > let encoded = try JSONEncoder().encode(array) // slow as all hell + /// + /// Rather than this method: + /// + /// > let encoded = array.encode() // footguns! + /// + /// Caveats aside, examples this functionality is useful for include: + /// - cache snapshots + /// - same-machine IPC + /// - (temporary) persistent storage where you control both ends + /// + /// This encoding is only guaranteed to round-trip within the same major + /// version of this library, on ABI-compatible platforms. + /// + /// Currently this does not checksum the payload, but that could be added as + /// an optional addition. + /// + /// The format of the underlying data is: + /// + /// Bytes Description + /// ┌────────────── + /// 0...3 │ magic (0x4d415252) + /// 4 │ encoding version + /// 5 │ flags + /// 6...7 │ reserved + /// 8...15 │ array count + /// 16...17 │ size of type encoding (t) + /// 18... │ type encoding + /// ┆ + /// 18+t... │ payload + /// ┆ + /// └────────────── + /// + public func encode() -> Data { + let version: UInt8 = 1 + let headerSize = 18 + let typeSize = Element.RawRepresentation.type.encodedSize() + let payloadSize = Element.RawRepresentation.rawSize(capacity: self.count, from: 0) + var data = Data(capacity: headerSize + typeSize + payloadSize) + + // Header (18 bytes) + data.append(UInt32(MultiArray.magic)) + data.append(UInt8(version)) + data.append(UInt8(0)) // flags + data.append(UInt16(0)) // reserved + data.append(UInt64(self.count)) + data.append(UInt16(typeSize)) + + // Type encoding (typeSize bytes) + Element.RawRepresentation.appendType(to: &data) + + // Payload (payloadSize bytes) + data.append(self.arrayData.context.assumingMemoryBound(to: UInt8.self), count: payloadSize) + + return data + } + + /// Create an array from a Data dump. + /// + /// Note that this copies the data into a freshly allocated MultiArray. If + /// you need (or want) a zero-copy implementation please contact us on + /// GitHub to signal your interest! + /// + public init(data: Data) throws { + let expectedVersion: UInt8 = 1 + let expectedHeaderSize = 18 // of the expected version + + // Check we have at least a complete header (based on the expected type) + let expectedTypeSize = Element.RawRepresentation.type.encodedSize() + guard data.count >= expectedHeaderSize + expectedTypeSize else { + throw BinaryMultiArrayError.truncated(index: 0, required: expectedHeaderSize + expectedTypeSize, total: data.count) + } + + // Start decoding the header + let magic: UInt32 = try data.load(fromByteOffset: 0) + if magic == MultiArray.magic.byteSwapped { throw BinaryMultiArrayError.endianMismatch } + if magic != MultiArray.magic { throw BinaryMultiArrayError.badMagic } + + let version: UInt8 = try data.load(fromByteOffset: 4) + guard version == expectedVersion else { throw BinaryMultiArrayError.unsupportedVersion(Int(version)) } + + _ = try data.load(fromByteOffset: 5) as UInt8 // flags + _ = try data.load(fromByteOffset: 6) as UInt16 // reserved + + let count64: UInt64 = try data.load(fromByteOffset: 8) + guard count64 <= UInt64(Int.max) else { throw BinaryMultiArrayError.overflow(count64) } + let count = Int(count64) + + // Ensure we have the correct number of bytes + let expectedByteCount = Element.RawRepresentation.rawSize(capacity: count, from: 0) + let expectedSize = expectedHeaderSize + expectedTypeSize + expectedByteCount + guard data.count == expectedSize else { + throw BinaryMultiArrayError.sizeMismatch(expected: expectedSize, actual: data.count) + } + + // Decode the (variable sized) type tag + let typeSize16: UInt16 = try data.load(fromByteOffset: 16) + let typeSize = Int(typeSize16) + var offset = expectedHeaderSize + try Element.RawRepresentation.verifyType(in: data, at: &offset) + guard offset == 18 + typeSize else { + throw BinaryMultiArrayError.malformedType(expected: typeSize, actual: offset - 18) + } + + // Header verification is complete. + // Allocate the buffer for the payload and memcpy dirctly into it. + self.arrayData = .init(unsafeUninitializedCapacity: count) + data.withUnsafeBytes { + // This force unwrap is safe because we've already accessed the + // underlying Data pointer many times before this point, so it can + // not possibly be nil. + self.arrayData.context.copyMemory(from: $0.baseAddress! + offset, byteCount: expectedByteCount) + } + } +} + +public enum BinaryMultiArrayError: Error, Equatable, CustomStringConvertible { + case badMagic + case endianMismatch + case overflow(UInt64) + case unsupportedVersion(Int) + case truncated(index: Int, required: Int, total: Int) + case sizeMismatch(expected: Int, actual: Int) + case typeMismatch(expected: UInt8, actual: UInt8) + case malformedType(expected: Int, actual: Int) + + public var description: String { + switch self { + case .badMagic: + "Incorrect magic value. Are you sure this is MultiArray data?" + case .endianMismatch: + "Attempt to load data that was produced on a machine of different endian-ness. This is not supported." + case let .overflow(value): + "Encoded value overflowed available Int range: \(value)" + case let .unsupportedVersion(version): + "Unknown or unsupported encoding version: \(version)" + case let .truncated(index, required, total): + "Ran out of data at index \(index): required \(required) bytes but only \(total - index) remain" + case let .sizeMismatch(expected, actual): + "Size mismatch: expected \(expected) bytes but found \(actual) instead" + case let .typeMismatch(expected, actual): + String( + format: "Type tag mismatch: expected 0x%x (%@) but found 0x%x (%@)", + expected, + TypeHead(rawValue: expected)?.description ?? "invalid", + actual, + TypeHead(rawValue: actual)?.description ?? "invalid" + ) + case let .malformedType(expected, actual): + "Incomplete type encoding: expected \(expected) bytes but consumed \(actual)" + } + } +} + +extension Data { + @inlinable + mutating func append(_ value: T) { + var value = value + Swift.withUnsafeBytes(of: &value) { self.append($0.assumingMemoryBound(to: T.self)) } + } + + @inlinable + func load(fromByteOffset offset: Int) throws -> T { + let required = MemoryLayout.size + guard offset + required <= self.count else { + throw BinaryMultiArrayError.truncated(index: offset, required: required, total: self.count) + } + return self.withUnsafeBytes { $0.loadUnaligned(fromByteOffset: offset, as: T.self) } + } +} + +// The subset of ArrayData types that are safe to serialise as raw bytes. By +// "safe" we mean: +// - no heap pointers / references embedded in the representation. This is +// enforced by not providing a conformance to the Box escape hatch. +// - the representation's byte layout is fully determined by the type. +// - data written is deterministic, padding bytes included. This is true even +// for structure types due to the aforementioned exclusion of Box, thus there +// is no way to encode types that might include internal padding. +// +public protocol BinaryArrayData: ArrayData { + static var type: Type { get } + static var typeHead: TypeHead { get } + + // Match and consume this type's encoding from `data[index...]`. On success + // the index will point to the start of the payload section. + static func verifyType(in data: Data, at offset: inout Int) throws + + // Append the type tag into the Data buffer + static func appendType(to data: inout Data) +} + +extension BinaryArrayData { + static func verifyByte(expecting: UInt8, in data: Data, at offset: inout Int) throws { + guard offset < data.count else { + throw BinaryMultiArrayError.truncated(index: offset, required: 1, total: data.count) + } + let found = data[offset] + guard expecting == found else { + throw BinaryMultiArrayError.typeMismatch(expected: expecting, actual: found) + } + offset += 1 + } +} + +extension BinaryArrayData where Self: SignedInteger { + public static var type: Type { + .int(bits: UInt8(MemoryLayout.size * 8)) + } + + public static var typeHead: TypeHead { + .int(bits: UInt8(MemoryLayout.size * 8)) + } + + public static func verifyType(in data: Data, at offset: inout Int) throws { + try verifyByte(expecting: Self.typeHead.rawValue, in: data, at: &offset) + } + + public static func appendType(to data: inout Data) { + data.append(Self.typeHead.rawValue) + } +} + +extension BinaryArrayData where Self: UnsignedInteger { + public static var type: Type { + .uint(bits: UInt8(MemoryLayout.size * 8)) + } + + public static var typeHead: TypeHead { + .uint(bits: UInt8(MemoryLayout.size * 8)) + } + + public static func verifyType(in data: Data, at offset: inout Int) throws { + try verifyByte(expecting: Self.typeHead.rawValue, in: data, at: &offset) + } + + public static func appendType(to data: inout Data) { + data.append(Self.typeHead.rawValue) + } +} + +extension BinaryArrayData where Self: BinaryFloatingPoint { + public static var type: Type { + .float(bits: UInt8(MemoryLayout.size * 8)) + } + + public static var typeHead: TypeHead { + .float(bits: UInt8(MemoryLayout.size * 8)) + } + + public static func verifyType(in data: Data, at offset: inout Int) throws { + try verifyByte(expecting: Self.typeHead.rawValue, in: data, at: &offset) + } + + public static func appendType(to data: inout Data) { + data.append(Self.typeHead.rawValue) + } +} + +extension BinaryArrayData where Self: SIMD, Self.Scalar: Generic, Self.Scalar.RawRepresentation: BinaryArrayData { + public static var type: Type { + .simd(lanes: UInt8(Self.scalarCount), element: Self.Scalar.RawRepresentation.type) + } + + public static var typeHead: TypeHead { + .simd(lanes: UInt8(Self.scalarCount)) + } + + public static func verifyType(in data: Data, at offset: inout Int) throws { + try verifyByte(expecting: Self.typeHead.rawValue, in: data, at: &offset) + try Self.Scalar.RawRepresentation.verifyType(in: data, at: &offset) + } + + public static func appendType(to data: inout Data) { + data.append(Self.typeHead.rawValue) + Self.Scalar.RawRepresentation.appendType(to: &data) + } +} + +// Primal types. MemoryLayout.size == MemoryLayout.stride +extension Int8: BinaryArrayData {} +extension Int16: BinaryArrayData {} +extension Int32: BinaryArrayData {} +extension Int64: BinaryArrayData {} +@available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, *) +extension Int128: BinaryArrayData {} +extension UInt8: BinaryArrayData {} +extension UInt16: BinaryArrayData {} +extension UInt32: BinaryArrayData {} +extension UInt64: BinaryArrayData {} +@available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, *) +extension UInt128: BinaryArrayData {} +#if arch(arm64) +@available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) +extension Float16: BinaryArrayData {} +#endif +extension Float32: BinaryArrayData {} +extension Float64: BinaryArrayData {} + +extension SIMD2: BinaryArrayData where Scalar: Generic, Scalar.RawRepresentation: BinaryArrayData {} +extension SIMD3: BinaryArrayData where Scalar: Generic, Scalar.RawRepresentation: BinaryArrayData {} +extension SIMD4: BinaryArrayData where Scalar: Generic, Scalar.RawRepresentation: BinaryArrayData {} +extension SIMD8: BinaryArrayData where Scalar: Generic, Scalar.RawRepresentation: BinaryArrayData {} +extension SIMD16: BinaryArrayData where Scalar: Generic, Scalar.RawRepresentation: BinaryArrayData {} +extension SIMD32: BinaryArrayData where Scalar: Generic, Scalar.RawRepresentation: BinaryArrayData {} +extension SIMD64: BinaryArrayData where Scalar: Generic, Scalar.RawRepresentation: BinaryArrayData {} + +// Datatype-generic markers +extension Unit: BinaryArrayData { + public static var type: Type { .unit } + public static var typeHead: TypeHead { .unit } + + public static func verifyType(in data: Data, at offset: inout Int) throws { + try verifyByte(expecting: Self.typeHead.rawValue, in: data, at: &offset) + } + + public static func appendType(to data: inout Data) { + data.append(Self.typeHead.rawValue) + } +} + +// extension Box: BinaryArrayData { } -- not allowed; may include pointers, etc. + +extension Product: BinaryArrayData where A: BinaryArrayData, B: BinaryArrayData { + public static var type: Type { + .product(lhs: A.type, rhs: B.type) + } + + public static var typeHead: TypeHead { + .product + } + + public static func verifyType(in data: Data, at offset: inout Int) throws { + try verifyByte(expecting: Self.typeHead.rawValue, in: data, at: &offset) + try A.verifyType(in: data, at: &offset) + try B.verifyType(in: data, at: &offset) + } + + public static func appendType(to data: inout Data) { + data.append(Self.typeHead.rawValue) + A.appendType(to: &data) + B.appendType(to: &data) + } +} + +// A representation of types for our tiny closed universe +public indirect enum Type: Equatable, CustomStringConvertible { + case unit + case int(bits: UInt8) // 8, 16, 32, 64, 128 + case uint(bits: UInt8) // 8, 16, 32, 64, 128 + case float(bits: UInt8) // 16, 32, 64 + case simd(lanes: UInt8, element: Type) + case product(lhs: Type, rhs: Type) + + public var description: String { + switch self { + case .unit: "Unit" + case let .int(bits): "Int\(bits)" + case let .uint(bits): "UInt\(bits)" + case let .float(bits): "Float\(bits)" + case let .simd(lanes, element): "SIMD\(lanes)<\(element)>" + case let .product(lhs, rhs): "Product<\(lhs), \(rhs)>" + } + } + + func encodedSize() -> Int { + switch self { + case .unit, .int, .uint, .float: + return 1 + case let .simd(_, scalar): + return 1 + scalar.encodedSize() + case let .product(lhs, rhs): + return 1 + lhs.encodedSize() + rhs.encodedSize() + } + } +} + +// A partial type fragment (the non-recursive head of the type) +public enum TypeHead: RawRepresentable, Equatable, CustomStringConvertible { + case unit + case product + case int(bits: UInt8) // 8, 16, 32, 64, 128 + case uint(bits: UInt8) // 8, 16, 32, 64, 128 + case float(bits: UInt8) // 16, 32, 64 + case simd(lanes: UInt8) // 2, 3, 4, 8, 16, 32, 64 + + public var description: String { + switch self { + case .unit: "Unit" + case .product: "Product<...>" + case let .int(bits): "Int\(bits)" + case let .uint(bits): "UInt\(bits)" + case let .float(bits): "Float\(bits)" + case let .simd(lanes): "SIMD\(lanes)<...>" + } + } + + public typealias RawValue = UInt8 + public var rawValue: UInt8 { + switch self { + case .unit: + Self.encode(kind: .unit, parameter: 0) + case .product: + Self.encode(kind: .product, parameter: 0) + case let .int(bits): + Self.encode(kind: .int, parameter: Self.encode(bits: bits)) + case let .uint(bits): + Self.encode(kind: .uint, parameter: Self.encode(bits: bits)) + case let .float(bits): + Self.encode(kind: .float, parameter: Self.encode(bits: bits)) + case let .simd(lanes): + Self.encode(kind: .simd, parameter: Self.encode(lanes: lanes)) + } + } + + public init?(rawValue: UInt8) { + let opcode = rawValue >> 4 + let parameter = rawValue & 0x0f + + if let kind = Kind(rawValue: opcode) { + switch (kind, parameter) { + case (.unit, 0): + self = .unit + + case (.product, 0): + self = .product + + case let (.int, bits): + self = .int(bits: Self.decode(bits: bits)) + + case let (.uint, bits): + self = .uint(bits: Self.decode(bits: bits)) + + case let (.float, bits): + self = .float(bits: Self.decode(bits: bits)) + + case let (.simd, lanes): + self = .simd(lanes: Self.decode(lanes: lanes)) + + default: + return nil + } + } + else { + return nil + } + } + + enum Kind: UInt8 { + case unit = 0 + case int = 1 + case uint = 2 + case float = 3 + case simd = 4 + case product = 5 + } + + // Encode type kind and parameter as high and low nibble of a byte + static func encode(kind: Kind, parameter: UInt8) -> UInt8 { + precondition(parameter < 16) + precondition(kind.rawValue < 16) + return (kind.rawValue << 4) | (parameter & 0x0f) + } + + // Encode log2(bits) as a nibble + static func encode(bits: UInt8) -> UInt8 { + precondition(bits > 0 && (bits & (bits - 1) == 0), "must be a power-of-two") + return UInt8(bits.trailingZeroBitCount) + } + + // Encode SIMD lane count as a nibble + // Uses log2(lanes) for power-of-two lane counts, and 0 for lanes=3 (this + // bit is free because SIMD1 is dumb and not present in the Swift language). + static func encode(lanes: UInt8) -> UInt8 { + precondition(lanes > 1) + switch lanes { + case 3: + return 0 + default: + precondition(lanes & (lanes - 1) == 0, "must be a power-of-two") + return UInt8(lanes.trailingZeroBitCount) + } + } + + static func decode(bits: UInt8) -> UInt8 { + precondition(bits < 16) + return 1 << bits + } + + static func decode(lanes: UInt8) -> UInt8 { + precondition(lanes < 16) + return switch lanes { + case 0: 3 + default: 1 << lanes + } + } +} From 6cb9008ec134e46397e94ae9da454a529798713b Mon Sep 17 00:00:00 2001 From: "Trevor L. McDonell" Date: Fri, 19 Dec 2025 12:36:43 +0100 Subject: [PATCH 5/7] test: add binary encoding tests --- Tests/MultiArrayTests/MultiArrayTests.swift | 158 ++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/Tests/MultiArrayTests/MultiArrayTests.swift b/Tests/MultiArrayTests/MultiArrayTests.swift index fd71811..5bf35a9 100644 --- a/Tests/MultiArrayTests/MultiArrayTests.swift +++ b/Tests/MultiArrayTests/MultiArrayTests.swift @@ -300,6 +300,146 @@ struct MultiArrayTests { // Leaving the do scope should drop arr and release everything it owns #expect(Tracked.liveCount == 0) } + + @Suite + struct DataTests { + @Test + func failMagic() throws { + let array: MultiArray = [1, 2, 3] + var encoded = array.encode() + encoded[0] ^= 0xff + + #expect(throws: BinaryMultiArrayError.badMagic) { + _ = try MultiArray(data: encoded) + } + } + + @Test + func failVersion() throws { + let array: MultiArray = [1, 2, 3] + var encoded = array.encode() + encoded[4] = 99 + + #expect(throws: BinaryMultiArrayError.unsupportedVersion(99)) { + _ = try MultiArray(data: encoded) + } + } + + @Test + func failTruncated() throws { + let array: MultiArray = [] + let encoded = array.encode() + + // Ensure we truncate into the payload + #expect(encoded.count == 19) + + let truncated = encoded.prefix(encoded.count - 1) + #expect(throws: BinaryMultiArrayError.truncated(index: 0, required: 19, total: 18)) { + _ = try MultiArray(data: truncated) + } + } + + @Test + func failCountMismatch() throws { + let array: MultiArray = [10, 20, 30] + var encoded = array.encode() + + // count field offset (8 ..< 16) + let count = encoded.withUnsafeBytes { $0.load(fromByteOffset: 8, as: UInt64.self) } + #expect(count == 3) + + encoded[8] ^= 0x01 + #expect(throws: BinaryMultiArrayError.self) { + _ = try MultiArray(data: encoded) + } + } + + @Test + func failTypeMismatch() throws { + let array: MultiArray = [1, 2, 3] + let encoded = array.encode() + + #expect(throws: BinaryMultiArrayError.typeMismatch(expected: 0x35, actual: 0x15)) { + _ = try MultiArray(data: encoded) + } + } + + @Suite + struct RoundTripTests { + @Suite + struct ScalarTests { + @Test func testDataInt() throws { try roundtripDataTest(Int.self) } + @Test func testDataInt8() throws { try roundtripDataTest(Int8.self) } + @Test func testDataInt16() throws { try roundtripDataTest(Int16.self) } + @Test func testDataInt32() throws { try roundtripDataTest(Int32.self) } + @Test func testDataInt64() throws { try roundtripDataTest(Int64.self) } + @available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, *) + @Test func testDataInt128() throws { try roundtripDataTest(Int128.self) } + @Test func testDataUInt() throws { try roundtripDataTest(UInt.self) } + @Test func testDataUInt8() throws { try roundtripDataTest(UInt8.self) } + @Test func testDataUInt16() throws { try roundtripDataTest(UInt16.self) } + @Test func testDataUInt32() throws { try roundtripDataTest(UInt32.self) } + @Test func testDataUInt64() throws { try roundtripDataTest(UInt64.self) } + @available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, *) + @Test func testDataUInt128() throws { try roundtripDataTest(UInt128.self) } + #if arch(arm64) + @available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) + @Test func testDataFloat16() throws { try roundtripDataTest(Float16.self) } + #endif + @Test func testDataFloat32() throws { try roundtripDataTest(Float32.self) } + @Test func testDataFloat64() throws { try roundtripDataTest(Float64.self) } + @Test func testDataBool() throws { try roundtripDataTest(Bool.self) } + } + + @Suite + struct StructTests { + @Test func testDataPoint() throws { try roundtripDataTest(Point.self) } + @Test func testDataZone() throws { try roundtripDataTest(Zone.self) } + @Test func testDataUUID() throws { try roundtripDataTest(UUID.self) } + @Test func testDataDate() throws { try roundtripDataTest(Date.self) } + } + + @Suite + struct SIMD2Tests { + @Test func testDataSIMD2Int() throws { try roundtripDataTest(SIMD2.self) } + @Test func testDataSIMD2Int8() throws { try roundtripDataTest(SIMD2.self) } + @Test func testDataSIMD2Int16() throws { try roundtripDataTest(SIMD2.self) } + @Test func testDataSIMD2Int32() throws { try roundtripDataTest(SIMD2.self) } + @Test func testDataSIMD2Int64() throws { try roundtripDataTest(SIMD2.self) } + @Test func testDataSIMD2UInt() throws { try roundtripDataTest(SIMD2.self) } + @Test func testDataSIMD2UInt8() throws { try roundtripDataTest(SIMD2.self) } + @Test func testDataSIMD2UInt16() throws { try roundtripDataTest(SIMD2.self) } + @Test func testDataSIMD2UInt32() throws { try roundtripDataTest(SIMD2.self) } + @Test func testDataSIMD2UInt64() throws { try roundtripDataTest(SIMD2.self) } + #if arch(arm64) + @available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) + @Test func testDataSIMD2Float16() throws { try roundtripDataTest(SIMD2.self) } + #endif + @Test func testDataSIMD2Float32() throws { try roundtripDataTest(SIMD2.self) } + @Test func testDataSIMD2Float64() throws { try roundtripDataTest(SIMD2.self) } + } + + @Suite + struct SIMD3Tests { + @Test func testDataSIMD3Int() throws { try roundtripDataTest(SIMD3.self) } + @Test func testDataSIMD3Int8() throws { try roundtripDataTest(SIMD3.self) } + @Test func testDataSIMD3Int16() throws { try roundtripDataTest(SIMD3.self) } + @Test func testDataSIMD3Int32() throws { try roundtripDataTest(SIMD3.self) } + @Test func testDataSIMD3Int64() throws { try roundtripDataTest(SIMD3.self) } + @Test func testDataSIMD3UInt() throws { try roundtripDataTest(SIMD3.self) } + @Test func testDataSIMD3UInt8() throws { try roundtripDataTest(SIMD3.self) } + @Test func testDataSIMD3UInt16() throws { try roundtripDataTest(SIMD3.self) } + @Test func testDataSIMD3UInt32() throws { try roundtripDataTest(SIMD3.self) } + @Test func testDataSIMD3UInt64() throws { try roundtripDataTest(SIMD3.self) } + #if arch(arm64) + @available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) + @Test func testDataSIMD3Float16() throws { try roundtripDataTest(SIMD3.self) } + #endif + @Test func testDataSIMD3Float32() throws { try roundtripDataTest(SIMD3.self) } + @Test func testDataSIMD3Float64() throws { try roundtripDataTest(SIMD3.self) } + } + } + } } func roundtripTest(_: T.Type, iterations: Int = 1000) @@ -341,3 +481,21 @@ func roundtripCodableTest(_: T. #expect(original == decoded) } } + +func roundtripDataTest(_: T.Type, iterations: Int = 1000) throws + where T.RawRepresentation: BinaryArrayData +{ + var generator = SystemRandomNumberGenerator() + let step = 100 / Double(iterations) + + for i in 0 ..< iterations { + let size: Int64 = min(99, Int64((Double(i) * step).rounded(.towardZero))) + let length = Int.random(in: linear(from: 0, to: 1024)(size), using: &generator) + let original: MultiArray = MultiArray(randomArray(count: length, using: &generator)) + + let encoded = original.encode() + let decoded = try MultiArray(data: encoded) + + #expect(original == decoded) + } +} From efa5da5b7fcc15fe88d365d6ccd81c27495755f9 Mon Sep 17 00:00:00 2001 From: "Trevor L. McDonell" Date: Fri, 19 Dec 2025 12:40:47 +0100 Subject: [PATCH 6/7] test: add missing Codable tests --- Tests/MultiArrayTests/MultiArrayTests.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/MultiArrayTests/MultiArrayTests.swift b/Tests/MultiArrayTests/MultiArrayTests.swift index 5bf35a9..0dbc6f9 100644 --- a/Tests/MultiArrayTests/MultiArrayTests.swift +++ b/Tests/MultiArrayTests/MultiArrayTests.swift @@ -227,6 +227,8 @@ struct MultiArrayTests { struct StructTests { @Test func testCodablePoint() throws { try roundtripCodableTest(Point.self) } @Test func testCodableZone() throws { try roundtripCodableTest(Zone.self) } + @Test func testCodableUUID() throws { try roundtripCodableTest(UUID.self) } + @Test func testCodableDate() throws { try roundtripCodableTest(Date.self) } } @Suite From 31455daf1714a5df6750e22b21d4d2a67a718fed Mon Sep 17 00:00:00 2001 From: "Trevor L. McDonell" Date: Fri, 19 Dec 2025 12:51:00 +0100 Subject: [PATCH 7/7] test: ensure we have a type with some internal padding --- Tests/MultiArrayTests/Zone.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tests/MultiArrayTests/Zone.swift b/Tests/MultiArrayTests/Zone.swift index a4dc09e..2b83a16 100644 --- a/Tests/MultiArrayTests/Zone.swift +++ b/Tests/MultiArrayTests/Zone.swift @@ -15,12 +15,12 @@ import MultiArray struct Zone: Generic, Equatable, Randomizable, Codable { - typealias RawRepresentation = T2>.RawRepresentation + typealias RawRepresentation = T2>.RawRepresentation - let id: Int + let id: Int8 let position: Vec3 - init(id: Int, position: Vec3) { + init(id: Int8, position: Vec3) { self.id = id self.position = position } @@ -39,7 +39,7 @@ struct Zone: Generic, Equatable, Randomizable, Codable { static func random(using generator: inout T) -> Self { Zone( - id: Int.random(using: &generator), + id: Int8.random(using: &generator), position: Vec3.random(using: &generator) ) }