Fastest way to save structs iOS / Swift

Question

I have structs like

struct RGBA: Codable {
        
   var r: UInt8
   var g: UInt8
   var b: UInt8
   var a: UInt8 
}

I want save large amount of this structs (>1_000_000)

Decode

guard let history = try? JSONDecoder().decode(HistoryRGBA.self, from: data) else { return }

Encode

guard let jsonData = try? encoder.encode(dataForSave) else { return false }

How can I improve encoding/decoding time and amount of RAM memory?

Answer 1

The performance of JSONEncoder/Decoder performance is...not great. ZippyJSON is a drop-in replacement that is supposedly about 4 times faster than Foundation's implmenetation, and if you're going for better performance and lower memory usage, you'll probably want to Google for some kind of streaming JSON decoder library.

However, you said in the comments that you don't need the JSON format. That's great, because we can store the data much more efficiently as just an array of raw bytes rather than a text-based format such as JSON:

extension RGBA {
    static let size = 4 // the size of a (packed) RGBA structure
}

// encoding
var data = Data(count: history.rgba.count * RGBA.size)
for i in 0..<history.rgba.count {
    let rgba = history.rgba[i]
    data[i*RGBA.size] = rgba.r
    data[i*RGBA.size+1] = rgba.g
    data[i*RGBA.size+2] = rgba.b
    data[i*RGBA.size+3] = rgba.a
}


// decoding
guard data.count % RGBA.size == 0 else {
    // data is incomplete, handle error
    return
}
let rgbaCount = data.count / RGBA.size
var result = [RGBA]()
result.reserveCapacity(rgbaCount)
for i in 0..<rgbaCount {
    result.append(RGBA(r: data[i*RGBA.size],
                       g: data[i*RGBA.size+1],
                       b: data[i*RGBA.size+2],
                       a: data[i*RGBA.size+3]))
}

This is already about 50 times faster than using JSONEncoder on my machine (~100ms instead of ~5 seconds).

You can get even faster by bypassing some of Swift's safety checks and memory management and dropping down to raw pointers:

// encoding
let byteCount = history.rgba.count * RGBA.size
let rawBuf = malloc(byteCount)!
let buf = rawBuf.bindMemory(to: UInt8.self, capacity: byteCount)

for i in 0..<history.rgba.count {
    let rgba = history.rgba[i]
    buf[i*RGBA.size] = rgba.r
    buf[i*RGBA.size+1] = rgba.g
    buf[i*RGBA.size+2] = rgba.b
    buf[i*RGBA.size+3] = rgba.a
}
let data = Data(bytesNoCopy: rawBuf, count: byteCount, deallocator: .free)


// decoding
guard data.count % RGBA.size == 0 else {
    // data is incomplete, handle error
    return
}
let result: [RGBA] = data.withUnsafeBytes { rawBuf in
    let buf = rawBuf.bindMemory(to: UInt8.self)
    let rgbaCount = buf.count / RGBA.size
    return [RGBA](unsafeUninitializedCapacity: rgbaCount) { resultBuf, initializedCount in
        for i in 0..<rgbaCount {
            resultBuf[i] = RGBA(r: data[i*RGBA.size],
                                g: data[i*RGBA.size+1],
                                b: data[i*RGBA.size+2],
                                a: data[i*RGBA.size+3])
        }
    }
}

Benchmark results on my machine (I did not test ZippyJSON):

JSON:
Encode: 4967.0ms; 32280478 bytes
Decode: 5673.0ms

Data:
Encode: 96.0ms; 4000000 bytes
Decode: 19.0ms

Pointers:
Encode: 1.0ms; 4000000 bytes
Decode: 18.0ms

You could probably get even faster by just writing your array directly from memory to disk without serializing it at all, although I haven't tested that either. And of course, when you're testing performance, be sure you're testing in Release mode.

Answer 2

Considering that all your properties are UInt8 (bytes) you can make your struct conform to ContiguousBytes and save its raw bytes:

struct RGBA {
   let r, g, b, a: UInt8
}

---

extension RGBA: ContiguousBytes {
    func withUnsafeBytes<R>(_ body: (UnsafeRawBufferPointer) throws -> R) rethrows -> R {
        try Swift.withUnsafeBytes(of: self) { try body($0) }
    }
}

---

extension ContiguousBytes {
    init<T: ContiguousBytes>(_ bytes: T) {
        self = bytes.withUnsafeBytes { $0.load(as: Self.self) }
    }
}

---

extension RGBA: ExpressibleByArrayLiteral {
    typealias ArrayLiteralElement = UInt8
    init(arrayLiteral elements: UInt8...) {
        self.init(elements)
    }
}

---

extension Array {
    var bytes: [UInt8] { withUnsafeBytes { .init($0) } }
    var data: Data { withUnsafeBytes { .init($0) } }
}

---

extension ContiguousBytes {
    var bytes: [UInt8] { withUnsafeBytes { .init($0) } }
    var data: Data { withUnsafeBytes { .init($0) } }
}

---

extension ContiguousBytes {
    func object<T>() -> T { withUnsafeBytes { $0.load(as: T.self) } }
    func objects<T>() -> [T] { withUnsafeBytes { .init($0.bindMemory(to: T.self)) } }
}

---

extension ContiguousBytes {
    var rgba: RGBA { object() }
    var rgbaCollection: [RGBA] { objects() }
}

---

extension UIColor {
    convenience init<T: Collection>(_ bytes: T) where T.Index == Int, T.Element == UInt8 {
        self.init(red:   CGFloat(bytes[0])/255,
                  green: CGFloat(bytes[1])/255,
                  blue:  CGFloat(bytes[2])/255,
                  alpha: CGFloat(bytes[3])/255)
    }
}

---

extension RGBA {
    var color: UIColor { .init(bytes) }
}

---

let red: RGBA = [255, 0, 0, 255]
let green: RGBA = [0, 255, 0, 255]
let blue: RGBA = [0, 0, 255, 255]

let redBytes = red.bytes            // [255, 0, 0, 255]
let redData = red.data              // 4 bytes
let rgbaFromBytes = redBytes.rgba    // RGBA
let rgbaFromData = redData.rgba      // RGBA
let colorFromRGBA = red.color       // r 1.0 g 0.0 b 0.0 a 1.0
let rgba: RGBA = [255,255,0,255]    // RGBA yellow
let yellow = rgba.color             // r 1.0 g 1.0 b 0.0 a 1.0

let colors = [red, green, blue]      // [{r 255, g 0, b 0, a 255}, {r 0, g 255, b 0, a 255}, {r 0, g 0, b 255, a 255}]
let colorsData = colors.data          // 12 bytes
let colorsFromData = colorsData.rgbaCollection // [{r 255, g 0, b 0, a 255}, {r 0, g 255, b 0, a 255}, {r 0, g 0, b 255, a 255}]

edit/update:

struct LayerRGBA {
    var canvas: [[RGBA]]
}

---

extension LayerRGBA {
    var data: Data { canvas.data }
    init(_ data: Data) { canvas = data.objects() }
}

---

struct AnimationRGBA {
    var layers: [LayerRGBA]
}

---

extension AnimationRGBA {
    var data: Data { layers.data }
    init(_ data: Data) {
        layers = data.objects()
    }
}

---

struct HistoryRGBA {
    var layers: [LayerRGBA] = []
    var animations: [AnimationRGBA] = []
}

---

extension HistoryRGBA {
    var data: Data {
        let layersData = layers.data
        return layersData.count.data + layersData + animations.data
    }
    init(data: Data)  {
        let index = Int(Data(data.prefix(8))).advanced(by: 8)
        self.init(layers: data.subdata(in: 8..<index).objects(),
                  animations: data.subdata(in: index..<data.endIndex).objects())
    }
}

---

extension Numeric {
    var data: Data {
        var bytes = self
        return .init(bytes: &bytes, count: MemoryLayout<Self>.size)
    }
}

---

extension Numeric {
    init<D: DataProtocol>(_ data: D) {
        var value: Self = .zero
        let _ = withUnsafeMutableBytes(of: &value, { data.copyBytes(to: $0)} )
        self = value
    }
}

---

Playground testing:

let layer1: LayerRGBA = .init(canvas: [colors,[red],[green, blue]])
let layer2: LayerRGBA = .init(canvas: [[red],[green, rgba]])
let loaded: LayerRGBA = .init(layer1.data)
loaded.canvas[0]
loaded.canvas[1]
loaded.canvas[2]

let animationRGBA: AnimationRGBA = .init(layers: [layer1,layer2])
let loadedAnimation: AnimationRGBA = .init(animationRGBA.data)
loadedAnimation.layers.count // 2
loadedAnimation.layers[0].canvas[0]
loadedAnimation.layers[0].canvas[1]
loadedAnimation.layers[0].canvas[2]
loadedAnimation.layers[1].canvas[0]
loadedAnimation.layers[1].canvas[1]

let hRGBA: HistoryRGBA = .init(layers: [loaded], animations: [animationRGBA])
let loadedHistory: HistoryRGBA = .init(data: hRGBA.data)
loadedHistory.layers[0].canvas[0]
loadedHistory.layers[0].canvas[1]
loadedHistory.layers[0].canvas[2]

loadedHistory.animations[0].layers[0].canvas[0]
loadedHistory.animations[0].layers[0].canvas[1]
loadedHistory.animations[0].layers[0].canvas[2]
loadedHistory.animations[0].layers[1].canvas[0]
loadedHistory.animations[0].layers[1].canvas[1]