Add RLE support to system state save/restore

12% compression ratio, sometimes better

Makefile

@@ -30,7 +30,7 @@ MAPDIR = $(SRCDIR)/map
 NESE_SRC_SRCS = f6502.c f6502_opcodes.c
 NESE_SRC_SRCS += nese.c nes.c cart.c mapper.c
 NESE_SRC_SRCS += ppu.c apu.c
-NESE_SRC_SRCS += memory.c serdes.c save.c
+NESE_SRC_SRCS += memory.c serdes.c save.c rle.c
 NESE_SRC_SRCS += $(OS)/port.c
 NESE_MAP_SRCS = $(notdir $(wildcard $(MAPDIR)/*.c))
 

src/f6502.c

@@ -1726,7 +1726,7 @@ int f6502_step(f6502_Core* core, int clocks) {
 /* Ser/Des */
 
 const Serdes_Item f6502_serdes[] = {
-    {offsetof(f6502_Core, registers), serdes_copy, serdes_copy, serdes_copy_size, (void*)(sizeof(f6502_Registers) + sizeof(f6502_Interrupt))},
-    {offsetof(f6502_Core, memory), deserialize, serialize, serdes_size, nes_memory_serdes},
+    {"CREG", offsetof(f6502_Core, registers), &serdes_mem, (void*)(sizeof(f6502_Registers) + sizeof(f6502_Interrupt))},
+    {"CMEM", offsetof(f6502_Core, memory), &serdes_chain, nes_memory_serdes},
     {0}
 };

src/mapper.c

@@ -16,25 +16,3 @@ const nes_Mapper* nes_mappers[256] = {
     &map003,
     &map004,
 };
-
-
-/* Ser/Des */
-
-int mapper_serialize(void* dst, const void* src, int avail, const void*) {
-    const nes_Mapper* mapper = (const nes_Mapper*)src;
-    if (avail > mapper->data_size) avail = mapper->data_size;
-    memcpy(dst, mapper->data, avail);
-    return avail;
-}
-
-int mapper_deserialize(void* dst, const void* src, int avail, const void*) {
-    nes_Mapper* mapper = (nes_Mapper*)dst;
-    if (avail > mapper->data_size) avail = mapper->data_size;
-    memcpy(mapper->data, src, avail);
-    return avail;
-}
-
-int mapper_serdes_size(const void* _mapper, const void*) {
-    const nes_Mapper* mapper = (const nes_Mapper*)_mapper;
-    return mapper->data_size;
-}

src/mapper.h

@@ -38,9 +38,4 @@ typedef struct nes_Mapper nes_Mapper;
 extern const nes_Mapper* nes_mappers[256];
 
 
-int mapper_serialize(void* dst, const void* src, int avail, const void*);
-int mapper_deserialize(void* dst, const void* src, int avail, const void*);
-int mapper_serdes_size(const void* src, const void*);
-
-
 #endif // NESE_MAPPER_H_

src/memory.c

@@ -1,10 +1,26 @@
 #include "memory.h"
 
 
+// TODO: Not ideal that these are here
+
+static void* mapper_data_ptr(const void* _mapper) {
+    return ((nes_Mapper*)_mapper)->data;
+}
+
+static size_t mapper_data_size(const void* _mapper) {
+    return ((nes_Mapper*)_mapper)->data_size;
+}
+
+static Serdes_Ptr_Ref mapper_data_ref = {
+    .ptr = mapper_data_ptr,
+    .size = mapper_data_size,
+};
+
+
 const Serdes_Item nes_memory_serdes[] = {
-    {offsetof(nes_Memory, mapper), mapper_deserialize, mapper_serialize, mapper_serdes_size},
-    {offsetof(nes_Memory, ppu), deserialize, serialize, serdes_size, nes_ppu_memory_serdes},
-    {offsetof(nes_Memory, input), serdes_copy, serdes_copy, serdes_copy_size, (void*)(sizeof(nes_Memory) - offsetof(nes_Memory, input))},
+    {"CART", offsetof(nes_Memory, mapper), &serdes_mem_ptr, &mapper_data_ref},
+    {"PMEM", offsetof(nes_Memory, ppu), &serdes_chain, nes_ppu_memory_serdes},
+    {"SMEM", offsetof(nes_Memory, input), &serdes_mem, (void*)(sizeof(nes_Memory) - offsetof(nes_Memory, input))},
     {0},
 };
 

src/nes.c

@@ -201,7 +201,7 @@ int nes_loop(nes* sys, void* plat) {
 /* Ser/Des */
 
 const Serdes_Item nes_serdes[] = {
-    {offsetof(nes, core), deserialize, serialize, serdes_size, f6502_serdes},
-    {offsetof(nes, ppu), serdes_copy, serdes_copy, serdes_copy_size, (void*)(sizeof(nes_PPU) + sizeof(nes_APU))},
+    {"CORE", offsetof(nes, core), &serdes_chain, f6502_serdes},
+    {"PAPU", offsetof(nes, ppu), &serdes_mem, (void*)(sizeof(nes_PPU) + sizeof(nes_APU))},
     {0}
 };

src/ppu.c

@@ -365,6 +365,7 @@ void nes_ppu_render_line(nes_PPU* ppu, nes_PPU_Memory* mem) {
     }
 }
 
+/*
 static int nes_ppu_chr_ram_size(const void* _ppu, const void*) {
     const nes_PPU_Memory* ppu = (const nes_PPU_Memory*)_ppu;
     return (ppu->chr_ram ?
@@ -396,8 +397,29 @@ static int nes_ppu_write_chr_ram(void* dst, const void* src,
     return size;
 }
 
+static const Serdes_IO ppu_serdes_io = {
+    .read = nes_ppu_read_chr_ram,
+    .write = nes_ppu_write_chr_ram,
+    .in_size = nes_ppu_chr_ram_size,
+    .out_size = nes_ppu_chr_ram_size,
+};
+*/
+
+static void* ppu_chr_ram_ptr(const void* _ppu) {
+    return ((nes_PPU_Memory*)_ppu)->chr_ram;
+}
+
+static size_t ppu_chr_ram_size(const void* _ppu) {
+    return ((nes_PPU_Memory*)_ppu)->n_chr_banks * NES_CHR_ROM_PAGE_SIZE;
+}
+
+static Serdes_Ptr_Ref ppu_chr_ram_ref = {
+    .ptr = ppu_chr_ram_ptr,
+    .size = ppu_chr_ram_size,
+};
+
 const Serdes_Item nes_ppu_memory_serdes[] = {
-    {0, nes_ppu_read_chr_ram, nes_ppu_write_chr_ram, nes_ppu_chr_ram_size},
-    {offsetof(nes_PPU_Memory, ctrl), serdes_copy, serdes_copy, serdes_copy_size, (void*)(sizeof(nes_PPU_Memory) - offsetof(nes_PPU_Memory, ctrl))},
+    {"CRAM", 0, &serdes_mem_ptr, &ppu_chr_ram_ref},
+    {"VRAM", offsetof(nes_PPU_Memory, ctrl), &serdes_mem, (void*)(sizeof(nes_PPU_Memory) - offsetof(nes_PPU_Memory, ctrl))},
     {0}
 };

src/rle.c

@@ -0,0 +1,240 @@
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "rle.h"
+
+
+static bool repeat_run_below(const uint8_t* data, int size,
+                             uint8_t val, int max) {
+    if (size < max) return true;
+    if (0 == max) return false;
+    if (data[0] != val) return true;
+    return repeat_run_below(data + 1, size - 1, val, max - 1);
+}
+
+static int repeat_run_size(const uint8_t* data, int size) {
+    const uint8_t* ptr = data;
+    const uint8_t val = ptr[0];
+    ++ptr;
+    --size;
+    while (size > 0 && ptr[0] == val) {
+        ++ptr;
+        --size;
+    }
+    return (ptr - data);
+}
+
+static int unique_run_size(const uint8_t* data, int size) {
+    const uint8_t* ptr = data;
+    while (size > 0 && repeat_run_below(ptr, size, ptr[0], 3)) {
+        ++ptr;
+        --size;
+    }
+    return (ptr - data);
+}
+
+
+static int rle_token_size(unsigned token) {
+    int n_bits = (8 * sizeof(unsigned)) -
+                 __builtin_clz(token | 1);
+    return (n_bits + 6) / 7;
+}
+
+static int rle_int_to_bytes(uint8_t* data, int size, int val) {
+    if (size <= 0) return -1;
+    if (0 == val) {
+        data[0] = 0;
+        return 1;
+    }
+    uint8_t* ptr = data;
+    while (val && size > 0) {
+        uint8_t byte = val & 0x7FU;
+        val >>= 7;
+        if (val) byte |= 0x80U;
+        ptr[0] = byte;
+        ++ptr;
+        --size;
+    }
+    return (val ? -1 : (ptr - data));
+}
+
+static int rle_token_length(const uint8_t* data, int size) {
+    if (size <= 0) return -1;
+    if (!(data[0] & 0x80)) return 1;
+    int remainder = rle_token_length(data + 1, size - 1);
+    return (remainder < 0) ? remainder : (1 + remainder);
+}
+
+static unsigned rle_bytes_to_int(const uint8_t* data, int size) {
+    unsigned val = data[0] & 0x7FU;
+    if (data[0] & 0x80) {
+        val |= rle_bytes_to_int(data + 1, size - 1) << 7;
+    }
+    return val;
+}
+
+
+int rle_encode(uint8_t* dst, int dst_size,
+               const uint8_t* src, int src_size) {
+    uint8_t* dst_ptr = dst;
+    const uint8_t* src_ptr = src;
+    while (src_size > 0 && dst_size > 0) {
+        int run_length = 0;
+        unsigned token = 0;
+        int copy_size = 0;
+        if (!repeat_run_below(src_ptr, src_size, src_ptr[0], 2)) {
+            run_length = repeat_run_size(src_ptr, src_size);
+            token = ((run_length - 2) * 2) + 1;
+            copy_size = 1;
+        } else {
+            run_length = unique_run_size(src_ptr, src_size);
+            token = ((run_length - 1) * 2);
+            copy_size = run_length;
+        }
+
+        int token_size = rle_int_to_bytes(dst_ptr, dst_size, token);
+        if (token_size <= 0 || copy_size > dst_size) {
+            break;
+        }
+        dst_ptr += token_size;
+        dst_size -= token_size;
+
+        memcpy(dst_ptr, src_ptr, copy_size);
+
+        dst_ptr += copy_size;
+        dst_size -= copy_size;
+
+        src_ptr += run_length;
+        src_size -= run_length;
+    }
+
+    return (src_size > 0 ? -1 : (dst_ptr - dst));
+}
+
+int rle_decode(uint8_t* dst, int dst_size,
+               const uint8_t* src, int src_size) {
+    uint8_t* dst_ptr = dst;
+    const uint8_t* src_ptr = src;
+    while (src_size > 0 && dst_size > 0) {
+        int token_size = rle_token_length(src_ptr, src_size);
+        if (token_size <= 0) break;
+        unsigned token = rle_bytes_to_int(src_ptr, src_size);
+        src_ptr += token_size;
+        src_size -= token_size;
+        int run_length = 0;
+        if (token & 1) {
+            run_length = (token / 2) + 2;
+            if (run_length > dst_size || src_size < 1) break;
+            memset(dst_ptr, src_ptr[0], run_length);
+            ++src_ptr;
+            --src_size;
+        } else {
+            run_length = (token / 2) + 1;
+            if (run_length > dst_size || src_size < run_length) break;
+            memcpy(dst_ptr, src_ptr, run_length);
+            src_ptr += run_length;
+            src_size -= run_length;
+        }
+        dst_ptr += run_length;
+        dst_size -= run_length;
+    }
+
+    return (dst_size > 0 ? -1 : (src_ptr - src));
+}
+
+int rle_encoded_size(const uint8_t* data, int size) {
+    int encoded = 0;
+    const uint8_t* ptr = data;
+    while (size > 0) {
+        int run_length = 0;
+        unsigned token = 0;
+        int copy_size = 0;
+        if (!repeat_run_below(ptr, size, ptr[0], 2)) {
+            run_length = repeat_run_size(ptr, size);
+            token = ((run_length - 2) * 2) + 1;
+            copy_size = 1;
+        } else {
+            run_length = unique_run_size(ptr, size);
+            token = ((run_length - 1) * 2);
+            copy_size = run_length;
+        }
+
+        encoded += copy_size + rle_token_size(token);
+
+        ptr += run_length;
+        size -= run_length;
+    }
+    return (0 == size ? encoded : -1);
+}
+
+int rle_encoded_size_for(const uint8_t* data, int size,
+                         int decoded) {
+    const uint8_t* ptr = data;
+    while (size > 0 && decoded > 0) {
+        int token_size = rle_token_length(ptr, size);
+        if (token_size <= 0) {
+            decoded = -1;
+            break;
+        }
+        unsigned token = rle_bytes_to_int(ptr, size);
+        ptr += token_size;
+        size -= token_size;
+        int run_length = 0;
+        if (token & 1) {
+            run_length = (token / 2) + 2;
+            if (size < 1) {
+                decoded = -1;
+                break;
+            }
+            ++ptr;
+            --size;
+        } else {
+            run_length = (token / 2) + 1;
+            if (size < run_length) {
+                decoded = -1;
+                break;
+            }
+            ptr += run_length;
+            size -= run_length;
+        }
+        decoded -= run_length;
+    }
+    return (0 == decoded ? ptr - data : -1);
+}
+
+int rle_decoded_size(const uint8_t* data, int size) {
+    int decoded = 0;
+    const uint8_t* ptr = data;
+    while (size > 0) {
+        int token_size = rle_token_length(ptr, size);
+        if (token_size <= 0) {
+            decoded = -1;
+            break;
+        }
+        unsigned token = rle_bytes_to_int(ptr, size);
+        ptr += token_size;
+        size -= token_size;
+        int run_length = 0;
+        if (token & 1) {
+            run_length = (token / 2) + 2;
+            if (size < 1) {
+                decoded = -1;
+                break;
+            }
+            ++ptr;
+            --size;
+        } else {
+            run_length = (token / 2) + 1;
+            if (size < run_length) {
+                decoded = -1;
+                break;
+            }
+            ptr += run_length;
+            size -= run_length;
+        }
+        decoded += run_length;
+    }
+    return decoded;
+}

src/rle.h

@@ -0,0 +1,18 @@
+#ifndef NESE_RLE_H_
+#define NESE_RLE_H_
+
+#include <stdint.h>
+
+
+int rle_encode(uint8_t* dst, int dst_size,
+               const uint8_t* src, int src_size);
+int rle_decode(uint8_t* dst, int dst_size,
+               const uint8_t* src, int src_size);
+
+int rle_decoded_size(const uint8_t* data, int size);
+int rle_encoded_size(const uint8_t* data, int size);
+int rle_encoded_size_for(const uint8_t* data, int size,
+                         int decoded);
+
+
+#endif // NESE_RLE_H_

src/save.c

@@ -89,8 +89,8 @@ static int make_state_filename(char* save_filename, int max_len,
                            "save", "nese");
 }
 
-static int state_size(const nes* sys) {
-    return serdes_size(sys, nes_serdes);
+static int state_write_size(const nes* sys) {
+    return serialize_size(sys, nes_serdes);
 }
 
 static int state_read(nes* sys, const void* mem, int size) {
@@ -155,7 +155,7 @@ int save_state(const nes* sys, const char* cart_filename) {
     FILE* file = fopen(state_filename, "w+b");
 
     if (NULL != file) {
-        int file_size = state_size(sys);
+        int file_size = state_write_size(sys);
 
         fseek(file, file_size - 1, SEEK_SET);
         fwrite("", 1, 1, file);

src/serdes.c

@@ -1,15 +1,20 @@
 #include <string.h>
 
+#include "rle.h"
 #include "serdes.h"
 
 
+/* Chained */
+
 int serialize(void* dst, const void* src, int avail,
               const void* _list) {
     const Serdes_Item* list = (const Serdes_Item*)_list;
     void* dst_ptr = dst;
-    for ( ; list->write; ++list) {
+    for ( ; list->io; ++list) {
+        if (!list->io->write) continue;
         const void* src_ptr = src + list->offset;
-        int count = list->write(dst_ptr, src_ptr, avail, list->arg);
+        int count = list->io->write(dst_ptr, src_ptr,
+                                    avail, list->arg);
         if (count < 0) break;
         dst_ptr += count;
         avail -= count;
@@ -21,9 +26,11 @@ int deserialize(void* dst, const void* src, int avail,
                 const void* _list) {
     const Serdes_Item* list = (const Serdes_Item*)_list;
     const void* src_ptr = src;
-    for ( ; list->read; ++list) {
+    for ( ; list->io; ++list) {
+        if (!list->io->read) continue;
         void* dst_ptr = dst + list->offset;
-        int count = list->read(dst_ptr, src_ptr, avail, list->arg);
+        int count = list->io->read(dst_ptr, src_ptr,
+                                   avail, list->arg);
         if (count < 0) break;
         src_ptr += count;
         avail -= count;
@@ -31,23 +38,132 @@ int deserialize(void* dst, const void* src, int avail,
     return (src_ptr - src);
 }
 
-int serdes_size(const void* src, const void* _list) {
+int serialize_size(const void* src, const void* _list) {
+    const Serdes_Item* list = (const Serdes_Item*)_list;
+    int size = 0;
+    for ( ; list->io; ++list) {
+        if (!list->io->out_size) continue;
+        int chunk = list->io->out_size(src + list->offset, list->arg);
+        size += chunk;
+    }
+    return size;
+}
+
+int deserialize_size(const void* src, const void* _list) {
     const Serdes_Item* list = (const Serdes_Item*)_list;
     int size = 0;
-    for ( ; list->size; ++list) {
-        size += list->size(src + list->offset, list->arg);
+    for ( ; list->io; ++list) {
+        if (!list->io->in_size) continue;
+        size += list->io->in_size(src + list->offset, list->arg);
     }
     return size;
 }
 
-int serdes_copy(void* dst, const void* src, int avail,
-                const void* arg) {
+const Serdes_IO serdes_chain = {
+    .read = deserialize,
+    .write = serialize,
+    .in_size = deserialize_size,
+    .out_size = serialize_size,
+};
+
+
+/* Direct Copy */
+/*
+static int serdes_mem_copy(void* dst, const void* src, int avail,
+                           const void* arg) {
     int size = (long)arg;
     if (size > avail) size = avail;
     memcpy(dst, src, size);
     return size;
 }
 
-int serdes_copy_size(const void*, const void* arg) {
+static int serdes_mem_size(const void*, const void* arg) {
+    return (long)arg;
+}
+
+const Serdes_IO serdes_mem = {
+    .read = serdes_mem_copy,
+    .write = serdes_mem_copy,
+    .in_size = serdes_mem_size,
+    .out_size = serdes_mem_size,
+};
+*/
+
+/* Compressed Direct Copy */
+
+static int serdes_rle_in(void* dst, const void* src, int avail,
+                         const void* arg) {
+    int dst_size = (long)arg;
+    int src_size = rle_encoded_size_for(src, avail, dst_size);
+    if (src_size > avail || src_size < 0) return 0;
+    int decoded = rle_decode(dst, dst_size, src, src_size);
+    return (decoded >= 0 ? decoded : 0);
+}
+
+static int serdes_rle_out(void* dst, const void* src, int avail,
+                          const void* arg) {
+    int src_size = (long)arg;
+    int dst_size = rle_encoded_size(src, src_size);
+    if (dst_size > avail || dst_size < 0) return 0;
+    int encoded = rle_encode(dst, dst_size, src, src_size);
+    return (encoded == dst_size ? encoded : 0);
+}
+
+static int serdes_rle_in_size(const void* data,
+                              const void* arg) {
     return (long)arg;
 }
+
+static int serdes_rle_out_size(const void* data,
+                               const void* arg) {
+    return rle_encoded_size(data, (long)arg);
+}
+
+const Serdes_IO serdes_mem = {
+    .read = serdes_rle_in,
+    .write = serdes_rle_out,
+    .in_size = serdes_rle_in_size,
+    .out_size = serdes_rle_out_size,
+};
+
+
+/* Pointer Copy */
+
+static int serdes_ptr_in(void* dst, const void* src, int avail,
+                         const void* arg) {
+    Serdes_Ptr_Ref* ref = (Serdes_Ptr_Ref*)arg;
+    void* ptr = ref->ptr(dst);
+    if (NULL == ptr) return 0;
+    size_t size = ref->size(dst);
+    return serdes_mem.read(ptr, src, avail, (void*)size);
+}
+
+static int serdes_ptr_out(void* dst, const void* src, int avail,
+                          const void* arg) {
+    Serdes_Ptr_Ref* ref = (Serdes_Ptr_Ref*)arg;
+    void* ptr = ref->ptr(src);
+    if (NULL == ptr) return 0;
+    size_t size = ref->size(src);
+    return serdes_mem.write(dst, ptr, avail, (void*)size);
+}
+
+static int serdes_ptr_in_size(const void* src, const void* arg) {
+    Serdes_Ptr_Ref* ref = (Serdes_Ptr_Ref*)arg;
+    void* ptr = ref->ptr(src);
+    if (NULL == ptr) return 0;
+    return serdes_mem.in_size(ptr, (void*)ref->size(src));
+}
+
+static int serdes_ptr_out_size(const void* src, const void* arg) {
+    Serdes_Ptr_Ref* ref = (Serdes_Ptr_Ref*)arg;
+    void* ptr = ref->ptr(src);
+    if (NULL == ptr) return 0;
+    return serdes_mem.out_size(ptr, (void*)ref->size(src));
+}
+
+const Serdes_IO serdes_mem_ptr = {
+    .read = serdes_ptr_in,
+    .write = serdes_ptr_out,
+    .in_size = serdes_ptr_in_size,
+    .out_size = serdes_ptr_out_size,
+};

src/serdes.h

@@ -8,22 +8,34 @@ typedef int(*serdes_io_fn)(void*, const void*, int, const void*);
 typedef int(*serdes_size_fn)(const void*, const void*);
 
 typedef struct {
+    serdes_io_fn read;
+    serdes_io_fn write;
+    serdes_size_fn in_size;
+    serdes_size_fn out_size;
+} Serdes_IO;
+
+typedef struct {
+    const char* tag;
     long offset;
-    int (*read)(void* dst, const void* src, int avail, const void* arg);
-    int (*write)(void* dst, const void* src, int avail, const void* arg);
-    int (*size)(const void*, const void*);
+    const Serdes_IO* io;
     const void* arg;
 } Serdes_Item;
 
+extern const Serdes_IO serdes_chain;
 
-int serialize(void* dst, const void* src, int avail, const void* list);
-int deserialize(void* dst, const void* src, int avail, const void* list);
-int serdes_size(const void*, const void* list);
+typedef struct {
+    void*(*ptr)(const void*);
+    size_t(*size)(const void*);
+} Serdes_Ptr_Ref;
+
+extern const Serdes_IO serdes_mem;
+extern const Serdes_IO serdes_mem_ptr;
 
-int serdes_copy(void* dst, const void* src, int avail, const void* arg);
-int serdes_copy_size(const void*, const void* arg);
 
-// TODO: Compression (Maybe just RLE)
+int serialize(void* dst, const void* src, int avail, const void* list);
+int deserialize(void* dst, const void* src, int avail, const void* list);
+int serialize_size(const void*, const void* list);
+int deserialize_size(const void*, const void* list);
 
 
 #endif // NESE_SERDES_H_