From 76d67f2dc2c0c51209de103827b2f6c3b6fa15b2 Mon Sep 17 00:00:00 2001 From: Nathaniel Walizer Date: Thu, 27 Mar 2025 00:44:25 -0700 Subject: [PATCH] Add RLE support to system state save/restore 12% compression ratio, sometimes better --- Makefile | 2 +- src/f6502.c | 4 +- src/mapper.c | 22 ----- src/mapper.h | 5 -- src/memory.c | 22 ++++- src/nes.c | 4 +- src/ppu.c | 26 +++++- src/rle.c | 240 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/rle.h | 18 ++++ src/save.c | 6 +- src/serdes.c | 136 ++++++++++++++++++++++++++--- src/serdes.h | 30 +++++-- 12 files changed, 456 insertions(+), 59 deletions(-) create mode 100644 src/rle.c create mode 100644 src/rle.h diff --git a/Makefile b/Makefile index 859e6d9..6000863 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ MAPDIR = $(SRCDIR)/map NESE_SRC_SRCS = f6502.c f6502_opcodes.c NESE_SRC_SRCS += nese.c nes.c cart.c mapper.c NESE_SRC_SRCS += ppu.c apu.c -NESE_SRC_SRCS += memory.c serdes.c save.c +NESE_SRC_SRCS += memory.c serdes.c save.c rle.c NESE_SRC_SRCS += $(OS)/port.c NESE_MAP_SRCS = $(notdir $(wildcard $(MAPDIR)/*.c)) diff --git a/src/f6502.c b/src/f6502.c index 882316d..196b68d 100644 --- a/src/f6502.c +++ b/src/f6502.c @@ -1726,7 +1726,7 @@ int f6502_step(f6502_Core* core, int clocks) { /* Ser/Des */ const Serdes_Item f6502_serdes[] = { - {offsetof(f6502_Core, registers), serdes_copy, serdes_copy, serdes_copy_size, (void*)(sizeof(f6502_Registers) + sizeof(f6502_Interrupt))}, - {offsetof(f6502_Core, memory), deserialize, serialize, serdes_size, nes_memory_serdes}, + {"CREG", offsetof(f6502_Core, registers), &serdes_mem, (void*)(sizeof(f6502_Registers) + sizeof(f6502_Interrupt))}, + {"CMEM", offsetof(f6502_Core, memory), &serdes_chain, nes_memory_serdes}, {0} }; diff --git a/src/mapper.c b/src/mapper.c index 193b80d..fec7534 100644 --- a/src/mapper.c +++ b/src/mapper.c @@ -16,25 +16,3 @@ const nes_Mapper* nes_mappers[256] = { &map003, &map004, }; - - -/* Ser/Des */ - -int mapper_serialize(void* dst, const void* src, int avail, const void*) { - const nes_Mapper* mapper = (const nes_Mapper*)src; - if (avail > mapper->data_size) avail = mapper->data_size; - memcpy(dst, mapper->data, avail); - return avail; -} - -int mapper_deserialize(void* dst, const void* src, int avail, const void*) { - nes_Mapper* mapper = (nes_Mapper*)dst; - if (avail > mapper->data_size) avail = mapper->data_size; - memcpy(mapper->data, src, avail); - return avail; -} - -int mapper_serdes_size(const void* _mapper, const void*) { - const nes_Mapper* mapper = (const nes_Mapper*)_mapper; - return mapper->data_size; -} diff --git a/src/mapper.h b/src/mapper.h index 2d7ecd2..1faeafd 100644 --- a/src/mapper.h +++ b/src/mapper.h @@ -38,9 +38,4 @@ typedef struct nes_Mapper nes_Mapper; extern const nes_Mapper* nes_mappers[256]; -int mapper_serialize(void* dst, const void* src, int avail, const void*); -int mapper_deserialize(void* dst, const void* src, int avail, const void*); -int mapper_serdes_size(const void* src, const void*); - - #endif // NESE_MAPPER_H_ diff --git a/src/memory.c b/src/memory.c index 7874ad7..e1d11f2 100644 --- a/src/memory.c +++ b/src/memory.c @@ -1,10 +1,26 @@ #include "memory.h" +// TODO: Not ideal that these are here + +static void* mapper_data_ptr(const void* _mapper) { + return ((nes_Mapper*)_mapper)->data; +} + +static size_t mapper_data_size(const void* _mapper) { + return ((nes_Mapper*)_mapper)->data_size; +} + +static Serdes_Ptr_Ref mapper_data_ref = { + .ptr = mapper_data_ptr, + .size = mapper_data_size, +}; + + const Serdes_Item nes_memory_serdes[] = { - {offsetof(nes_Memory, mapper), mapper_deserialize, mapper_serialize, mapper_serdes_size}, - {offsetof(nes_Memory, ppu), deserialize, serialize, serdes_size, nes_ppu_memory_serdes}, - {offsetof(nes_Memory, input), serdes_copy, serdes_copy, serdes_copy_size, (void*)(sizeof(nes_Memory) - offsetof(nes_Memory, input))}, + {"CART", offsetof(nes_Memory, mapper), &serdes_mem_ptr, &mapper_data_ref}, + {"PMEM", offsetof(nes_Memory, ppu), &serdes_chain, nes_ppu_memory_serdes}, + {"SMEM", offsetof(nes_Memory, input), &serdes_mem, (void*)(sizeof(nes_Memory) - offsetof(nes_Memory, input))}, {0}, }; diff --git a/src/nes.c b/src/nes.c index 14a8eab..90add48 100644 --- a/src/nes.c +++ b/src/nes.c @@ -201,7 +201,7 @@ int nes_loop(nes* sys, void* plat) { /* Ser/Des */ const Serdes_Item nes_serdes[] = { - {offsetof(nes, core), deserialize, serialize, serdes_size, f6502_serdes}, - {offsetof(nes, ppu), serdes_copy, serdes_copy, serdes_copy_size, (void*)(sizeof(nes_PPU) + sizeof(nes_APU))}, + {"CORE", offsetof(nes, core), &serdes_chain, f6502_serdes}, + {"PAPU", offsetof(nes, ppu), &serdes_mem, (void*)(sizeof(nes_PPU) + sizeof(nes_APU))}, {0} }; diff --git a/src/ppu.c b/src/ppu.c index d6b7297..704049d 100644 --- a/src/ppu.c +++ b/src/ppu.c @@ -365,6 +365,7 @@ void nes_ppu_render_line(nes_PPU* ppu, nes_PPU_Memory* mem) { } } +/* static int nes_ppu_chr_ram_size(const void* _ppu, const void*) { const nes_PPU_Memory* ppu = (const nes_PPU_Memory*)_ppu; return (ppu->chr_ram ? @@ -396,8 +397,29 @@ static int nes_ppu_write_chr_ram(void* dst, const void* src, return size; } +static const Serdes_IO ppu_serdes_io = { + .read = nes_ppu_read_chr_ram, + .write = nes_ppu_write_chr_ram, + .in_size = nes_ppu_chr_ram_size, + .out_size = nes_ppu_chr_ram_size, +}; +*/ + +static void* ppu_chr_ram_ptr(const void* _ppu) { + return ((nes_PPU_Memory*)_ppu)->chr_ram; +} + +static size_t ppu_chr_ram_size(const void* _ppu) { + return ((nes_PPU_Memory*)_ppu)->n_chr_banks * NES_CHR_ROM_PAGE_SIZE; +} + +static Serdes_Ptr_Ref ppu_chr_ram_ref = { + .ptr = ppu_chr_ram_ptr, + .size = ppu_chr_ram_size, +}; + const Serdes_Item nes_ppu_memory_serdes[] = { - {0, nes_ppu_read_chr_ram, nes_ppu_write_chr_ram, nes_ppu_chr_ram_size}, - {offsetof(nes_PPU_Memory, ctrl), serdes_copy, serdes_copy, serdes_copy_size, (void*)(sizeof(nes_PPU_Memory) - offsetof(nes_PPU_Memory, ctrl))}, + {"CRAM", 0, &serdes_mem_ptr, &ppu_chr_ram_ref}, + {"VRAM", offsetof(nes_PPU_Memory, ctrl), &serdes_mem, (void*)(sizeof(nes_PPU_Memory) - offsetof(nes_PPU_Memory, ctrl))}, {0} }; diff --git a/src/rle.c b/src/rle.c new file mode 100644 index 0000000..a663010 --- /dev/null +++ b/src/rle.c @@ -0,0 +1,240 @@ +#include +#include +#include +#include + +#include "rle.h" + + +static bool repeat_run_below(const uint8_t* data, int size, + uint8_t val, int max) { + if (size < max) return true; + if (0 == max) return false; + if (data[0] != val) return true; + return repeat_run_below(data + 1, size - 1, val, max - 1); +} + +static int repeat_run_size(const uint8_t* data, int size) { + const uint8_t* ptr = data; + const uint8_t val = ptr[0]; + ++ptr; + --size; + while (size > 0 && ptr[0] == val) { + ++ptr; + --size; + } + return (ptr - data); +} + +static int unique_run_size(const uint8_t* data, int size) { + const uint8_t* ptr = data; + while (size > 0 && repeat_run_below(ptr, size, ptr[0], 3)) { + ++ptr; + --size; + } + return (ptr - data); +} + + +static int rle_token_size(unsigned token) { + int n_bits = (8 * sizeof(unsigned)) - + __builtin_clz(token | 1); + return (n_bits + 6) / 7; +} + +static int rle_int_to_bytes(uint8_t* data, int size, int val) { + if (size <= 0) return -1; + if (0 == val) { + data[0] = 0; + return 1; + } + uint8_t* ptr = data; + while (val && size > 0) { + uint8_t byte = val & 0x7FU; + val >>= 7; + if (val) byte |= 0x80U; + ptr[0] = byte; + ++ptr; + --size; + } + return (val ? -1 : (ptr - data)); +} + +static int rle_token_length(const uint8_t* data, int size) { + if (size <= 0) return -1; + if (!(data[0] & 0x80)) return 1; + int remainder = rle_token_length(data + 1, size - 1); + return (remainder < 0) ? remainder : (1 + remainder); +} + +static unsigned rle_bytes_to_int(const uint8_t* data, int size) { + unsigned val = data[0] & 0x7FU; + if (data[0] & 0x80) { + val |= rle_bytes_to_int(data + 1, size - 1) << 7; + } + return val; +} + + +int rle_encode(uint8_t* dst, int dst_size, + const uint8_t* src, int src_size) { + uint8_t* dst_ptr = dst; + const uint8_t* src_ptr = src; + while (src_size > 0 && dst_size > 0) { + int run_length = 0; + unsigned token = 0; + int copy_size = 0; + if (!repeat_run_below(src_ptr, src_size, src_ptr[0], 2)) { + run_length = repeat_run_size(src_ptr, src_size); + token = ((run_length - 2) * 2) + 1; + copy_size = 1; + } else { + run_length = unique_run_size(src_ptr, src_size); + token = ((run_length - 1) * 2); + copy_size = run_length; + } + + int token_size = rle_int_to_bytes(dst_ptr, dst_size, token); + if (token_size <= 0 || copy_size > dst_size) { + break; + } + dst_ptr += token_size; + dst_size -= token_size; + + memcpy(dst_ptr, src_ptr, copy_size); + + dst_ptr += copy_size; + dst_size -= copy_size; + + src_ptr += run_length; + src_size -= run_length; + } + + return (src_size > 0 ? -1 : (dst_ptr - dst)); +} + +int rle_decode(uint8_t* dst, int dst_size, + const uint8_t* src, int src_size) { + uint8_t* dst_ptr = dst; + const uint8_t* src_ptr = src; + while (src_size > 0 && dst_size > 0) { + int token_size = rle_token_length(src_ptr, src_size); + if (token_size <= 0) break; + unsigned token = rle_bytes_to_int(src_ptr, src_size); + src_ptr += token_size; + src_size -= token_size; + int run_length = 0; + if (token & 1) { + run_length = (token / 2) + 2; + if (run_length > dst_size || src_size < 1) break; + memset(dst_ptr, src_ptr[0], run_length); + ++src_ptr; + --src_size; + } else { + run_length = (token / 2) + 1; + if (run_length > dst_size || src_size < run_length) break; + memcpy(dst_ptr, src_ptr, run_length); + src_ptr += run_length; + src_size -= run_length; + } + dst_ptr += run_length; + dst_size -= run_length; + } + + return (dst_size > 0 ? -1 : (src_ptr - src)); +} + +int rle_encoded_size(const uint8_t* data, int size) { + int encoded = 0; + const uint8_t* ptr = data; + while (size > 0) { + int run_length = 0; + unsigned token = 0; + int copy_size = 0; + if (!repeat_run_below(ptr, size, ptr[0], 2)) { + run_length = repeat_run_size(ptr, size); + token = ((run_length - 2) * 2) + 1; + copy_size = 1; + } else { + run_length = unique_run_size(ptr, size); + token = ((run_length - 1) * 2); + copy_size = run_length; + } + + encoded += copy_size + rle_token_size(token); + + ptr += run_length; + size -= run_length; + } + return (0 == size ? encoded : -1); +} + +int rle_encoded_size_for(const uint8_t* data, int size, + int decoded) { + const uint8_t* ptr = data; + while (size > 0 && decoded > 0) { + int token_size = rle_token_length(ptr, size); + if (token_size <= 0) { + decoded = -1; + break; + } + unsigned token = rle_bytes_to_int(ptr, size); + ptr += token_size; + size -= token_size; + int run_length = 0; + if (token & 1) { + run_length = (token / 2) + 2; + if (size < 1) { + decoded = -1; + break; + } + ++ptr; + --size; + } else { + run_length = (token / 2) + 1; + if (size < run_length) { + decoded = -1; + break; + } + ptr += run_length; + size -= run_length; + } + decoded -= run_length; + } + return (0 == decoded ? ptr - data : -1); +} + +int rle_decoded_size(const uint8_t* data, int size) { + int decoded = 0; + const uint8_t* ptr = data; + while (size > 0) { + int token_size = rle_token_length(ptr, size); + if (token_size <= 0) { + decoded = -1; + break; + } + unsigned token = rle_bytes_to_int(ptr, size); + ptr += token_size; + size -= token_size; + int run_length = 0; + if (token & 1) { + run_length = (token / 2) + 2; + if (size < 1) { + decoded = -1; + break; + } + ++ptr; + --size; + } else { + run_length = (token / 2) + 1; + if (size < run_length) { + decoded = -1; + break; + } + ptr += run_length; + size -= run_length; + } + decoded += run_length; + } + return decoded; +} diff --git a/src/rle.h b/src/rle.h new file mode 100644 index 0000000..48aaaef --- /dev/null +++ b/src/rle.h @@ -0,0 +1,18 @@ +#ifndef NESE_RLE_H_ +#define NESE_RLE_H_ + +#include + + +int rle_encode(uint8_t* dst, int dst_size, + const uint8_t* src, int src_size); +int rle_decode(uint8_t* dst, int dst_size, + const uint8_t* src, int src_size); + +int rle_decoded_size(const uint8_t* data, int size); +int rle_encoded_size(const uint8_t* data, int size); +int rle_encoded_size_for(const uint8_t* data, int size, + int decoded); + + +#endif // NESE_RLE_H_ diff --git a/src/save.c b/src/save.c index 53cf6ec..251e9a1 100644 --- a/src/save.c +++ b/src/save.c @@ -89,8 +89,8 @@ static int make_state_filename(char* save_filename, int max_len, "save", "nese"); } -static int state_size(const nes* sys) { - return serdes_size(sys, nes_serdes); +static int state_write_size(const nes* sys) { + return serialize_size(sys, nes_serdes); } static int state_read(nes* sys, const void* mem, int size) { @@ -155,7 +155,7 @@ int save_state(const nes* sys, const char* cart_filename) { FILE* file = fopen(state_filename, "w+b"); if (NULL != file) { - int file_size = state_size(sys); + int file_size = state_write_size(sys); fseek(file, file_size - 1, SEEK_SET); fwrite("", 1, 1, file); diff --git a/src/serdes.c b/src/serdes.c index 1d30b25..666e026 100644 --- a/src/serdes.c +++ b/src/serdes.c @@ -1,15 +1,20 @@ #include +#include "rle.h" #include "serdes.h" +/* Chained */ + int serialize(void* dst, const void* src, int avail, const void* _list) { const Serdes_Item* list = (const Serdes_Item*)_list; void* dst_ptr = dst; - for ( ; list->write; ++list) { + for ( ; list->io; ++list) { + if (!list->io->write) continue; const void* src_ptr = src + list->offset; - int count = list->write(dst_ptr, src_ptr, avail, list->arg); + int count = list->io->write(dst_ptr, src_ptr, + avail, list->arg); if (count < 0) break; dst_ptr += count; avail -= count; @@ -21,9 +26,11 @@ int deserialize(void* dst, const void* src, int avail, const void* _list) { const Serdes_Item* list = (const Serdes_Item*)_list; const void* src_ptr = src; - for ( ; list->read; ++list) { + for ( ; list->io; ++list) { + if (!list->io->read) continue; void* dst_ptr = dst + list->offset; - int count = list->read(dst_ptr, src_ptr, avail, list->arg); + int count = list->io->read(dst_ptr, src_ptr, + avail, list->arg); if (count < 0) break; src_ptr += count; avail -= count; @@ -31,23 +38,132 @@ int deserialize(void* dst, const void* src, int avail, return (src_ptr - src); } -int serdes_size(const void* src, const void* _list) { +int serialize_size(const void* src, const void* _list) { + const Serdes_Item* list = (const Serdes_Item*)_list; + int size = 0; + for ( ; list->io; ++list) { + if (!list->io->out_size) continue; + int chunk = list->io->out_size(src + list->offset, list->arg); + size += chunk; + } + return size; +} + +int deserialize_size(const void* src, const void* _list) { const Serdes_Item* list = (const Serdes_Item*)_list; int size = 0; - for ( ; list->size; ++list) { - size += list->size(src + list->offset, list->arg); + for ( ; list->io; ++list) { + if (!list->io->in_size) continue; + size += list->io->in_size(src + list->offset, list->arg); } return size; } -int serdes_copy(void* dst, const void* src, int avail, - const void* arg) { +const Serdes_IO serdes_chain = { + .read = deserialize, + .write = serialize, + .in_size = deserialize_size, + .out_size = serialize_size, +}; + + +/* Direct Copy */ +/* +static int serdes_mem_copy(void* dst, const void* src, int avail, + const void* arg) { int size = (long)arg; if (size > avail) size = avail; memcpy(dst, src, size); return size; } -int serdes_copy_size(const void*, const void* arg) { +static int serdes_mem_size(const void*, const void* arg) { + return (long)arg; +} + +const Serdes_IO serdes_mem = { + .read = serdes_mem_copy, + .write = serdes_mem_copy, + .in_size = serdes_mem_size, + .out_size = serdes_mem_size, +}; +*/ + +/* Compressed Direct Copy */ + +static int serdes_rle_in(void* dst, const void* src, int avail, + const void* arg) { + int dst_size = (long)arg; + int src_size = rle_encoded_size_for(src, avail, dst_size); + if (src_size > avail || src_size < 0) return 0; + int decoded = rle_decode(dst, dst_size, src, src_size); + return (decoded >= 0 ? decoded : 0); +} + +static int serdes_rle_out(void* dst, const void* src, int avail, + const void* arg) { + int src_size = (long)arg; + int dst_size = rle_encoded_size(src, src_size); + if (dst_size > avail || dst_size < 0) return 0; + int encoded = rle_encode(dst, dst_size, src, src_size); + return (encoded == dst_size ? encoded : 0); +} + +static int serdes_rle_in_size(const void* data, + const void* arg) { return (long)arg; } + +static int serdes_rle_out_size(const void* data, + const void* arg) { + return rle_encoded_size(data, (long)arg); +} + +const Serdes_IO serdes_mem = { + .read = serdes_rle_in, + .write = serdes_rle_out, + .in_size = serdes_rle_in_size, + .out_size = serdes_rle_out_size, +}; + + +/* Pointer Copy */ + +static int serdes_ptr_in(void* dst, const void* src, int avail, + const void* arg) { + Serdes_Ptr_Ref* ref = (Serdes_Ptr_Ref*)arg; + void* ptr = ref->ptr(dst); + if (NULL == ptr) return 0; + size_t size = ref->size(dst); + return serdes_mem.read(ptr, src, avail, (void*)size); +} + +static int serdes_ptr_out(void* dst, const void* src, int avail, + const void* arg) { + Serdes_Ptr_Ref* ref = (Serdes_Ptr_Ref*)arg; + void* ptr = ref->ptr(src); + if (NULL == ptr) return 0; + size_t size = ref->size(src); + return serdes_mem.write(dst, ptr, avail, (void*)size); +} + +static int serdes_ptr_in_size(const void* src, const void* arg) { + Serdes_Ptr_Ref* ref = (Serdes_Ptr_Ref*)arg; + void* ptr = ref->ptr(src); + if (NULL == ptr) return 0; + return serdes_mem.in_size(ptr, (void*)ref->size(src)); +} + +static int serdes_ptr_out_size(const void* src, const void* arg) { + Serdes_Ptr_Ref* ref = (Serdes_Ptr_Ref*)arg; + void* ptr = ref->ptr(src); + if (NULL == ptr) return 0; + return serdes_mem.out_size(ptr, (void*)ref->size(src)); +} + +const Serdes_IO serdes_mem_ptr = { + .read = serdes_ptr_in, + .write = serdes_ptr_out, + .in_size = serdes_ptr_in_size, + .out_size = serdes_ptr_out_size, +}; diff --git a/src/serdes.h b/src/serdes.h index 6ce56a7..b7efb06 100644 --- a/src/serdes.h +++ b/src/serdes.h @@ -8,22 +8,34 @@ typedef int(*serdes_io_fn)(void*, const void*, int, const void*); typedef int(*serdes_size_fn)(const void*, const void*); typedef struct { + serdes_io_fn read; + serdes_io_fn write; + serdes_size_fn in_size; + serdes_size_fn out_size; +} Serdes_IO; + +typedef struct { + const char* tag; long offset; - int (*read)(void* dst, const void* src, int avail, const void* arg); - int (*write)(void* dst, const void* src, int avail, const void* arg); - int (*size)(const void*, const void*); + const Serdes_IO* io; const void* arg; } Serdes_Item; +extern const Serdes_IO serdes_chain; -int serialize(void* dst, const void* src, int avail, const void* list); -int deserialize(void* dst, const void* src, int avail, const void* list); -int serdes_size(const void*, const void* list); +typedef struct { + void*(*ptr)(const void*); + size_t(*size)(const void*); +} Serdes_Ptr_Ref; + +extern const Serdes_IO serdes_mem; +extern const Serdes_IO serdes_mem_ptr; -int serdes_copy(void* dst, const void* src, int avail, const void* arg); -int serdes_copy_size(const void*, const void* arg); -// TODO: Compression (Maybe just RLE) +int serialize(void* dst, const void* src, int avail, const void* list); +int deserialize(void* dst, const void* src, int avail, const void* list); +int serialize_size(const void*, const void* list); +int deserialize_size(const void*, const void* list); #endif // NESE_SERDES_H_