From 5b6c93a5bcc9d76ec878472e6b9d8e650d2e52c3 Mon Sep 17 00:00:00 2001 From: Joe Osborn Date: Sat, 30 Aug 2025 03:35:08 -0700 Subject: [PATCH] Incremental statestream checkpoints for replays (#18213) --- .dir-locals.el | 7 +- CHANGES.md | 5 + Makefile | 1 + Makefile.common | 6 + Makefile.emscripten | 1 + config.def.h | 5 + configuration.c | 3 + configuration.h | 4 + griffin/griffin.c | 4 + input/bsv/bsvmovie.c | 1444 ++++++++++++++++++++++++++++++++++ input/bsv/bsvmovie.h | 47 ++ input/bsv/uint32s_index.c | 363 +++++++++ input/bsv/uint32s_index.h | 70 ++ input/input_driver.c | 935 +--------------------- input/input_driver.h | 54 +- intl/msg_hash_lbl.h | 4 + intl/msg_hash_us.c | 3 + intl/msg_hash_us.h | 12 + libretro-db/rmsgpack.h | 2 +- libretro-db/rmsgpack_dom.c | 108 ++- libretro-db/rmsgpack_dom.h | 6 + menu/cbs/menu_cbs_sublabel.c | 4 + menu/menu_displaylist.c | 1 + menu/menu_setting.c | 15 + msg_hash.h | 1 + tasks/task_movie.c | 216 +++-- tasks/task_save.c | 1 - 27 files changed, 2269 insertions(+), 1053 deletions(-) create mode 100644 input/bsv/bsvmovie.c create mode 100644 input/bsv/bsvmovie.h create mode 100644 input/bsv/uint32s_index.c create mode 100644 input/bsv/uint32s_index.h diff --git a/.dir-locals.el b/.dir-locals.el index 0284f75bb9..109ec67d07 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -2,9 +2,12 @@ ;;; See Info node `(emacs) Directory Variables' for more information. ( - (c-mode . ((c-basic-offset . 3) + (c-mode . ((standard-indent . 3) + (c-basic-offset . 3) (c-file-offsets . ((arglist-intro . ++) - (arglist-cont-nonempty . ++))) + (arglist-cont-nonempty . ++) + (block-close . 0) + (block-open . 0))) (eval . (setq-local c-cleanup-list (cl-set-difference c-cleanup-list '(brace-else-brace diff --git a/CHANGES.md b/CHANGES.md index cd12c241aa..b252ae1323 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -31,6 +31,11 @@ - INPUT: Reset and close content hotkeys now require confirmation, similar to quit - INPUT/ANDROID: Favor mouse coordinates for lightgun - INPUT/UDEV: Fix lost terminal settings after restart from menu +- INPUT/BSV/REPLAY: Bumped replay format version to 2. Old replays will still play back fine. +- INPUT/BSV/REPLAY: Add option to skip deserializing checkpoints from replay files (it introduces jank in some emulators). +- INPUT/BSV/REPLAY: Add checkpoint and initial savestate compression, following the `savestate_file_compression` config boolean. Use zstd if available, or fall back to zlib. +- INPUT/BSV/REPLAY: Add incremental checkpoints based on statestreams (depending on `HAVE_STATESTREAM` compile time flag). As an example, 60 `pcsx_rearmed` savestates would take 267MB uncompressed; with incremental encoding this is reduced to 77MB. Compressing the result can reduce the size to just 4MB. +- INPUT/BSV/REPLAY: Checkpoint compression and encoding can be combined. For example, 60 `pcsx_rearmed` checkpoints can take up just 15MB if each state is incremental and compressed. This is not as optimal as using incremental states without save state compression followed by offline compression, but is a good compromise in many use cases. - INTL: Add Irish Gaelic to selectable languages - IOS: Fix crash on iOS9 when fetching refresh rate - LINUX: Add full complement of key/value pairs to desktop entry diff --git a/Makefile b/Makefile index e4c348b4b5..e17bb145f3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ HAVE_FILE_LOGGER=1 +HAVE_STATESTREAM?=1 NEED_CXX_LINKER?=0 NEED_GOLD_LINKER?=0 MISSING_DECLS =0 diff --git a/Makefile.common b/Makefile.common index 01738131e2..3a6dc9685c 100644 --- a/Makefile.common +++ b/Makefile.common @@ -25,6 +25,10 @@ ifeq ($(HAVE_SAPI), 1) LIBS += sapi.dll endif +ifeq ($(HAVE_STATESTREAM), 1) + DEF_FLAGS += -DHAVE_STATESTREAM +endif + ifeq ($(HAVE_GL_CONTEXT),) HAVE_GL_CONTEXT = 0 HAVE_GL_MODERN = 0 @@ -451,6 +455,8 @@ endif ifeq ($(HAVE_BSV_MOVIE), 1) DEFINES += -DHAVE_BSV_MOVIE + OBJ += input/bsv/bsvmovie.o \ + input/bsv/uint32s_index.o endif ifeq ($(HAVE_RUNAHEAD), 1) diff --git a/Makefile.emscripten b/Makefile.emscripten index d6d3751b8c..364d945df0 100644 --- a/Makefile.emscripten +++ b/Makefile.emscripten @@ -14,6 +14,7 @@ OBJ := DEFINES := -DRARCH_INTERNAL -DHAVE_MAIN -DEMSCRIPTEN DEFINES += -DHAVE_FILTERS_BUILTIN -DHAVE_ONLINE_UPDATER -DHAVE_UPDATE_ASSETS -DHAVE_UPDATE_CORE_INFO +HAVE_STATESTREAM ?= 1 HAVE_PATCH = 1 HAVE_DSP_FILTER = 1 HAVE_VIDEO_FILTER = 1 diff --git a/config.def.h b/config.def.h index 2357bb90ba..dd18d6d946 100644 --- a/config.def.h +++ b/config.def.h @@ -1407,6 +1407,11 @@ /* Specifies how often checkpoints will be saved to replay files during recording. * > Setting value to zero disables recording checkpoints. */ #define DEFAULT_REPLAY_CHECKPOINT_INTERVAL 0 +/* Specifies whether checkpoints in replay files should be loaded + * during playback. This can be helpful for cores that are not + * deterministic but in some cores produces janky results depending on + * when inputs are processed. */ +#define DEFAULT_REPLAY_CHECKPOINT_DESERIALIZE true /* Automatically saves a savestate at the end of RetroArch's lifetime. * The path is $SRAM_PATH.auto. diff --git a/configuration.c b/configuration.c index 0946599a7f..2f1e8f031f 100644 --- a/configuration.c +++ b/configuration.c @@ -2247,6 +2247,9 @@ static struct config_bool_setting *populate_settings_bool( SETTING_BOOL("network_remote_enable", &settings->bools.network_remote_enable, false, false /* TODO */, false); #endif #endif +#ifdef HAVE_BSV_MOVIE + SETTING_BOOL("replay_checkpoint_deserialize", &settings->bools.replay_checkpoint_deserialize, true, DEFAULT_REPLAY_CHECKPOINT_DESERIALIZE, false); +#endif #ifdef ANDROID SETTING_BOOL("android_input_disconnect_workaround", &settings->bools.android_input_disconnect_workaround, true, false, false); diff --git a/configuration.h b/configuration.h index a0a65b243e..073678a155 100644 --- a/configuration.h +++ b/configuration.h @@ -1108,6 +1108,10 @@ typedef struct settings bool ai_service_pause; bool gamemode_enable; +#ifdef HAVE_BSV_MOVIE + bool replay_checkpoint_deserialize; +#endif + #ifdef _3DS bool new3ds_speedup_enable; bool bottom_font_enable; diff --git a/griffin/griffin.c b/griffin/griffin.c index 78a519fb8c..d4792cd112 100644 --- a/griffin/griffin.c +++ b/griffin/griffin.c @@ -604,6 +604,10 @@ INPUT ============================================================ */ #include "../input/input_driver.c" +#ifdef HAVE_BSV_MOVIE +#include "../input/bsv/bsvmovie.c" +#include "../input/bsv/uint32s_index.c" +#endif #include "../input/input_keymaps.c" #include "../tasks/task_autodetect.c" #include "../input/input_autodetect_builtin.c" diff --git a/input/bsv/bsvmovie.c b/input/bsv/bsvmovie.c new file mode 100644 index 0000000000..cc9eebbe35 --- /dev/null +++ b/input/bsv/bsvmovie.c @@ -0,0 +1,1444 @@ +#include "bsvmovie.h" +#include +#include "../../retroarch.h" +#include "../../state_manager.h" +#include "../../verbosity.h" +#include "../input_driver.h" +#include "../../tasks/task_content.h" +#include "../../libretro-db/rmsgpack.h" +#include "../../libretro-db/rmsgpack_dom.h" +#ifdef HAVE_STATESTREAM +#include "input/bsv/uint32s_index.h" +#endif +#include "libretro.h" +#include "streams/interface_stream.h" +#ifdef HAVE_CHEEVOS +#include "../../cheevos/cheevos.h" +#endif + +#ifdef HAVE_ZLIB +#include +#endif + +#ifdef HAVE_ZSTD +#include +#endif + +#define BSV_IFRAME_START_TOKEN 0x00 +/* after START: + frame counter uint + state size (uncompressed) uint + new block and new superblock data (see below) + superblock seq (see below) + */ +#define BSV_IFRAME_NEW_BLOCK_TOKEN 0x01 +/* after NEW_BLOCK: + index uint + binary + */ +#define BSV_IFRAME_NEW_SUPERBLOCK_TOKEN 0x02 +/* after NEW_SUPERBLOCK: + index uint + array of uints +*/ +#define BSV_IFRAME_SUPERBLOCK_SEQ_TOKEN 0x03 +/* after SUPERBLOCK_SEQ: + array of uints + */ + +/* Later, tokens for pframes */ + +/* Forward declarations */ +void bsv_movie_free(bsv_movie_t*); + +#ifdef HAVE_STATESTREAM +int64_t bsv_movie_write_deduped_state(bsv_movie_t *movie, uint8_t *state, size_t state_size, uint8_t *output, size_t output_capacity); +bool bsv_movie_read_deduped_state(bsv_movie_t *movie, + uint8_t *encoded, size_t encoded_size, bool output); +#endif + +bool bsv_movie_reset_recording(bsv_movie_t *handle) +{ + size_t state_size, state_size_; + uint8_t compression = handle->checkpoint_compression; +#if HAVE_STATESTREAM + uint8_t encoding = REPLAY_CHECKPOINT2_ENCODING_STATESTREAM; + /* If recording, we simply reset + * the starting point. Nice and easy. */ + uint32s_index_clear(handle->superblocks); + uint32s_index_clear(handle->blocks); +#else + uint8_t encoding = REPLAY_CHECKPOINT2_ENCODING_RAW; +#endif + + intfstream_seek(handle->file, REPLAY_HEADER_LEN_BYTES, SEEK_SET); + intfstream_truncate(handle->file, REPLAY_HEADER_LEN_BYTES); + + intfstream_write(handle->file, &compression, 1); + intfstream_write(handle->file, &encoding, 1); + handle->frame_counter = 0; + state_size = 2 + bsv_movie_write_checkpoint(handle, compression, encoding); + handle->min_file_pos = intfstream_tell(handle->file); + /* Have to write initial state size header too */ + state_size_ = swap_if_big32(state_size); + intfstream_seek(handle->file, 3*sizeof(uint32_t), SEEK_SET); + intfstream_write(handle->file, &state_size_, sizeof(uint32_t)); + intfstream_seek(handle->file, handle->min_file_pos, SEEK_SET); + return true; +} + +void bsv_movie_enqueue(input_driver_state_t *input_st, + bsv_movie_t * state, enum bsv_flags flags) +{ + if (input_st->bsv_movie_state_next_handle) + bsv_movie_free(input_st->bsv_movie_state_next_handle); + input_st->bsv_movie_state_next_handle = state; + input_st->bsv_movie_state.flags = flags; +} + +void bsv_movie_deinit(input_driver_state_t *input_st) +{ + if (input_st->bsv_movie_state_handle) + bsv_movie_free(input_st->bsv_movie_state_handle); + input_st->bsv_movie_state_handle = NULL; +} + +void bsv_movie_deinit_full(input_driver_state_t *input_st) +{ + bsv_movie_deinit(input_st); + if (input_st->bsv_movie_state_next_handle) + bsv_movie_free(input_st->bsv_movie_state_next_handle); + input_st->bsv_movie_state_next_handle = NULL; +} + +void bsv_movie_frame_rewind() +{ + input_driver_state_t *input_st = input_state_get_ptr(); + bsv_movie_t *handle = input_st->bsv_movie_state_handle; + bool recording = (input_st->bsv_movie_state.flags + & BSV_FLAG_MOVIE_RECORDING) ? true : false; + + if (!handle) + return; + + handle->did_rewind = true; + handle->cur_save_valid = false; + if (((handle->frame_counter & handle->frame_mask) <= 1) + && (handle->frame_pos[0] == handle->min_file_pos)) + { + /* If we're at the beginning... */ + RARCH_LOG("[REPLAY] rewound to beginning\n"); + handle->frame_counter = 0; + intfstream_seek(handle->file, (int)handle->min_file_pos, SEEK_SET); + /* clear incremental checkpoint table data. We do this both on recording and playback for simplicity. */ +#ifdef HAVE_STATESTREAM + uint32s_index_remove_after(handle->superblocks, 0); + uint32s_index_remove_after(handle->blocks, 0); +#endif + if (recording) + intfstream_truncate(handle->file, (int)handle->min_file_pos); + else + bsv_movie_read_next_events(handle, false); + } + else + { + /* First time rewind is performed, the old frame is simply replayed. + * However, playing back that frame caused us to read data, and push + * data to the ring buffer. + * + * Successively rewinding frames, we need to rewind past the read data, + * plus another. */ + uint8_t delta = handle->first_rewind ? 1 : 2; + if (handle->frame_counter >= delta) + handle->frame_counter -= delta; + else + handle->frame_counter = 0; +#ifdef HAVE_STATESTREAM + uint32s_index_remove_after(handle->superblocks, handle->frame_counter); + uint32s_index_remove_after(handle->blocks, handle->frame_counter); +#endif + RARCH_LOG("[REPLAY] rewound to %d\n", handle->frame_counter); + intfstream_seek(handle->file, (int)handle->frame_pos[handle->frame_counter & handle->frame_mask], SEEK_SET); + if (recording) + intfstream_truncate(handle->file, (int)handle->frame_pos[handle->frame_counter & handle->frame_mask]); + else + bsv_movie_read_next_events(handle, false); + } + + if (intfstream_tell(handle->file) <= (long)handle->min_file_pos) + { + RARCH_LOG("[Replay] rewound past beginning\n"); + /* We rewound past the beginning. */ + if (handle->playback) + { + intfstream_seek(handle->file, (int)handle->min_file_pos, SEEK_SET); +#ifdef HAVE_STATESTREAM + uint32s_index_remove_after(handle->superblocks, 0); + uint32s_index_remove_after(handle->blocks, 0); +#endif + bsv_movie_read_next_events(handle, false); + } + else + { + bsv_movie_reset_recording(handle); + } + } +} + +void bsv_movie_push_key_event(bsv_movie_t *movie, + uint8_t down, uint16_t mod, uint32_t code, uint32_t character) +{ + bsv_key_data_t data; + data.down = down; + data._padding = 0; + data.mod = swap_if_big16(mod); + data.code = swap_if_big32(code); + data.character = swap_if_big32(character); + movie->key_events[movie->key_event_count] = data; + movie->key_event_count++; +} + +void bsv_movie_push_input_event(bsv_movie_t *movie, + uint8_t port, uint8_t dev, uint8_t idx, uint16_t id, int16_t val) +{ + bsv_input_data_t data; + data.port = port; + data.device = dev; + data.idx = idx; + data._padding = 0; + data.id = swap_if_big16(id); + data.value = swap_if_big16(val); + movie->input_events[movie->input_event_count] = data; + movie->input_event_count++; +} + +bool bsv_movie_handle_read_input_event(bsv_movie_t *movie, + uint8_t port, uint8_t dev, uint8_t idx, uint16_t id, int16_t* val) +{ + int i; + /* if movie is old, just read two bytes and hope for the best */ + if (movie->version == 0) + { + int64_t read = intfstream_read(movie->file, val, 2); + *val = swap_if_big16(*val); + return (read == 2); + } + for (i = 0; i < movie->input_event_count; i++) + { + bsv_input_data_t evt = movie->input_events[i]; + if ( (evt.port == port) + && (evt.device == dev) + && (evt.idx == idx) + && (evt.id == id)) + { + *val = swap_if_big16(evt.value); + return true; + } + } + return false; +} + +void bsv_movie_finish_rewind(input_driver_state_t *input_st) +{ + bsv_movie_t *handle = input_st->bsv_movie_state_handle; + if (!handle) + return; + handle->frame_counter += 1; + handle->first_rewind = !handle->did_rewind; + handle->did_rewind = false; +} + +bool bsv_movie_load_checkpoint(bsv_movie_t *handle, uint8_t compression, uint8_t encoding, bool just_update_structures) +{ + input_driver_state_t *input_st = input_state_get_ptr(); + uint32_t compressed_encoded_size, encoded_size, size; + uint8_t *compressed_data = NULL, *encoded_data = NULL; + retro_ctx_serialize_info_t serial_info; + bool ret = true; + + if (intfstream_read(handle->file, &(size), + sizeof(uint32_t)) != sizeof(uint32_t)) + { + RARCH_ERR("[Replay] Replay truncated before uncompressed unencoded size\n"); + ret = false; + goto exit; + } + if (intfstream_read(handle->file, &(encoded_size), + sizeof(uint32_t)) != sizeof(uint32_t)) + { + RARCH_ERR("[Replay] Replay truncated before uncompressed encoded size\n"); + ret = false; + goto exit; + } + if (intfstream_read(handle->file, &(compressed_encoded_size), + sizeof(uint32_t)) != sizeof(uint32_t)) + { + RARCH_ERR("[Replay] Replay truncated before compressed encoded size\n"); + ret = false; + goto exit; + } + size = swap_if_big32(size); + encoded_size = swap_if_big32(encoded_size); + compressed_encoded_size = swap_if_big32(compressed_encoded_size); + if (just_update_structures && encoding == REPLAY_CHECKPOINT2_ENCODING_RAW) + { + intfstream_seek(handle->file, compressed_encoded_size, SEEK_CUR); + goto exit; + } + + if (handle->cur_save_size < size) + { + free(handle->cur_save); + handle->cur_save = NULL; + } + if (!handle->cur_save) + { + handle->cur_save_size = size; + handle->cur_save = malloc(size); + handle->cur_save_valid = false; + } + + if (compression == REPLAY_CHECKPOINT2_COMPRESSION_NONE && encoding == REPLAY_CHECKPOINT2_ENCODING_RAW) + compressed_data = handle->cur_save; + else + compressed_data = malloc(compressed_encoded_size); + if (intfstream_read(handle->file, compressed_data, compressed_encoded_size) != (int64_t)compressed_encoded_size) + { + RARCH_ERR("[Replay] Truncated checkpoint, terminating movie\n"); + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + ret = false; + goto exit; + } + switch (compression) + { + case REPLAY_CHECKPOINT2_COMPRESSION_NONE: + encoded_data = compressed_data; + compressed_data = NULL; + break; +#ifdef HAVE_ZLIB + case REPLAY_CHECKPOINT2_COMPRESSION_ZLIB: + { +#ifdef EMSCRIPTEN + uLongf uncompressed_size_zlib = encoded_size; +#else + uint32_t uncompressed_size_zlib = encoded_size; +#endif + encoded_data = calloc(encoded_size, sizeof(uint8_t)); + if (uncompress(encoded_data, &uncompressed_size_zlib, compressed_data, compressed_encoded_size) != Z_OK) + { + ret = false; + goto exit; + } + break; + } +#endif +#ifdef HAVE_ZSTD + case REPLAY_CHECKPOINT2_COMPRESSION_ZSTD: + { + size_t uncompressed_size_big; + /* TODO: figure out how to support in-place decompression to + avoid allocating a second buffer; would need to allocate + the compressed_data buffer to be decompressed size + + margin. but, how could the margin be known without + calling the function that takes the compressed frames as + an input? */ + encoded_data = calloc(encoded_size, sizeof(uint8_t)); + uncompressed_size_big = ZSTD_decompress(encoded_data, encoded_size, compressed_data, compressed_encoded_size); + if (ZSTD_isError(uncompressed_size_big)) + { + ret = false; + goto exit; + } + break; + } +#endif + default: + RARCH_WARN("[Replay] Unrecognized compression scheme %d\n", compression); + ret = false; + goto exit; + } + switch (encoding) + { + case REPLAY_CHECKPOINT2_ENCODING_RAW: + size = encoded_size; + /* If decompression wasn't zerocopy, need to copy here; + otherwise decoding is also free */ + if (handle->cur_save != encoded_data) + memcpy(handle->cur_save, encoded_data, size); + else + encoded_data = NULL; + break; +#ifdef HAVE_STATESTREAM + case REPLAY_CHECKPOINT2_ENCODING_STATESTREAM: + if(!bsv_movie_read_deduped_state(handle, encoded_data, encoded_size, !just_update_structures)) + { + RARCH_ERR("[STATESTREAM] Couldn't load incremental checkpoint"); + ret = false; + goto exit; + } + break; +#endif + default: + RARCH_WARN("[Replay] Unrecognized encoding scheme %d\n", encoding); + ret = false; + goto exit; + } + if (just_update_structures) + goto exit; + serial_info.data_const = handle->cur_save; + serial_info.size = size; + /* TODO: should this happen at the end of the current frame, or at the beginning before inputs have been polled/etc? FCEUMM and PPSSPP have some jankiness here */ + if (!core_unserialize(&serial_info)) + { + abort(); + ret = false; + goto exit; + } + exit: + handle->cur_save_size = size; + handle->last_save_size = handle->cur_save_size; + + if (compressed_data) + free(compressed_data); + if (encoded_data) + free(encoded_data); + return ret; +} + +int64_t bsv_movie_write_checkpoint(bsv_movie_t *handle, uint8_t compression, uint8_t encoding) +{ + int64_t ret = -1; + uint32_t encoded_size, compressed_encoded_size, size_; + uint8_t *swap; + size_t size_swap; + uint8_t *encoded_data = NULL, *compressed_encoded_data = NULL; + bool owns_encoded = false, owns_compressed_encoded = false; + retro_ctx_serialize_info_t serial_info; + serial_info.size = core_serialize_size(); + if (handle->cur_save_size < serial_info.size) + { + free(handle->cur_save); + handle->cur_save = NULL; + } + if (!handle->cur_save) + { + handle->cur_save_size = serial_info.size; + handle->cur_save = malloc(serial_info.size); + handle->cur_save_valid = false; + } + serial_info.data = handle->cur_save; + core_serialize(&serial_info); + switch (encoding) + { + case REPLAY_CHECKPOINT2_ENCODING_RAW: + encoded_size = serial_info.size; + encoded_data = serial_info.data; + break; +#ifdef HAVE_STATESTREAM + case REPLAY_CHECKPOINT2_ENCODING_STATESTREAM: + encoded_size = serial_info.size + serial_info.size / 2; + encoded_data = malloc(encoded_size); + owns_encoded = true; + encoded_size = bsv_movie_write_deduped_state(handle, serial_info.data, serial_info.size, encoded_data, encoded_size); + break; +#endif + default: + RARCH_ERR("[Replay] Unrecognized encoding scheme %d\n", encoding); + ret = -1; + goto exit; + } + switch (compression) + { + case REPLAY_CHECKPOINT2_COMPRESSION_NONE: + compressed_encoded_size = encoded_size; + compressed_encoded_data = encoded_data; + break; +#ifdef HAVE_ZLIB + case REPLAY_CHECKPOINT2_COMPRESSION_ZLIB: + { + uLongf zlib_compressed_encoded_size = compressBound(encoded_size); + compressed_encoded_data = calloc(zlib_compressed_encoded_size, sizeof(uint8_t)); + owns_compressed_encoded = true; + if (compress2(compressed_encoded_data, &zlib_compressed_encoded_size, encoded_data, encoded_size, 6) != Z_OK) + { + ret = -1; + goto exit; + } + compressed_encoded_size = zlib_compressed_encoded_size; + break; + } +#endif +#ifdef HAVE_ZSTD + case REPLAY_CHECKPOINT2_COMPRESSION_ZSTD: + { + size_t compressed_encoded_size_big = ZSTD_compressBound(encoded_size); + compressed_encoded_data = calloc(compressed_encoded_size_big, sizeof(uint8_t)); + owns_compressed_encoded = true; + compressed_encoded_size_big = ZSTD_compress(compressed_encoded_data, compressed_encoded_size_big, encoded_data, encoded_size, 3); + if (ZSTD_isError(compressed_encoded_size_big)) + { + ret = -1; + goto exit; + } + compressed_encoded_size = compressed_encoded_size_big; + break; + } +#endif + default: + RARCH_WARN("[Replay] Unrecognized compression scheme %d\n", compression); + ret = -1; + goto exit; + } + /* uncompressed, unencoded size */ + size_ = swap_if_big32(serial_info.size); + if (intfstream_write(handle->file, &size_, sizeof(uint32_t)) < (int64_t)sizeof(uint32_t)) + { + ret = -1; + goto exit; + } + /* uncompressed, encoded size */ + size_ = swap_if_big32(encoded_size); + if (intfstream_write(handle->file, &size_, sizeof(uint32_t)) < (int64_t)sizeof(uint32_t)) + { + ret = -1; + goto exit; + } + /* compressed, encoded size */ + size_ = swap_if_big32(compressed_encoded_size); + if (intfstream_write(handle->file, &size_, sizeof(uint32_t)) < (int64_t)sizeof(uint32_t)) + { + ret = -1; + goto exit; + } + /* data */ + if (intfstream_write(handle->file, compressed_encoded_data, compressed_encoded_size) < compressed_encoded_size) + { + ret = -1; + goto exit; + } + ret = 3 * sizeof(uint32_t) + compressed_encoded_size; + exit: + size_swap = handle->cur_save_size; + handle->cur_save_size = handle->last_save_size; + handle->last_save_size = size_swap; + swap = handle->cur_save; + handle->cur_save = handle->last_save; + handle->last_save = swap; + handle->cur_save_valid = handle->cur_save != NULL; + if (encoded_data && owns_encoded) + free(encoded_data); + if (compressed_encoded_data && owns_compressed_encoded) + free(compressed_encoded_data); + return ret; +} + +bool bsv_movie_read_next_events(bsv_movie_t *handle, bool skip_checkpoints) +{ + input_driver_state_t *input_st = input_state_get_ptr(); + /* Skip over backref */ + intfstream_seek(handle->file, sizeof(uint32_t), SEEK_CUR); + /* Start by reading key event */ + if (intfstream_read(handle->file, &(handle->key_event_count), 1) == 1) + { + int i; + for (i = 0; i < handle->key_event_count; i++) + { + if (intfstream_read(handle->file, &(handle->key_events[i]), + sizeof(bsv_key_data_t)) != sizeof(bsv_key_data_t)) + { + /* Unnatural EOF */ + RARCH_ERR("[Replay] Keyboard replay ran out of keyboard inputs too early\n"); + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + return false; + } + } + } + else + { + RARCH_LOG("[Replay] EOF after buttons\n"); + /* Natural(?) EOF */ + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + return false; + } + if (handle->version > 0) + { + if (intfstream_read(handle->file, &(handle->input_event_count), 2) == 2) + { + int i; + handle->input_event_count = swap_if_big16(handle->input_event_count); + for (i = 0; i < handle->input_event_count; i++) + { + if (intfstream_read(handle->file, &(handle->input_events[i]), + sizeof(bsv_input_data_t)) != sizeof(bsv_input_data_t)) + { + /* Unnatural EOF */ + RARCH_ERR("[Replay] Input replay ran out of inputs too early\n"); + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + return false; + } + } + } + else + { + RARCH_LOG("[Replay] EOF after inputs\n"); + /* Natural(?) EOF */ + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + return false; + } + } + + { + uint8_t next_frame_type=REPLAY_TOKEN_INVALID; + if (intfstream_read(handle->file, (uint8_t *)(&next_frame_type), + sizeof(uint8_t)) != sizeof(uint8_t)) + { + /* Unnatural EOF */ + RARCH_ERR("[Replay] Replay ran out of frames\n"); + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + return false; + } + else if (next_frame_type == REPLAY_TOKEN_CHECKPOINT_FRAME) + { + uint64_t size; + uint8_t *state; + retro_ctx_serialize_info_t serial_info; + if (intfstream_read(handle->file, &(size), sizeof(uint64_t)) != sizeof(uint64_t)) + { + RARCH_ERR("[Replay] Replay ran out of frames\n"); + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + return false; + } + size = swap_if_big64(size); + if(skip_checkpoints) + intfstream_seek(handle->file, size, SEEK_CUR); + else + { + state = (uint8_t*)malloc(size); + if (intfstream_read(handle->file, state, size) != (int64_t)size) + { + RARCH_ERR("[Replay] Replay checkpoint truncated\n"); + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + free(state); + return false; + } + serial_info.data_const = state; + serial_info.size = size; + if (!core_unserialize(&serial_info)) + { + RARCH_ERR("[Replay] Failed to load movie checkpoint, failing\n"); + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + } + free(state); + } + } + else if (next_frame_type == REPLAY_TOKEN_CHECKPOINT2_FRAME) + { + uint8_t compression, encoding; + if (intfstream_read(handle->file, &(compression), sizeof(uint8_t)) != sizeof(uint8_t) || + intfstream_read(handle->file, &(encoding), sizeof(uint8_t)) != sizeof(uint8_t)) + { + /* Unexpected EOF */ + RARCH_ERR("[Replay] Replay checkpoint truncated.\n"); + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + return false; + } + if (!bsv_movie_load_checkpoint(handle, compression, encoding, skip_checkpoints)) + RARCH_WARN("[Replay] Failed to load movie checkpoint\n"); + } + } + return true; +} + +void bsv_movie_scan_from_start(input_driver_state_t *input_st, int32_t len) +{ + bsv_movie_t *movie = input_st->bsv_movie_state_handle; + if (movie->version == 0) + return; /* Old movies don't store enough information to fixup the frame counters. */ + intfstream_seek(movie->file, movie->min_file_pos, SEEK_SET); + movie->frame_counter = 0; + movie->frame_pos[0] = intfstream_tell(movie->file); + while(intfstream_tell(movie->file) < len && bsv_movie_read_next_events(movie, true)) + { + movie->frame_counter += 1; + movie->frame_pos[movie->frame_counter & movie->frame_mask] = intfstream_tell(movie->file); + } +} + +void bsv_movie_next_frame(input_driver_state_t *input_st) +{ + unsigned checkpoint_interval = config_get_ptr()->uints.replay_checkpoint_interval; + unsigned checkpoint_deserialize= config_get_ptr()->bools.replay_checkpoint_deserialize; + /* if bsv_movie_state_next_handle is not null, deinit and set + bsv_movie_state_handle to bsv_movie_state_next_handle and clear + next_handle */ + bsv_movie_t *handle = input_st->bsv_movie_state_handle; + if (input_st->bsv_movie_state_next_handle) + { + if (handle) + bsv_movie_deinit(input_st); + handle = input_st->bsv_movie_state_next_handle; + input_st->bsv_movie_state_handle = handle; + input_st->bsv_movie_state_next_handle = NULL; + } + + if (!handle) + return; +#ifdef HAVE_REWIND + if (state_manager_frame_is_reversed()) + return; +#endif + + if (input_st->bsv_movie_state.flags & BSV_FLAG_MOVIE_RECORDING) + { + int i; + uint16_t evt_count = swap_if_big16(handle->input_event_count); + size_t last_pos = handle->frame_pos[(MAX(handle->frame_counter,2)-2) & handle->frame_mask], cur_pos = intfstream_tell(handle->file); + uint32_t back_distance = swap_if_big32((uint32_t)(cur_pos-last_pos)); + /* write backref */ + intfstream_write(handle->file, &back_distance, sizeof(uint32_t)); + /* write key events, frame is over */ + intfstream_write(handle->file, &(handle->key_event_count), 1); + for (i = 0; i < handle->key_event_count; i++) + intfstream_write(handle->file, &(handle->key_events[i]), + sizeof(bsv_key_data_t)); + /* Zero out key events when playing back or recording */ + handle->key_event_count = 0; + /* write input events, frame is over */ + intfstream_write(handle->file, &evt_count, 2); + for (i = 0; i < handle->input_event_count; i++) + intfstream_write(handle->file, &(handle->input_events[i]), + sizeof(bsv_input_data_t)); + /* Zero out input events when playing back or recording */ + handle->input_event_count = 0; + + /* Maybe record checkpoint */ + if ( (checkpoint_interval != 0) + && (handle->frame_counter > 0) + && (handle->frame_counter % (checkpoint_interval*60) == 0)) + { + uint8_t frame_tok = REPLAY_TOKEN_CHECKPOINT2_FRAME; + uint8_t compression = handle->checkpoint_compression; +#if HAVE_STATESTREAM + uint8_t encoding = REPLAY_CHECKPOINT2_ENCODING_STATESTREAM; +#else + uint8_t encoding = REPLAY_CHECKPOINT2_ENCODING_RAW; +#endif + /* "next frame is a checkpoint" */ + intfstream_write(handle->file, (uint8_t *)(&frame_tok), sizeof(uint8_t)); + /* compression and encoding schemes */ + intfstream_write(handle->file, (uint8_t *)(&compression), sizeof(uint8_t)); + intfstream_write(handle->file, (uint8_t *)(&encoding), sizeof(uint8_t)); + if (bsv_movie_write_checkpoint(handle, compression, encoding) < 0) + { + RARCH_ERR("[Replay] failed to write checkpoint, exiting record\n"); + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + } + } + else + { + uint8_t frame_tok = REPLAY_TOKEN_REGULAR_FRAME; + /* write "next frame is not a checkpoint" */ + intfstream_write(handle->file, (uint8_t *)(&frame_tok), sizeof(uint8_t)); + } + } + + if (input_st->bsv_movie_state.flags & BSV_FLAG_MOVIE_PLAYBACK) + bsv_movie_read_next_events(handle, !checkpoint_deserialize); + handle->frame_pos[handle->frame_counter & handle->frame_mask] = intfstream_tell(handle->file); +} + +size_t replay_get_serialize_size(void) +{ + input_driver_state_t *input_st = input_state_get_ptr(); + if (input_st->bsv_movie_state.flags & (BSV_FLAG_MOVIE_RECORDING | BSV_FLAG_MOVIE_PLAYBACK)) + return sizeof(int32_t)+intfstream_tell(input_st->bsv_movie_state_handle->file); + return 0; +} + +bool replay_get_serialized_data(void* buffer) +{ + input_driver_state_t *input_st = input_state_get_ptr(); + bsv_movie_t *handle = input_st->bsv_movie_state_handle; + + if (input_st->bsv_movie_state.flags & (BSV_FLAG_MOVIE_RECORDING | BSV_FLAG_MOVIE_PLAYBACK)) + { + int32_t file_end = (uint32_t)intfstream_tell(handle->file); + int64_t read_amt = 0; + int32_t file_end_ = swap_if_big32(file_end); + uint8_t *buf; + ((uint32_t *)buffer)[0] = file_end_; + buf = ((uint8_t *)buffer) + sizeof(uint32_t); + intfstream_rewind(handle->file); + read_amt = intfstream_read(handle->file, buf, file_end); + ((uint32_t *)buffer)[1+REPLAY_HEADER_FRAME_COUNT_INDEX] = swap_if_big32(handle->frame_counter); + if (read_amt != file_end) + RARCH_ERR("[Replay] Failed to write correct number of replay bytes into state file: %d / %d.\n", + read_amt, file_end); + } + return true; +} + +bool replay_check_same_timeline(bsv_movie_t *movie, uint8_t *other_movie, int64_t other_len) +{ + int64_t check_limit = MIN(other_len, intfstream_tell(movie->file)); + intfstream_t *check_stream = intfstream_open_memory(other_movie, RETRO_VFS_FILE_ACCESS_READ, RETRO_VFS_FILE_ACCESS_HINT_NONE, other_len); + bool ret = true; + int64_t check_cap = MAX(128 << 10, MAX(128*sizeof(bsv_key_data_t), 512*sizeof(bsv_input_data_t))); + uint8_t *buf1 = calloc(check_cap,1), *buf2 = calloc(check_cap,1); + size_t movie_pos = intfstream_tell(movie->file); + uint8_t keycount1, keycount2, frametok1, frametok2; + uint16_t btncount1, btncount2; + uint64_t size1, size2; + intfstream_rewind(movie->file); + intfstream_read(movie->file, buf1, REPLAY_HEADER_LEN_BYTES); + intfstream_read(check_stream, buf2, REPLAY_HEADER_LEN_BYTES); + /* Don't want to compare frame counts */ + ((uint32_t *)buf1)[REPLAY_HEADER_FRAME_COUNT_INDEX] = 0; + ((uint32_t *)buf2)[REPLAY_HEADER_FRAME_COUNT_INDEX] = 0; + if (memcmp(buf1, buf2, REPLAY_HEADER_LEN_BYTES) != 0) + { + RARCH_ERR("[Replay] Headers of two movies differ, not same timeline\n"); + ret = false; + goto exit; + } + intfstream_seek(movie->file, movie->min_file_pos, SEEK_SET); + /* assumption: both headers have the same state size */ + intfstream_seek(check_stream, movie->min_file_pos, SEEK_SET); + if (movie->version == 0) + { + int64_t i; + /* no choice but to memcmp the whole stream against the other */ + for (i = 0; ret && i < check_limit; i+=check_cap) + { + int64_t read_end = MIN(check_limit - i, check_cap); + int64_t read1 = intfstream_read(movie->file, buf1, read_end); + int64_t read2 = intfstream_read(check_stream, buf2, read_end); + if (read1 != read_end || read2 != read_end || memcmp(buf1, buf2, read_end) != 0) + { + RARCH_ERR("[Replay] One or the other replay checkpoint has different byte values\n"); + ret = false; + goto exit; + } + } + goto exit; + } + while(intfstream_tell(movie->file) < check_limit && intfstream_tell(check_stream) < check_limit) + { + if (intfstream_tell(movie->file) < 0 || intfstream_tell(check_stream) < 0) + { + RARCH_ERR("[Replay] One or the other replay checkpoint has ended prematurely\n"); + ret = false; + goto exit; + } + /* skip past backref */ + intfstream_seek(movie->file, 4, SEEK_CUR); + intfstream_seek(check_stream, 4, SEEK_CUR); + if (intfstream_read(movie->file, &keycount1, 1) < 1 || + intfstream_read(check_stream, &keycount2, 1) < 1 || + keycount1 != keycount2) + { + RARCH_ERR("[Replay] Replay checkpoints disagree on key count, %d vs %d\n", keycount1, keycount2); + ret = false; + goto exit; + } + if ((uint64_t)intfstream_read(movie->file, buf1, keycount1*sizeof(bsv_key_data_t)) < keycount1*sizeof(bsv_key_data_t) || + (uint64_t)intfstream_read(check_stream, buf2, keycount2*sizeof(bsv_key_data_t)) < keycount2*sizeof(bsv_key_data_t) || + memcmp(buf1, buf2, keycount1*sizeof(bsv_key_data_t)) != 0) + { + RARCH_ERR("[Replay] Replay checkpoints disagree on key data\n"); + ret = false; + goto exit; + } + if (intfstream_read(movie->file, &btncount1, 2) < 2 || + intfstream_read(check_stream, &btncount2, 2) < 2 || + btncount1 != btncount2) + { + RARCH_ERR("[Replay] Replay checkpoints disagree on input count\n"); + ret = false; + goto exit; + } + btncount1 = swap_if_big16(btncount1); + btncount2 = swap_if_big16(btncount2); + if ((uint64_t)intfstream_read(movie->file, buf1, btncount1*sizeof(bsv_input_data_t)) < btncount1*sizeof(bsv_input_data_t) || + (uint64_t)intfstream_read(check_stream, buf2, btncount2*sizeof(bsv_input_data_t)) < btncount2*sizeof(bsv_input_data_t) || + memcmp(buf1, buf2, btncount1*sizeof(bsv_input_data_t)) != 0) + { + RARCH_ERR("[Replay] Replay checkpoints disagree on input data\n"); + ret = false; + goto exit; + } + if (intfstream_read(movie->file, &frametok1, 1) < 1 || + intfstream_read(check_stream, &frametok2, 1) < 1 || + frametok1 != frametok2) + { + RARCH_ERR("[Replay] Replay checkpoints disagree on frame token\n"); + ret = false; + goto exit; + } + switch (frametok1) + { + case REPLAY_TOKEN_INVALID: + RARCH_ERR("[Replay] Both replays are somehow invalid\n"); + ret = false; + goto exit; + case REPLAY_TOKEN_REGULAR_FRAME: + break; + case REPLAY_TOKEN_CHECKPOINT_FRAME: + if ((uint64_t)intfstream_read(movie->file, &size1, sizeof(uint64_t)) < sizeof(uint64_t) || + (uint64_t)intfstream_read(check_stream, &size2, sizeof(uint64_t)) < sizeof(uint64_t) || + size1 != size2) + { + RARCH_ERR("[Replay] Replay checkpoints disagree on size or scheme\n"); + ret = false; + goto exit; + } + size1 = swap_if_big64(size1); + intfstream_seek(movie->file, size1, SEEK_CUR); + intfstream_seek(check_stream, size1, SEEK_CUR); + break; + case REPLAY_TOKEN_CHECKPOINT2_FRAME: + { + uint32_t cpsize1, cpsize2; + /* read cp2 header: + - one byte compression codec, one byte encoding scheme + - 4 byte uncompressed unencoded size, 4 byte uncompressed encoded size + - 4 byte compressed, encoded size + - the data will follow + */ + if (intfstream_read(movie->file, buf1, 2+sizeof(uint32_t)*3) != 2+sizeof(uint32_t)*3 || + intfstream_read(check_stream, buf2, 2+sizeof(uint32_t)*3) != 2+sizeof(uint32_t)*3 || + memcmp(buf1, buf2, 2+sizeof(uint32_t)*3) != 0 + ) + { + ret = false; + goto exit; + } + memcpy(&cpsize1, buf1+10, sizeof(uint32_t)); + memcpy(&cpsize2, buf2+10, sizeof(uint32_t)); + cpsize1 = swap_if_big32(cpsize1); + cpsize2 = swap_if_big32(cpsize2); + intfstream_seek(movie->file, cpsize1, SEEK_CUR); + intfstream_seek(check_stream, cpsize2, SEEK_CUR); + break; + } + default: + RARCH_ERR("[Replay] Unrecognized frame token in both replays\n"); + ret = false; + goto exit; + } + } + exit: + free(buf1); + free(buf2); + intfstream_close(check_stream); + intfstream_seek(movie->file, movie_pos, SEEK_SET); + return ret; +} + +bool replay_set_serialized_data(void* buf) +{ + uint8_t *buffer = buf; + input_driver_state_t *input_st = input_state_get_ptr(); + bsv_movie_t *handle = input_st->bsv_movie_state_handle; + bool playback = (input_st->bsv_movie_state.flags & BSV_FLAG_MOVIE_PLAYBACK) ? true : false; + bool recording = (input_st->bsv_movie_state.flags & BSV_FLAG_MOVIE_RECORDING) ? true : false; + /* If there is no current replay, ignore this entirely. + TODO/FIXME: Later, consider loading up the replay + and allow the user to continue it? + Or would that be better done from the replay hotkeys? + */ + if (!(playback || recording)) + return true; + + if (!handle) + return false; + handle->cur_save_valid = false; + if (!buffer) + { + if (recording) + { + const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_FAILED_INCOMPAT); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + RARCH_ERR("[Replay] %s.\n", _msg); + return false; + } + + if (playback) + { + const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_HALT_INCOMPAT); + runloop_msg_queue_push(_msg, sizeof(_msg), 1, 180, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_WARNING); + RARCH_WARN("[Replay] %s.\n", _msg); + movie_stop(input_st); + } + } + else + { + /* TODO: should factor the next few lines away, magic numbers ahoy */ + uint32_t *header = (uint32_t *)(buffer + sizeof(uint32_t)); + int64_t *ident_spot = (int64_t *)(header + REPLAY_HEADER_IDENTIFIER_INDEX); + int64_t ident; + /* avoid unaligned 8-byte read */ + memcpy(&ident, ident_spot, sizeof(int64_t)); + ident = swap_if_big64(ident); + + if (ident == handle->identifier) /* is compatible? */ + { + int32_t loaded_len = swap_if_big32(((int32_t *)buffer)[0]); + int64_t handle_idx = intfstream_tell(handle->file); + bool same_timeline = replay_check_same_timeline(handle, (uint8_t *)header, loaded_len); + /* If the state is part of this replay, go back to that state + and fast forward/rewind the replay. + + If the savestate movie is after the current replay + length we can replace the current replay data with it, + but if it's earlier we can rewind the replay to the + savestate movie time point. + + This can truncate the current replay if we're in recording mode. + */ + if (playback && loaded_len > handle_idx) + { + const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_FAILED_FUTURE_STATE); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + RARCH_ERR("[Replay] %s.\n", _msg); + return false; + } + else if (playback && !same_timeline) + { + const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_FAILED_WRONG_TIMELINE); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + RARCH_ERR("[Replay] %s.\n", _msg); + return false; + } + else if (recording && (loaded_len > handle_idx || !same_timeline)) + { + if (!same_timeline) + { + const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_OVERWRITING_REPLAY); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_WARNING); + RARCH_WARN("[Replay] %s.\n", _msg); + } +#ifdef HAVE_STATESTREAM + uint32s_index_remove_after(handle->superblocks, 0); + uint32s_index_remove_after(handle->blocks, 0); +#endif + intfstream_rewind(handle->file); + intfstream_write(handle->file, header, loaded_len); + /* also need to update/reinit frame_pos, + frame_counter--rewind won't work properly unless we do. */ + /* TODO: in the future, if same_timeline, don't clear + indices above and only scan forward from handle_idx. */ + /* TODO use backrefs to help here */ + bsv_movie_scan_from_start(input_st, loaded_len); + } + else + { +#ifdef HAVE_STATESTREAM + uint32s_index_remove_after(handle->superblocks, 0); + uint32s_index_remove_after(handle->blocks, 0); +#endif + intfstream_seek(handle->file, loaded_len, SEEK_SET); + /* TODO: in the future, don't clear indices above and only + update frame counter and remove index entries after the + loaded movie's frame counter */ + /* TODO use backrefs to help here */ + bsv_movie_scan_from_start(input_st, loaded_len); + if (recording) + intfstream_truncate(handle->file, loaded_len); + } + } + else + { + /* otherwise, if recording do not allow the load */ + if (recording) + { + const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_FAILED_INCOMPAT); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + RARCH_ERR("[Replay] %s.\n", _msg); + return false; + } + /* if in playback, halt playback and go to that state normally */ + if (playback) + { + const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_HALT_INCOMPAT); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_WARNING); + RARCH_WARN("[Replay] %s.\n", _msg); + movie_stop(input_st); + } + } + } + return true; +} + + +void bsv_movie_poll(input_driver_state_t *input_st) { + runloop_state_t *runloop_st = runloop_state_get_ptr(); + retro_keyboard_event_t *key_event = &runloop_st->key_event; + bsv_movie_t *handle = input_st->bsv_movie_state_handle; + if (*key_event && *key_event == runloop_st->frontend_key_event) + { + int i; + bsv_key_data_t k; + for (i = 0; i < handle->key_event_count; i++) + { +#ifdef HAVE_CHEEVOS + rcheevos_pause_hardcore(); +#endif + k = handle->key_events[i]; + input_keyboard_event(k.down, swap_if_big32(k.code), + swap_if_big32(k.character), swap_if_big16(k.mod), + RETRO_DEVICE_KEYBOARD); + } + /* Have to clear here so we don't double-apply key events */ + /* Zero out key events when playing back or recording */ + handle->key_event_count = 0; + } +} + +int16_t bsv_movie_read_state(input_driver_state_t *input_st, + unsigned port, unsigned device, + unsigned idx, unsigned id) { + int16_t bsv_result = 0; + bsv_movie_t *movie = input_st->bsv_movie_state_handle; + if (bsv_movie_handle_read_input_event(movie, port, device, idx, id, &bsv_result)) + { +#ifdef HAVE_CHEEVOS + rcheevos_pause_hardcore(); +#endif + return bsv_result; + } + input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; + return 0; +} + +#ifdef HAVE_STATESTREAM +int64_t bsv_movie_write_deduped_state(bsv_movie_t *movie, uint8_t *state, size_t state_size, uint8_t *output, size_t output_capacity) +{ + static uint32_t skipped_blocks = 0; + static uint32_t memcmps = 0, hashes = 0; + static uint32_t reused_blocks = 0; + static uint32_t reused_superblocks = 0; + static uint32_t total_blocks = 0; + static uint32_t total_superblocks = 0; + static uint32_t total_checkpoints = 0; + static uint32_t total_kbs_input = 0; + static uint32_t total_kbs_written = 0; + static retro_perf_tick_t total_encode_micros = 0; + retro_perf_tick_t start = cpu_features_get_time_usec(); + size_t block_byte_size = movie->blocks->object_size*4; + size_t superblock_size = movie->superblocks->object_size; + size_t superblock_byte_size = superblock_size*block_byte_size; + size_t superblock_count = state_size / superblock_byte_size + (state_size % superblock_byte_size != 0); + uint32_t *superblock_buf = calloc(superblock_size, sizeof(uint32_t)); + uint8_t *padded_block = NULL; + intfstream_t *out_stream = intfstream_open_writable_memory(output, RETRO_VFS_FILE_ACCESS_READ_WRITE, RETRO_VFS_FILE_ACCESS_HINT_NONE, output_capacity); + int64_t encoded_size; + size_t superblock, block; + uint32_t i; + bool can_compare_saves = movie->cur_save_valid && movie->last_save && movie->last_save_size >= state_size; + if (movie->last_save_size < state_size) + { + free(movie->superblock_seq); + movie->superblock_seq = NULL; + } + if (!movie->superblock_seq) + { + movie->cur_save_valid = false; + movie->superblock_seq = calloc(superblock_count, sizeof(uint32_t)); + } + rmsgpack_write_int(out_stream, BSV_IFRAME_START_TOKEN); + rmsgpack_write_int(out_stream, movie->frame_counter); + for(superblock = 0; superblock < superblock_count; superblock++) + { + uint32s_insert_result_t found_block; + total_superblocks++; + for(block = 0; block < superblock_size; block++) + { + size_t block_start = superblock*superblock_byte_size+block*block_byte_size; + size_t block_end = MIN(block_start + block_byte_size, state_size); + if(block_start > state_size) + { + /* pad superblocks with zero blocks */ + found_block.index = 0; + found_block.is_new = false; + } + else if (can_compare_saves && + (++memcmps) && + memcmp(movie->last_save + block_start, + state + block_start, + block_end-block_start) == 0) + { + skipped_blocks++; + found_block.index = uint32s_index_get(movie->superblocks, + movie->superblock_seq[superblock])[block]; + found_block.is_new = false; + /* bump usage count */ + uint32s_index_bump_count(movie->blocks, found_block.index); + } + else if(block_start + block_byte_size > state_size) + { + if(!padded_block) + padded_block = calloc(block_byte_size, sizeof(uint8_t)); + else + memset(padded_block+(state_size-block_start), + 0, + block_byte_size-(state_size-block_start)); + memcpy(padded_block, state+block_start, state_size - block_start); + found_block = uint32s_index_insert(movie->blocks, + (uint32_t*)padded_block, + movie->frame_counter); + hashes++; + } + else + { + hashes++; + found_block = uint32s_index_insert(movie->blocks, + (uint32_t*)(state+block_start), + movie->frame_counter); + } + total_blocks++; + if(found_block.is_new) + { + /* write "here is a new block" and new block to file */ + rmsgpack_write_int(out_stream, BSV_IFRAME_NEW_BLOCK_TOKEN); + rmsgpack_write_int(out_stream, found_block.index); + rmsgpack_write_bin(out_stream, state+block_start, block_byte_size); + } + else + reused_blocks++; + superblock_buf[block] = found_block.index; + } + found_block = uint32s_index_insert(movie->superblocks, superblock_buf, movie->frame_counter); + if(found_block.is_new) + { + /* write "here is a new superblock" and new superblock to file */ + rmsgpack_write_int(out_stream, BSV_IFRAME_NEW_SUPERBLOCK_TOKEN); + rmsgpack_write_int(out_stream, found_block.index); + rmsgpack_write_array_header(out_stream, superblock_size); + for(i = 0; i < superblock_size; i++) + rmsgpack_write_int(out_stream, superblock_buf[i]); + } + else + reused_superblocks++; + movie->superblock_seq[superblock] = found_block.index; + } + uint32s_index_commit(movie->blocks); + /* Superblocks are small enough that there's no real benefit to garbage collecting them */ + /* uint32s_index_commit(movie->superblocks); */ + /* write "here is the superblock seq" and superblock seq to file */ + rmsgpack_write_int(out_stream, BSV_IFRAME_SUPERBLOCK_SEQ_TOKEN); + rmsgpack_write_array_header(out_stream, superblock_count); + for(i = 0; i < superblock_count; i++) + rmsgpack_write_int(out_stream, movie->superblock_seq[i]); + free(superblock_buf); + if(padded_block) + free(padded_block); + movie->cur_save_valid = true; + total_checkpoints++; + total_encode_micros += cpu_features_get_time_usec() - start; + total_kbs_input += state_size/1024; + encoded_size = intfstream_tell(out_stream); + total_kbs_written += encoded_size/1024; + RARCH_DBG("[STATESTREAM] Encode stats at checkpoint %d: %d blocks (%d reused, %d skipped [%d checks], %d distinct [%d hashes])\n", total_checkpoints, total_blocks, reused_blocks, skipped_blocks, memcmps, uint32s_index_count(movie->blocks), hashes); + RARCH_DBG("[STATESTREAM] %d superblocks (%d reused, %d distinct); unencoded size (KB) %d, encoded size (KB) %d; net time (secs) %f\n", total_superblocks, reused_superblocks, uint32s_index_count(movie->superblocks), total_kbs_input, total_kbs_written, ((float)total_encode_micros) / (float)1000000.0); + intfstream_close(out_stream); + return encoded_size; +} + +bool bsv_movie_read_deduped_state(bsv_movie_t *movie, + uint8_t *encoded, size_t encoded_size, bool output) +{ + static retro_perf_tick_t total_decode_micros = 0; + static retro_perf_tick_t total_decode_count = 0; + retro_perf_tick_t start = cpu_features_get_time_usec(); + bool ret = false; + size_t state_size = movie->cur_save_size; + /*uint32_t frame_counter = 0;*/ + struct rmsgpack_dom_value item; + struct rmsgpack_dom_reader_state *reader_state = rmsgpack_dom_reader_state_new(); + size_t block_byte_size = movie->blocks->object_size*4; + size_t superblock_byte_size = movie->superblocks->object_size*block_byte_size; + intfstream_t *read_mem = intfstream_open_memory(encoded, RETRO_VFS_FILE_ACCESS_READ, RETRO_VFS_FILE_ACCESS_HINT_NONE, encoded_size); + size_t i; + if (state_size > movie->last_save_size && movie->superblock_seq) + { + free(movie->superblock_seq); + movie->superblock_seq = NULL; + } + total_decode_count++; + rmsgpack_dom_read_with(read_mem, &item, reader_state); + if(item.type != RDT_INT) + { + RARCH_ERR("[STATESTREAM] start token type is wrong\n"); + goto exit; + } + if(item.val.int_ != BSV_IFRAME_START_TOKEN) + { + RARCH_ERR("[STATESTREAM] start token value is wrong\n"); + goto exit; + } + rmsgpack_dom_read_with(read_mem, &item, reader_state); + if(item.type != RDT_INT) + { + RARCH_ERR("[STATESTREAM] frame counter type is wrong\n"); + goto exit; + } + /*frame_counter = item.val.int_;*/ + while(rmsgpack_dom_read_with(read_mem, &item, reader_state) >= 0) + { + uint32_t index, *superblock; + size_t len; + if(item.type != RDT_INT) + { + RARCH_ERR("[STATESTREAM] state update chunk token type is wrong\n"); + goto exit; + } + switch(item.val.int_) { + case BSV_IFRAME_NEW_BLOCK_TOKEN: + rmsgpack_dom_read_with(read_mem, &item, reader_state); + if(item.type != RDT_INT) + { + RARCH_ERR("[STATESTREAM] new block index type is wrong\n"); + goto exit; + } + index = item.val.int_; + rmsgpack_dom_read_with(read_mem, &item, reader_state); + if(item.type != RDT_BINARY) + { + RARCH_ERR("[STATESTREAM] new block value type is wrong\n"); + rmsgpack_dom_value_free(&item); + goto exit; + } + if(item.val.binary.len != block_byte_size) + { + RARCH_ERR("[STATESTREAM] new block binary length is wrong: %d vs %d\n", item.val.binary.len, block_byte_size); + rmsgpack_dom_value_free(&item); + goto exit; + } + if(!uint32s_index_insert_exact(movie->blocks, index, (uint32_t *)item.val.binary.buff, movie->frame_counter)) + { + RARCH_ERR("[STATESTREAM] couldn't insert new block at right index %d\n", index); + rmsgpack_dom_value_free(&item); + goto exit; + } + /* do not free binary rmsgpack item since insert_exact takes over its allocation */ + break; + case BSV_IFRAME_NEW_SUPERBLOCK_TOKEN: + rmsgpack_dom_read_with(read_mem, &item, reader_state); + if(item.type != RDT_INT) + { + RARCH_ERR("[STATESTREAM] new superblock index type is wrong\n"); + goto exit; + } + index = item.val.int_; + if(rmsgpack_dom_read_with(read_mem, &item, reader_state) < 0) + { + RARCH_ERR("[STATESTREAM] array read failed\n"); + goto exit; + } + if(item.type != RDT_ARRAY) + { + RARCH_ERR("[STATESTREAM] new superblock contents type is wrong\n"); + goto exit; + } + if(item.val.array.len != movie->superblocks->object_size) + { + RARCH_ERR("[STATESTREAM] new superblock contents length is wrong\n"); + goto exit; + } + len = movie->superblocks->object_size; + superblock = calloc(len, sizeof(uint32_t)); + for(i = 0; i < len; i++) + { + struct rmsgpack_dom_value inner_item = item.val.array.items[i]; + /* assert(inner_item.type == RDT_INT); */ + superblock[i] = inner_item.val.int_; + } + if(!uint32s_index_insert_exact(movie->superblocks, index, superblock, movie->frame_counter)) + { + RARCH_ERR("[STATESTREAM] new superblock couldn't be inserted at right index\n"); + rmsgpack_dom_value_free(&item); + free(superblock); + goto exit; + } + /* Do not free superblock since insert_exact takes over its allocation */ + rmsgpack_dom_value_free(&item); + break; + case BSV_IFRAME_SUPERBLOCK_SEQ_TOKEN: + rmsgpack_dom_read_with(read_mem, &item, reader_state); + if(item.type != RDT_ARRAY) + { + RARCH_ERR("[STATESTREAM] superblock seq type is wrong\n"); + goto exit; + } + len = item.val.array.len; + if (output) + { + if (!movie->superblock_seq) + movie->superblock_seq = calloc(len,sizeof(uint32_t)); + for(i = 0; i < len; i++) + { + struct rmsgpack_dom_value inner_item = item.val.array.items[i]; + /* assert(inner_item.type == RDT_INT); */ + uint32_t superblock_idx = inner_item.val.int_; + uint32_t *superblock; + size_t j; + /* if this superblock is the same as last time, no need to scan the blocks. */ + if (movie->cur_save_valid && movie->cur_save && superblock_idx == movie->superblock_seq[i]) + { + superblock = uint32s_index_get(movie->superblocks, movie->superblock_seq[i]); + uint32s_index_bump_count(movie->superblocks, movie->superblock_seq[i]); + /* We do need to increment all the involved block counts though */ + for (j = 0; j < movie->superblocks->object_size; j++) + uint32s_index_bump_count(movie->blocks, superblock[j]); + continue; + } + movie->superblock_seq[i] = superblock_idx; + superblock = uint32s_index_get(movie->superblocks, superblock_idx); + uint32s_index_bump_count(movie->superblocks, superblock_idx); + for(j = 0; j < movie->superblocks->object_size; j++) + { + uint32_t block_idx = superblock[j]; + size_t block_start = MIN(i*superblock_byte_size+j*block_byte_size, state_size); + size_t block_end = MIN(block_start+block_byte_size, state_size); + uint8_t *block; + /* This (==) can only happen in the last superblock, if it was padded with extra blocks. */ + if(block_end <= block_start) { break; } + block = (uint8_t *)uint32s_index_get(movie->blocks, block_idx); + uint32s_index_bump_count(movie->blocks, block_idx); + memcpy(movie->cur_save+block_start, (uint8_t*)block, block_end-block_start); + } + } + + } + rmsgpack_dom_value_free(&item); + ret = true; + goto exit; + default: + RARCH_ERR("[STATESTREAM] state update chunk token value is invalid: %d @ %x\n", item.val.int_, intfstream_tell(read_mem)); + goto exit; + } + } +exit: + uint32s_index_commit(movie->blocks); + /* Superblocks are small enough that there's no real benefit to garbage collecting them */ + /* uint32s_index_commit(movie->superblocks); */ + rmsgpack_dom_reader_state_free(reader_state); + intfstream_close(read_mem); + if(!ret) + { + RARCH_ERR("[STATESTREAM] made it to end without superblock seq\n"); + abort(); + } + total_decode_micros += cpu_features_get_time_usec() - start; + RARCH_DBG("[STATESTREAM] Total statestream decodes %d ; net time (secs): %f\n", total_decode_count, (double)total_decode_micros / (1000000.0)); + return ret; +} +#endif diff --git a/input/bsv/bsvmovie.h b/input/bsv/bsvmovie.h new file mode 100644 index 0000000000..c7dc086bea --- /dev/null +++ b/input/bsv/bsvmovie.h @@ -0,0 +1,47 @@ +#ifndef __BSV_MOVIE__H +#define __BSV_MOVIE__H + +#include +#include "../input_driver.h" +#include + +#define REPLAY_HEADER_MAGIC_INDEX 0 +#define REPLAY_HEADER_VERSION_INDEX 1 +#define REPLAY_HEADER_CRC_INDEX 2 +#define REPLAY_HEADER_STATE_SIZE_INDEX 3 +/* Identifier is int64_t, so takes up two slots */ +#define REPLAY_HEADER_IDENTIFIER_INDEX 4 +#define REPLAY_HEADER_FRAME_COUNT_INDEX 6 +#define REPLAY_HEADER_BLOCK_SIZE_INDEX 7 +#define REPLAY_HEADER_SUPERBLOCK_SIZE_INDEX 8 +#define REPLAY_HEADER_CHECKPOINT_CONFIG_INDEX 9 +#define REPLAY_HEADER_LEN 10 +#define REPLAY_HEADER_LEN_BYTES (REPLAY_HEADER_LEN*4) + +#define REPLAY_HEADER_V0V1_LEN 6 +#define REPLAY_HEADER_V0V1_LEN_BYTES (REPLAY_HEADER_V0V1_LEN*4) + +#define REPLAY_FORMAT_VERSION 2 +#define REPLAY_MAGIC 0x42535632 + + +RETRO_BEGIN_DECLS + +void bsv_movie_poll(input_driver_state_t *input_st); +int16_t bsv_movie_read_state(input_driver_state_t *input_st, + unsigned port, unsigned device, + unsigned idx, unsigned id); +void bsv_movie_push_key_event(bsv_movie_t *movie, + uint8_t down, uint16_t mod, uint32_t code, uint32_t character); +void bsv_movie_push_input_event(bsv_movie_t *movie, + uint8_t port, uint8_t dev, uint8_t idx, uint16_t id, int16_t val); + +bool bsv_movie_load_checkpoint(bsv_movie_t *movie, + uint8_t compression, uint8_t encoding, + bool just_update_structures); +int64_t bsv_movie_write_checkpoint(bsv_movie_t *movie, + uint8_t compression, uint8_t encoding); + +RETRO_END_DECLS + +#endif /* __BSV_MOVIE__H */ diff --git a/input/bsv/uint32s_index.c b/input/bsv/uint32s_index.c new file mode 100644 index 0000000000..2191a1141e --- /dev/null +++ b/input/bsv/uint32s_index.c @@ -0,0 +1,363 @@ +#ifdef HAVE_STATESTREAM +#include "uint32s_index.h" +#include +#include +#include +#include "../../verbosity.h" + +#define XXH_INLINE_ALL +#include + +#define HASHMAP_CAP (1<<16) +#define uint32s_hash_bytes(bytes, len) XXH32(bytes,len,0) + +uint32s_index_t *uint32s_index_new(size_t object_size, uint8_t commit_interval, uint8_t commit_threshold) +{ + uint32_t *zeros = calloc(object_size, sizeof(uint32_t)); + uint32s_index_t *index = malloc(sizeof(uint32s_index_t)); + index->object_size = object_size; + index->index = NULL; + RHMAP_FIT(index->index, HASHMAP_CAP); + index->objects = NULL; + index->counts = NULL; + index->hashes = NULL; + index->additions = NULL; + index->commit_interval = commit_interval; + index->commit_threshold = commit_threshold; + /* transfers ownership of zero buffer */ + uint32s_index_insert_exact(index, 0, zeros, 0); + RBUF_CLEAR(index->additions); /* scrap first addition, we never want to delete 0s during rewind */ + return index; +} + +void uint32s_bucket_free(struct uint32s_bucket *bucket) +{ + if(bucket->len > 3) + free(bucket->contents.vec.idxs); +} + + +bool uint32s_bucket_get(uint32s_index_t *index, struct uint32s_bucket *bucket, uint32_t *object, size_t size_bytes, uint32_t *out_idx) +{ + uint32_t *coll = bucket->len < 4 ? bucket->contents.idxs : bucket->contents.vec.idxs; + uint32_t i; + for(i = 0; i < bucket->len; i++) + { + uint32_t idx = coll[i]; + if(memcmp(index->objects[idx], object, size_bytes) == 0) + { + *out_idx = idx; + return true; + } + } + return false; +} + +void uint32s_bucket_expand(struct uint32s_bucket *bucket, uint32_t idx) +{ + if(bucket->len < 3) + { + bucket->contents.idxs[bucket->len] = idx; + bucket->len++; + } + else if(bucket->len == 3) + { + uint32_t *idxs = calloc(8, sizeof(uint32_t)); + memcpy(idxs, bucket->contents.idxs, 3*sizeof(uint32_t)); + bucket->contents.vec.cap = 8; + bucket->contents.vec.idxs = idxs; + bucket->contents.vec.idxs[bucket->len] = idx; + bucket->len++; + } + else if(bucket->len < bucket->contents.vec.cap) + { + bucket->contents.vec.idxs[bucket->len] = idx; + bucket->len++; + } + else /* bucket->len == bucket->contents.vec.cap */ + { + bucket->contents.vec.cap *= 2; + bucket->contents.vec.idxs = realloc(bucket->contents.vec.idxs, bucket->contents.vec.cap*sizeof(uint32_t)); + bucket->contents.vec.idxs[bucket->len] = idx; + bucket->len++; + } +} + +bool uint32s_bucket_remove(struct uint32s_bucket *bucket, uint32_t idx) +{ + bool small = bucket->len < 4; + uint32_t *coll = small ? bucket->contents.idxs : bucket->contents.vec.idxs; + int i; + if(idx == 0) /* never remove 0s pattern */ + return false; + for(i = 0; i < (int)bucket->len; i++) + { + if(coll[i] == idx) + { + memmove((uint8_t*)(coll+i), (uint8_t*)(coll+i+1), (bucket->len-(i+1))*sizeof(uint32_t)); + bucket->len--; + if (bucket->len == 3) + { + memcpy(bucket->contents.idxs, coll, 3*sizeof(uint32_t)); + free(coll); + } + return true; + } + } + RARCH_ERR("[STATESTREAM] didn't find index %d during remove\n",idx); + return false; +} + +uint32s_insert_result_t uint32s_index_insert(uint32s_index_t *index, uint32_t *object, uint64_t frame) +{ + struct uint32s_bucket *bucket; + uint32s_insert_result_t result; + size_t size_bytes = index->object_size * sizeof(uint32_t); + uint32_t hash = uint32s_hash_bytes((uint8_t *)object, size_bytes); + uint32_t idx; + uint32_t *copy; + uint32_t additions_len = RBUF_LEN(index->additions); + result.index = 0; + result.is_new = false; + if(RHMAP_HAS(index->index, hash)) + { + bucket = RHMAP_PTR(index->index, hash); + if(uint32s_bucket_get(index, bucket, object, size_bytes, &result.index)) + { + if (index->objects[result.index] == NULL) + { + RARCH_LOG("[STATESTREAM] accessed collected index %d\n",result.index); + } + else + { + result.is_new = false; + index->counts[result.index]++; + return result; + } + } + idx = RBUF_LEN(index->objects); + copy = malloc(size_bytes); + memcpy(copy, object, size_bytes); + RBUF_PUSH(index->objects, copy); + RBUF_PUSH(index->counts, 1); + RBUF_PUSH(index->hashes, hash); + result.index = idx; + result.is_new = true; + uint32s_bucket_expand(bucket, idx); + } + else + { + struct uint32s_bucket new_bucket; + idx = RBUF_LEN(index->objects); + copy = malloc(size_bytes); + memcpy(copy, object, size_bytes); + RBUF_PUSH(index->objects, copy); + RBUF_PUSH(index->counts, 1); + RBUF_PUSH(index->hashes, hash); + new_bucket.len = 1; + new_bucket.contents.idxs[0] = idx; + new_bucket.contents.idxs[1] = 0; + new_bucket.contents.idxs[2] = 0; + RHMAP_SET(index->index, hash, new_bucket); + result.index = idx; + result.is_new = true; + } + if(additions_len == 0 || index->additions[additions_len-1].frame_counter < frame) + { + struct uint32s_frame_addition addition; + addition.frame_counter = frame; + addition.first_index = result.index; + RBUF_PUSH(index->additions, addition); + } + return result; +} + +bool uint32s_index_insert_exact(uint32s_index_t *index, uint32_t idx, uint32_t *object, uint64_t frame) +{ + struct uint32s_bucket *bucket; + uint32_t hash; + size_t size_bytes; + uint32_t additions_len; + if (idx != RBUF_LEN(index->objects)) + return false; + size_bytes = index->object_size * sizeof(uint32_t); + hash = uint32s_hash_bytes((uint8_t *)object, size_bytes); + additions_len = RBUF_LEN(index->additions); + if (RHMAP_HAS(index->index, hash)) + { + uint32_t idx = 0; + bucket = RHMAP_PTR(index->index, hash); + uint32s_bucket_expand(bucket, idx); + } + else + { + struct uint32s_bucket new_bucket; + new_bucket.len = 1; + new_bucket.contents.idxs[0] = idx; + new_bucket.contents.idxs[1] = 0; + new_bucket.contents.idxs[2] = 0; + RHMAP_SET(index->index, hash, new_bucket); + } + /* RARCH_LOG("[STATESTREAM] insert index %d\n",idx); */ + RBUF_PUSH(index->objects, object); + RBUF_PUSH(index->counts, 1); + RBUF_PUSH(index->hashes, hash); + if (additions_len == 0 || + index->additions[additions_len-1].frame_counter < frame) + { + struct uint32s_frame_addition addition; + addition.frame_counter = frame; + addition.first_index = idx; + RBUF_PUSH(index->additions, addition); + } + return true; +} + +void uint32s_index_commit(uint32s_index_t *index) +{ + uint32_t i, interval=index->commit_interval,threshold=index->commit_threshold; + struct uint32s_frame_addition prev,cur; + uint32_t additions_len = RBUF_LEN(index->additions), limit; + if (additions_len < interval || interval == 0) return; + prev = index->additions[additions_len-interval]; + cur = index->additions[additions_len-(interval-1)]; + limit = cur.first_index; + for (i = prev.first_index; i < limit; i++) + { + struct uint32s_bucket *bucket; + if (index->counts[i] >= threshold) continue; + free(index->objects[i]); + index->objects[i] = NULL; + bucket = RHMAP_PTR(index->index, index->hashes[i]); + uint32s_bucket_remove(bucket, i); + if (bucket->len == 0) + { + uint32s_bucket_free(bucket); + if (!RHMAP_DEL(index->index, index->hashes[i])) + RARCH_ERR("[STATESTREAM] Trying to remove absent hash %x\n",index->hashes[i]); + } + } +} + +void uint32s_index_bump_count(uint32s_index_t *index, uint32_t which) +{ + if (which >= RBUF_LEN(index->counts)) + return; + index->counts[which]++; +} + +uint32_t *uint32s_index_get(uint32s_index_t *index, uint32_t which) +{ + if (which >= RBUF_LEN(index->objects)) + return NULL; + if (!index->objects[which]) + { + RARCH_LOG("[STATESTREAM] accessed garbage collected block %d\n", which); + return NULL; + } + return index->objects[which]; +} + +void uint32s_index_pop(uint32s_index_t *index) +{ + uint32_t idx = RBUF_LEN(index->objects)-1; + uint32_t *object = RBUF_POP(index->objects); + size_t size_bytes = index->object_size * sizeof(uint32_t); + uint32_t hash = index->hashes[idx]; + struct uint32s_bucket *bucket = RHMAP_PTR(index->index, hash); + RBUF_RESIZE(index->counts, idx); + RBUF_RESIZE(index->hashes, idx); + uint32s_bucket_remove(bucket, idx); + if (bucket->len == 0) + { + uint32s_bucket_free(bucket); + if (!RHMAP_DEL(index->index, hash)) + RARCH_ERR("[STATESTREAM] Trying to remove absent hash %x\n",hash); + } +} + +/* goes backwards from end of additions */ +void uint32s_index_remove_after(uint32s_index_t *index, uint64_t frame) +{ + int i; + for(i = RBUF_LEN(index->additions)-1; i >= 0; i--) + { + struct uint32s_frame_addition add = index->additions[i]; + if(add.frame_counter <= frame) + break; + while(add.first_index < RBUF_LEN(index->objects)) + uint32s_index_pop(index); + } + RBUF_RESIZE(index->additions, i+1); +} + +/* removes all data from index */ +void uint32s_index_clear(uint32s_index_t *index) +{ + size_t i, cap; + uint32_t *zeros = index->objects[0]; + for(i = 0, cap = RHMAP_CAP(index->index); i != cap; i++) + if(RHMAP_KEY(index->index, i)) + uint32s_bucket_free(&index->index[i]); + RHMAP_CLEAR(index->index); + /* don't dealloc all-zeros pattern */ + for(i = 1; i < RBUF_LEN(index->objects); i++) + free(index->objects[i]); + RBUF_CLEAR(index->objects); + RBUF_CLEAR(index->counts); + RBUF_CLEAR(index->hashes); + uint32s_index_insert_exact(index, 0, zeros, 0); + /* wipe additions */ + RBUF_CLEAR(index->additions); +} + +void uint32s_index_free(uint32s_index_t *index) +{ + size_t i, cap; + for(i = 0, cap = RHMAP_CAP(index->index); i != cap; i++) + if(RHMAP_KEY(index->index, i)) + uint32s_bucket_free(&index->index[i]); + RHMAP_FREE(index->index); + for(i = 0; i < RBUF_LEN(index->objects); i++) + free(index->objects[i]); + RBUF_FREE(index->objects); + RBUF_FREE(index->counts); + RBUF_FREE(index->hashes); + RBUF_FREE(index->additions); + free(index); +} + +uint32_t uint32s_index_count(uint32s_index_t *index) +{ + if (!index || !index->objects) + return 0; + return RBUF_LEN(index->objects); +} + +#if DEBUG +#define BIN_COUNT 1000 +uint32_t bins[BIN_COUNT]; +void uint32s_index_print_count_data(uint32s_index_t *index) +{ + /* TODO: don't count or differently count NULL objects entries */ + uint32_t max=1; + uint32_t i; + for(i = 0; i < BIN_COUNT; i++) + bins[i] = 0; + for(i = 1; i < RBUF_LEN(index->counts); i++) + { + max = MAX(max, index->counts[i]); + } + max = max + 1; + for(i = 1; i < RBUF_LEN(index->counts); i++) + bins[(int)((((float)index->counts[i]) / (float)max)*BIN_COUNT)]++; + for(i = 0; i < BIN_COUNT; i++) + { + uint32_t bin_start = MAX(1,(i*(float)max/(float)BIN_COUNT)); + uint32_t bin_end = ((i+1)*(float)max/(float)BIN_COUNT); + if (bins[i] == 0) continue; + RARCH_DBG("%d--%d: %d\n", bin_start, bin_end, bins[i]); + } +} +#endif +#endif diff --git a/input/bsv/uint32s_index.h b/input/bsv/uint32s_index.h new file mode 100644 index 0000000000..40b1f79fef --- /dev/null +++ b/input/bsv/uint32s_index.h @@ -0,0 +1,70 @@ +#ifndef __UINT32S_INDEX__H +#define __UINT32S_INDEX__H +#ifdef HAVE_STATESTREAM +#include +#include +#include +#include +#include +#include + +struct uint32s_bucket +{ + uint32_t len; /* if < 4, contents is idxs. */ + union { + uint32_t idxs[3]; + struct { + uint32_t cap; + uint32_t *idxs; + } vec; + } contents; +}; +struct uint32s_frame_addition { + uint64_t frame_counter; + uint32_t first_index; /* lowest index added on this frame */ +}; +struct uint32s_index +{ + size_t object_size; /* measured in ints */ + struct uint32s_bucket *index; /* an rhmap of buckets for value->index lookup */ + uint32_t **objects; /* an rbuf of the actual buffers */ + uint32_t *counts; /* an rbuf of the times each object was used */ + uint32_t *hashes; /* an rbuf of each object's hash code */ + struct uint32s_frame_addition *additions; /* an rbuf of addition info */ + uint8_t commit_interval, commit_threshold; +}; +typedef struct uint32s_index uint32s_index_t; + +struct uint32s_insert_result +{ + uint32_t index; + bool is_new; +}; +typedef struct uint32s_insert_result uint32s_insert_result_t; + +RETRO_BEGIN_DECLS + +uint32s_index_t *uint32s_index_new(size_t object_size, uint8_t commit_interval, uint8_t commit_threshold); +/* Does not take ownership of object */ +uint32s_insert_result_t uint32s_index_insert(uint32s_index_t *index, uint32_t *object, uint64_t frame); +/* Does take ownership, requires idx is the exact next index and object not in index */ +bool uint32s_index_insert_exact(uint32s_index_t *index, uint32_t idx, uint32_t *object, uint64_t frame); +/* Does not grant ownership of return value */ +uint32_t *uint32s_index_get(uint32s_index_t *index, uint32_t which); +/* Just bump the count, don't try to get the results back */ +void uint32s_index_bump_count(uint32s_index_t *index, uint32_t which); +/* Call once the superblocks and blocks are all identified; transient blocks that have not been used this frame will be dropped. */ +void uint32s_index_commit(uint32s_index_t *index); +void uint32s_index_free(uint32s_index_t *index); + +/* goes backwards from end of additions */ +void uint32s_index_remove_after(uint32s_index_t *index, uint64_t frame); +/* removes all data from index */ +void uint32s_index_clear(uint32s_index_t *index); +uint32_t uint32s_index_count(uint32s_index_t *index); +#if DEBUG +void uint32s_index_print_count_data(uint32s_index_t *index); +#endif +RETRO_END_DECLS +#endif +#endif /* __UINT32S_INDEX__H */ diff --git a/input/input_driver.c b/input/input_driver.c index a74da50bde..a82cb0a21b 100644 --- a/input/input_driver.c +++ b/input/input_driver.c @@ -34,6 +34,10 @@ #include "input_osk.h" #include "input_types.h" +#ifdef HAVE_BSV_MOVIE +#include "bsv/bsvmovie.h" +#endif + #ifdef HAVE_CHEEVOS #include "../cheevos/cheevos.h" #endif @@ -58,15 +62,6 @@ #include "../paths.h" #include "../performance_counters.h" #include "../retroarch.h" -#ifdef HAVE_BSV_MOVIE -#include "../tasks/task_content.h" -#endif -#if defined(HAVE_ZLIB) && defined(HAVE_BSV_MOVIE) -#include -#endif -#if defined(HAVE_ZSTD) && defined(HAVE_BSV_MOVIE) -#include -#endif #include "../tasks/tasks_internal.h" #include "../verbosity.h" @@ -5975,880 +5970,6 @@ static void input_keys_pressed( input_st->input_hotkey_block_counter = 0; } -#ifdef HAVE_BSV_MOVIE -/* Forward declaration */ -void bsv_movie_free(bsv_movie_t*); - -void bsv_movie_enqueue(input_driver_state_t *input_st, - bsv_movie_t * state, enum bsv_flags flags) -{ - if (input_st->bsv_movie_state_next_handle) - bsv_movie_free(input_st->bsv_movie_state_next_handle); - input_st->bsv_movie_state_next_handle = state; - input_st->bsv_movie_state.flags = flags; -} - -void bsv_movie_deinit(input_driver_state_t *input_st) -{ - if (input_st->bsv_movie_state_handle) - bsv_movie_free(input_st->bsv_movie_state_handle); - input_st->bsv_movie_state_handle = NULL; -} - -void bsv_movie_deinit_full(input_driver_state_t *input_st) -{ - bsv_movie_deinit(input_st); - if (input_st->bsv_movie_state_next_handle) - bsv_movie_free(input_st->bsv_movie_state_next_handle); - input_st->bsv_movie_state_next_handle = NULL; -} - -void bsv_movie_frame_rewind(void) -{ - input_driver_state_t *input_st = &input_driver_st; - bsv_movie_t *handle = input_st->bsv_movie_state_handle; - bool recording = (input_st->bsv_movie_state.flags - & BSV_FLAG_MOVIE_RECORDING) ? true : false; - - if (!handle) - return; - - handle->did_rewind = true; - - if ( ( (handle->frame_counter & handle->frame_mask) <= 1) - && (handle->frame_pos[0] == handle->min_file_pos)) - { - /* If we're at the beginning... */ - handle->frame_counter = 0; - intfstream_seek(handle->file, (int)handle->min_file_pos, SEEK_SET); - if (recording) - intfstream_truncate(handle->file, (int)handle->min_file_pos); - else - bsv_movie_read_next_events(handle); - } - else - { - /* First time rewind is performed, the old frame is simply replayed. - * However, playing back that frame caused us to read data, and push - * data to the ring buffer. - * - * Successively rewinding frames, we need to rewind past the read data, - * plus another. */ - uint8_t delta = handle->first_rewind ? 1 : 2; - if (handle->frame_counter >= delta) - handle->frame_counter -= delta; - else - handle->frame_counter = 0; - intfstream_seek(handle->file, (int)handle->frame_pos[handle->frame_counter & handle->frame_mask], SEEK_SET); - if (recording) - intfstream_truncate(handle->file, (int)handle->frame_pos[handle->frame_counter & handle->frame_mask]); - else - bsv_movie_read_next_events(handle); - } - - if (intfstream_tell(handle->file) <= (long)handle->min_file_pos) - { - /* We rewound past the beginning. */ - - if (handle->playback) - { - intfstream_seek(handle->file, (int)handle->min_file_pos, SEEK_SET); - bsv_movie_read_next_events(handle); - } - else - { - retro_ctx_serialize_info_t serial_info; - - /* If recording, we simply reset - * the starting point. Nice and easy. */ - - intfstream_seek(handle->file, 6 * sizeof(uint32_t), SEEK_SET); - intfstream_truncate(handle->file, 6 * sizeof(uint32_t)); - - serial_info.data = handle->state; - serial_info.size = handle->state_size; - - core_serialize(&serial_info); - - intfstream_write(handle->file, handle->state, handle->state_size); - } - } -} - -void bsv_movie_handle_push_key_event(bsv_movie_t *movie, - uint8_t down, uint16_t mod, uint32_t code, uint32_t character) -{ - bsv_key_data_t data; - data.down = down; - data._padding = 0; - data.mod = swap_if_big16(mod); - data.code = swap_if_big32(code); - data.character = swap_if_big32(character); - movie->key_events[movie->key_event_count] = data; - movie->key_event_count++; -} - -void bsv_movie_handle_push_input_event(bsv_movie_t *movie, - uint8_t port, uint8_t dev, uint8_t idx, uint16_t id, int16_t val) -{ - bsv_input_data_t data; - data.port = port; - data.device = dev; - data.idx = idx; - data._padding = 0; - data.id = swap_if_big16(id); - data.value = swap_if_big16(val); - movie->input_events[movie->input_event_count] = data; - movie->input_event_count++; -} - -bool bsv_movie_handle_read_input_event(bsv_movie_t *movie, - uint8_t port, uint8_t dev, uint8_t idx, uint16_t id, int16_t* val) -{ - int i; - /* if movie is old, just read two bytes and hope for the best */ - if (movie->version == 0) - { - int64_t read = intfstream_read(movie->file, val, 2); - *val = swap_if_big16(*val); - return (read == 2); - } - for (i = 0; i < movie->input_event_count; i++) - { - bsv_input_data_t evt = movie->input_events[i]; - if ( (evt.port == port) - && (evt.device == dev) - && (evt.idx == idx) - && (evt.id == id)) - { - *val = swap_if_big16(evt.value); - return true; - } - } - return false; -} - -void bsv_movie_finish_rewind(input_driver_state_t *input_st) -{ - bsv_movie_t *handle = input_st->bsv_movie_state_handle; - if (!handle) - return; - handle->frame_counter += 1; - handle->first_rewind = !handle->did_rewind; - handle->did_rewind = false; -} - -bool bsv_movie_load_checkpoint(bsv_movie_t *handle, uint8_t compression, uint8_t encoding) -{ -#ifdef HAVE_ZSTD - size_t uncompressed_size_big; -#endif - retro_ctx_serialize_info_t serial_info; - input_driver_state_t *input_st = input_state_get_ptr(); - uint32_t compressed_encoded_size, encoded_size, size; - uint8_t *compressed_data = NULL; - uint8_t *encoded_data = NULL; - uint8_t *state = NULL; - bool ret = true; - - if (intfstream_read(handle->file, &(size), - sizeof(uint32_t)) != sizeof(uint32_t)) - { - RARCH_ERR("[Replay] Replay truncated before uncompressed unencoded size\n"); - ret = false; - goto exit; - } - if (intfstream_read(handle->file, &(encoded_size), - sizeof(uint32_t)) != sizeof(uint32_t)) - { - RARCH_ERR("[Replay] Replay truncated before uncompressed encoded size\n"); - ret = false; - goto exit; - } - if (intfstream_read(handle->file, &(compressed_encoded_size), - sizeof(uint32_t)) != sizeof(uint32_t)) - { - RARCH_ERR("[Replay] Replay truncated before compressed encoded size\n"); - ret = false; - goto exit; - } - size = swap_if_big32(size); - encoded_size = swap_if_big32(encoded_size); - compressed_encoded_size = swap_if_big32(compressed_encoded_size); - compressed_data = (uint8_t*)malloc(compressed_encoded_size); - if (intfstream_read(handle->file, compressed_data, compressed_encoded_size) != (int64_t)compressed_encoded_size) - { - RARCH_ERR("[Replay] Truncated checkpoint, terminating movie\n"); - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - ret = false; - goto exit; - } - switch (compression) - { - case REPLAY_CHECKPOINT2_COMPRESSION_NONE: - encoded_data = compressed_data; - compressed_data = NULL; - break; -#ifdef HAVE_ZLIB - case REPLAY_CHECKPOINT2_COMPRESSION_ZLIB: - { -#ifdef EMSCRIPTEN - uLongf uncompressed_size_zlib = encoded_size; -#else - uint32_t uncompressed_size_zlib = encoded_size; -#endif - encoded_data = (uint8_t*)calloc(encoded_size, sizeof(uint8_t)); - if (uncompress(encoded_data, &uncompressed_size_zlib, compressed_data, compressed_encoded_size) != Z_OK) - { - ret = false; - goto exit; - } - break; - } -#endif -#ifdef HAVE_ZSTD - case REPLAY_CHECKPOINT2_COMPRESSION_ZSTD: - /* TODO: figure out how to support in-place decompression to - avoid allocating a second buffer; would need to allocate - the compressed_data buffer to be decompressed size + - margin. but, how could the margin be known without - calling the function that takes the compressed frames as - an input? */ - encoded_data = (uint8_t*)calloc(encoded_size, sizeof(uint8_t)); - uncompressed_size_big = ZSTD_decompress(encoded_data, encoded_size, compressed_data, compressed_encoded_size); - if (ZSTD_isError(uncompressed_size_big)) - { - ret = false; - goto exit; - } - break; -#endif - default: - RARCH_WARN("[Replay] Unrecognized compression scheme %d\n", compression); - ret = false; - goto exit; - } - switch (encoding) - { - case REPLAY_CHECKPOINT2_ENCODING_RAW: - size = encoded_size; - state = encoded_data; - encoded_data = NULL; - break; - default: - RARCH_WARN("[Replay] Unrecognized encoding scheme %d\n", encoding); - ret = false; - goto exit; - } - serial_info.data_const = state; - serial_info.size = size; - if (!core_unserialize(&serial_info)) - { - ret = false; - goto exit; - } - exit: - if (compressed_data) - free(compressed_data); - if (encoded_data) - free(encoded_data); - if (state) - free(state); - return ret; -} - -bool bsv_movie_write_checkpoint(bsv_movie_t *handle, uint8_t compression, uint8_t encoding, retro_ctx_serialize_info_t serial_info) -{ - bool ret = true; - uint32_t encoded_size, compressed_encoded_size, size_; - uint8_t *encoded_data = NULL; - uint8_t *compressed_encoded_data = NULL; - bool owns_encoded = false; - bool owns_compressed_encoded = false; - switch (encoding) - { - case REPLAY_CHECKPOINT2_ENCODING_RAW: - encoded_size = serial_info.size; - encoded_data = (uint8_t*)serial_info.data; - break; - default: - RARCH_ERR("[Replay] Unrecognized encoding scheme %d\n", encoding); - ret = false; - goto exit; - } - switch (compression) - { - case REPLAY_CHECKPOINT2_COMPRESSION_NONE: - compressed_encoded_size = encoded_size; - compressed_encoded_data = (uint8_t*)encoded_data; - break; -#ifdef HAVE_ZLIB - case REPLAY_CHECKPOINT2_COMPRESSION_ZLIB: - { - uLongf zlib_compressed_encoded_size = compressBound(encoded_size); - compressed_encoded_data = (uint8_t*)calloc(zlib_compressed_encoded_size, sizeof(uint8_t)); - owns_compressed_encoded = true; - if (compress2(compressed_encoded_data, &zlib_compressed_encoded_size, encoded_data, encoded_size, 6) != Z_OK) - { - ret = false; - goto exit; - } - compressed_encoded_size = zlib_compressed_encoded_size; - break; - } -#endif -#ifdef HAVE_ZSTD - case REPLAY_CHECKPOINT2_COMPRESSION_ZSTD: - { - size_t compressed_encoded_size_big = ZSTD_compressBound(encoded_size); - compressed_encoded_data = (uint8_t*)calloc(compressed_encoded_size_big, sizeof(uint8_t)); - owns_compressed_encoded = true; - compressed_encoded_size_big = ZSTD_compress(compressed_encoded_data, compressed_encoded_size_big, encoded_data, encoded_size, 9); - if (ZSTD_isError(compressed_encoded_size_big)) - { - ret = false; - goto exit; - } - compressed_encoded_size = compressed_encoded_size_big; - break; - } -#endif - default: - RARCH_WARN("[Replay] Unrecognized compression scheme %d\n", compression); - ret = false; - goto exit; - } - /* uncompressed, unencoded size */ - size_ = swap_if_big32(serial_info.size); - intfstream_write(handle->file, &size_, sizeof(uint32_t)); - /* uncompressed, encoded size */ - size_ = swap_if_big32(encoded_size); - intfstream_write(handle->file, &size_, sizeof(uint32_t)); - /* compressed, encoded size */ - size_ = swap_if_big32(compressed_encoded_size); - intfstream_write(handle->file, &size_, sizeof(uint32_t)); - /* data */ - intfstream_write(handle->file, compressed_encoded_data, compressed_encoded_size); - exit: - if (encoded_data && owns_encoded) - free(encoded_data); - if (compressed_encoded_data && owns_compressed_encoded) - free(compressed_encoded_data); - return ret; -} - -void bsv_movie_read_next_events(bsv_movie_t *handle) -{ - input_driver_state_t *input_st = input_state_get_ptr(); - if (intfstream_read(handle->file, &(handle->key_event_count), 1) == 1) - { - int i; - for (i = 0; i < handle->key_event_count; i++) - { - if (intfstream_read(handle->file, &(handle->key_events[i]), - sizeof(bsv_key_data_t)) != sizeof(bsv_key_data_t)) - { - /* Unnatural EOF */ - RARCH_ERR("[Replay] Keyboard replay ran out of keyboard inputs too early.\n"); - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - return; - } - } - } - else - { - RARCH_LOG("[Replay] EOF after buttons.\n"); - /* Natural(?) EOF */ - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - return; - } - if (handle->version > 0) - { - if (intfstream_read(handle->file, &(handle->input_event_count), 2) == 2) - { - int i; - handle->input_event_count = swap_if_big16(handle->input_event_count); - for (i = 0; i < handle->input_event_count; i++) - { - if (intfstream_read(handle->file, &(handle->input_events[i]), - sizeof(bsv_input_data_t)) != sizeof(bsv_input_data_t)) - { - /* Unnatural EOF */ - RARCH_ERR("[Replay] Input replay ran out of inputs too early.\n"); - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - return; - } - } - } - else - { - RARCH_LOG("[Replay] EOF after inputs.\n"); - /* Natural(?) EOF */ - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - return; - } - } - - { - uint8_t next_frame_type=REPLAY_TOKEN_INVALID; - if (intfstream_read(handle->file, (uint8_t *)(&next_frame_type), - sizeof(uint8_t)) != sizeof(uint8_t)) - { - /* Unnatural EOF */ - RARCH_ERR("[Replay] Replay ran out of frames.\n"); - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - return; - } - else if (next_frame_type == REPLAY_TOKEN_CHECKPOINT_FRAME) - { - size_t _len; - uint8_t *state; - retro_ctx_serialize_info_t serial_info; - if (intfstream_read(handle->file, &_len, sizeof(uint64_t)) != sizeof(uint64_t)) - { - /* Unnatural EOF */ - RARCH_ERR("[Replay] Replay truncated before reading size.\n"); - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - return; - } - _len = swap_if_big64(_len); - state = (uint8_t*)calloc(_len, sizeof(uint8_t)); - if (intfstream_read(handle->file, state, _len) != (int64_t)_len) - { - /* Unnatural EOF */ - RARCH_ERR("[Replay] Replay checkpoint truncated.\n"); - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - free(state); - return; - } - serial_info.data_const = state; - serial_info.size = _len; - if (!core_unserialize(&serial_info)) - { - RARCH_ERR("[Replay] Failed to load movie checkpoint, failing\n"); - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - } - free(state); - } - else if (next_frame_type == REPLAY_TOKEN_CHECKPOINT2_FRAME) - { - uint8_t compression, encoding; - if ( intfstream_read(handle->file, &(compression), sizeof(uint8_t)) != sizeof(uint8_t) - || intfstream_read(handle->file, &(encoding), sizeof(uint8_t)) != sizeof(uint8_t)) - { - /* Unexpected EOF */ - RARCH_ERR("[Replay] Replay checkpoint truncated.\n"); - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - return; - } - if (!bsv_movie_load_checkpoint(handle, compression, encoding)) - RARCH_WARN("[Replay] Failed to load movie checkpoint\n"); - } - } -} - -void bsv_movie_next_frame(input_driver_state_t *input_st) -{ - unsigned checkpoint_interval = config_get_ptr()->uints.replay_checkpoint_interval; - /* if bsv_movie_state_next_handle is not null, deinit and set - bsv_movie_state_handle to bsv_movie_state_next_handle and clear - next_handle */ - bsv_movie_t *handle = input_st->bsv_movie_state_handle; - if (input_st->bsv_movie_state_next_handle) - { - if (handle) - bsv_movie_deinit(input_st); - handle = input_st->bsv_movie_state_next_handle; - input_st->bsv_movie_state_handle = handle; - input_st->bsv_movie_state_next_handle = NULL; - } - - if (!handle) - return; -#ifdef HAVE_REWIND - if (state_manager_frame_is_reversed()) - return; -#endif - - if (input_st->bsv_movie_state.flags & BSV_FLAG_MOVIE_RECORDING) - { - int i; - uint16_t evt_count = swap_if_big16(handle->input_event_count); - /* write key events, frame is over */ - intfstream_write(handle->file, &(handle->key_event_count), 1); - for (i = 0; i < handle->key_event_count; i++) - intfstream_write(handle->file, &(handle->key_events[i]), - sizeof(bsv_key_data_t)); - /* Zero out key events when playing back or recording */ - handle->key_event_count = 0; - /* write input events, frame is over */ - intfstream_write(handle->file, &evt_count, 2); - for (i = 0; i < handle->input_event_count; i++) - intfstream_write(handle->file, &(handle->input_events[i]), - sizeof(bsv_input_data_t)); - /* Zero out input events when playing back or recording */ - handle->input_event_count = 0; - - /* Maybe record checkpoint */ - if ( (checkpoint_interval != 0) - && (handle->frame_counter > 0) - && (handle->frame_counter % (checkpoint_interval*60) == 0)) - { - uint8_t frame_tok = REPLAY_TOKEN_CHECKPOINT2_FRAME; - retro_ctx_serialize_info_t serial_info; -#if defined(HAVE_ZSTD) - uint8_t compression = REPLAY_CHECKPOINT2_COMPRESSION_ZSTD; -#elif defined(HAVE_ZLIB) - uint8_t compression = REPLAY_CHECKPOINT2_COMPRESSION_ZLIB; -#else - uint8_t compression = REPLAY_CHECKPOINT2_COMPRESSION_NONE; -#endif - uint8_t encoding = REPLAY_CHECKPOINT2_ENCODING_RAW; - size_t _len = core_serialize_size(); - uint8_t *st = (uint8_t*)malloc(_len); - serial_info.data = st; - serial_info.size = _len; - core_serialize(&serial_info); - /* "next frame is a checkpoint" */ - intfstream_write(handle->file, (uint8_t *)(&frame_tok), sizeof(uint8_t)); - /* compression and encoding schemes */ - intfstream_write(handle->file, (uint8_t *)(&compression), sizeof(uint8_t)); - intfstream_write(handle->file, (uint8_t *)(&encoding), sizeof(uint8_t)); - if (!bsv_movie_write_checkpoint(handle, compression, encoding, serial_info)) - { - RARCH_ERR("[Replay] failed to write checkpoint, exiting record\n"); - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - } - free(st); - } - else - { - uint8_t frame_tok = REPLAY_TOKEN_REGULAR_FRAME; - /* write "next frame is not a checkpoint" */ - intfstream_write(handle->file, (uint8_t *)(&frame_tok), sizeof(uint8_t)); - } - } - - if (input_st->bsv_movie_state.flags & BSV_FLAG_MOVIE_PLAYBACK) - bsv_movie_read_next_events(handle); - handle->frame_pos[handle->frame_counter & handle->frame_mask] = intfstream_tell(handle->file); -} - -size_t replay_get_serialize_size(void) -{ - input_driver_state_t *input_st = &input_driver_st; - if (input_st->bsv_movie_state.flags & (BSV_FLAG_MOVIE_RECORDING | BSV_FLAG_MOVIE_PLAYBACK)) - return sizeof(uint32_t)+intfstream_tell(input_st->bsv_movie_state_handle->file); - return 0; -} - -bool replay_get_serialized_data(void* buffer) -{ - input_driver_state_t *input_st = &input_driver_st; - bsv_movie_t *handle = input_st->bsv_movie_state_handle; - - if (input_st->bsv_movie_state.flags & (BSV_FLAG_MOVIE_RECORDING | BSV_FLAG_MOVIE_PLAYBACK)) - { - int32_t file_end = (uint32_t)intfstream_tell(handle->file); - int64_t read_amt = 0; - int32_t file_end_ = swap_if_big32(file_end); - uint8_t *buf; - ((uint32_t *)buffer)[0] = file_end_; - buf = ((uint8_t *)buffer) + sizeof(uint32_t); - intfstream_rewind(handle->file); - read_amt = intfstream_read(handle->file, buf, file_end); - if (read_amt != file_end) - RARCH_ERR("[Replay] Failed to write correct number of replay bytes into state file: %d / %d.\n", - read_amt, file_end); - } - return true; -} - -static bool replay_check_same_timeline(bsv_movie_t *movie, - uint8_t *other_movie, int64_t other_len) -{ - uint64_t size1, size2; - uint16_t btncount1, btncount2; - int64_t check_limit = MIN(other_len, intfstream_tell(movie->file)); - intfstream_t *check_stream = intfstream_open_memory(other_movie, RETRO_VFS_FILE_ACCESS_READ, RETRO_VFS_FILE_ACCESS_HINT_NONE, other_len); - bool ret = true; - int64_t check_cap = MAX(128 << 10, MAX(128*sizeof(bsv_key_data_t), 512*sizeof(bsv_input_data_t))); - uint8_t *buf1 = (uint8_t*)calloc(check_cap,1); - uint8_t *buf2 = (uint8_t*)calloc(check_cap,1); - size_t movie_pos = intfstream_tell(movie->file); - uint8_t frametok1 = 0; - uint8_t frametok2 = 0; - uint8_t keycount1 = 0; - uint8_t keycount2 = 0; - - intfstream_rewind(movie->file); - intfstream_read(movie->file, buf1, 6*sizeof(uint32_t)); - intfstream_read(check_stream, buf2, 6*sizeof(uint32_t)); - - if (memcmp(buf1, buf2, 6*sizeof(uint32_t)) != 0) - { - RARCH_ERR("[Replay] Headers of two movies differ, not same timeline\n"); - ret = false; - goto exit; - } - - intfstream_seek(movie->file, movie->min_file_pos, SEEK_SET); - /* assumption: both headers have the same state size */ - intfstream_seek(check_stream, movie->min_file_pos, SEEK_SET); - if (movie->version == 0) - { - int64_t i; - /* no choice but to memcmp the whole stream against the other */ - for (i = 0; ret && i < check_limit; i += check_cap) - { - int64_t read_end = MIN(check_limit - i, check_cap); - int64_t read1 = intfstream_read(movie->file, buf1, read_end); - int64_t read2 = intfstream_read(check_stream, buf2, read_end); - if (read1 != read_end || read2 != read_end || memcmp(buf1, buf2, read_end) != 0) - { - RARCH_ERR("[Replay] One or the other replay checkpoint has different byte values\n"); - ret = false; - goto exit; - } - } - goto exit; - } - while (intfstream_tell(movie->file) < check_limit && intfstream_tell(check_stream) < check_limit) - { - if (intfstream_tell(movie->file) < 0 || intfstream_tell(check_stream) < 0) - { - RARCH_ERR("[Replay] One or the other replay checkpoint has ended prematurely\n"); - ret = false; - goto exit; - } - if ( intfstream_read(movie->file, &keycount1, 1) < 1 - || intfstream_read(check_stream, &keycount2, 1) < 1 - || keycount1 != keycount2) - { - RARCH_ERR("[Replay] Replay checkpoints disagree on key count, %d vs %d\n", keycount1, keycount2); - ret = false; - goto exit; - } - if ( - (uint64_t)intfstream_read(movie->file, buf1, keycount1*sizeof(bsv_key_data_t)) - < keycount1*sizeof(bsv_key_data_t) - || (uint64_t)intfstream_read(check_stream, buf2, keycount2*sizeof(bsv_key_data_t)) - < keycount2*sizeof(bsv_key_data_t) - || memcmp(buf1, buf2, keycount1*sizeof(bsv_key_data_t)) != 0) - { - RARCH_ERR("[Replay] Replay checkpoints disagree on key data\n"); - ret = false; - goto exit; - } - if ( intfstream_read(movie->file, &btncount1, 2) < 2 - || intfstream_read(check_stream, &btncount2, 2) < 2 - || btncount1 != btncount2) - { - RARCH_ERR("[Replay] Replay checkpoints disagree on input count\n"); - ret = false; - goto exit; - } - btncount1 = swap_if_big16(btncount1); - btncount2 = swap_if_big16(btncount2); - if ( (uint64_t)intfstream_read(movie->file, buf1, btncount1*sizeof(bsv_input_data_t)) - < btncount1*sizeof(bsv_input_data_t) - || (uint64_t)intfstream_read(check_stream, buf2, btncount2*sizeof(bsv_input_data_t)) - < btncount2*sizeof(bsv_input_data_t) - || memcmp(buf1, buf2, btncount1*sizeof(bsv_input_data_t)) != 0) - { - RARCH_ERR("[Replay] Replay checkpoints disagree on input data\n"); - ret = false; - goto exit; - } - if ( intfstream_read(movie->file, &frametok1, 1) < 1 - || intfstream_read(check_stream, &frametok2, 1) < 1 - || frametok1 != frametok2) - { - RARCH_ERR("[Replay] Replay checkpoints disagree on frame token\n"); - ret = false; - goto exit; - } - switch (frametok1) - { - case REPLAY_TOKEN_INVALID: - RARCH_ERR("[Replay] Both replays are somehow invalid\n"); - ret = false; - goto exit; - case REPLAY_TOKEN_REGULAR_FRAME: - break; - case REPLAY_TOKEN_CHECKPOINT_FRAME: - if ( (uint64_t)intfstream_read(movie->file, &size1, sizeof(uint64_t)) < sizeof(uint64_t) - || (uint64_t)intfstream_read(check_stream, &size2, sizeof(uint64_t)) < sizeof(uint64_t) - || size1 != size2) - { - RARCH_ERR("[Replay] Replay checkpoints disagree on size or scheme\n"); - ret = false; - goto exit; - } - size1 = swap_if_big64(size1); - intfstream_seek(movie->file, size1, SEEK_CUR); - intfstream_seek(check_stream, size1, SEEK_CUR); - break; - case REPLAY_TOKEN_CHECKPOINT2_FRAME: - { - uint32_t cpsize1, cpsize2; - /* read cp2 header: - - one byte compression codec, one byte encoding scheme - - 4 byte uncompressed unencoded size, 4 byte uncompressed encoded size - - 4 byte compressed, encoded size - - the data will follow - */ - if (intfstream_read(movie->file, buf1, 2+sizeof(uint32_t)*3) != 2+sizeof(uint32_t)*3 || - intfstream_read(check_stream, buf2, 2+sizeof(uint32_t)*3) != 2+sizeof(uint32_t)*3 || - memcmp(buf1, buf2, 2+sizeof(uint32_t)*3) != 0 - ) - { - ret = false; - goto exit; - } - memcpy(&cpsize1, buf1+10, sizeof(uint32_t)); - memcpy(&cpsize2, buf2+10, sizeof(uint32_t)); - cpsize1 = swap_if_big32(cpsize1); - cpsize2 = swap_if_big32(cpsize2); - intfstream_seek(movie->file, cpsize1, SEEK_CUR); - intfstream_seek(check_stream, cpsize2, SEEK_CUR); - break; - } - default: - RARCH_ERR("[Replay] Unrecognized frame token in both replays\n"); - ret = false; - goto exit; - } - } - exit: - free(buf1); - free(buf2); - intfstream_close(check_stream); - intfstream_seek(movie->file, movie_pos, SEEK_SET); - return ret; -} - -bool replay_set_serialized_data(void* buf) -{ - uint8_t *buffer = (uint8_t*)buf; - input_driver_state_t *input_st = &input_driver_st; - bool playback = (input_st->bsv_movie_state.flags & BSV_FLAG_MOVIE_PLAYBACK) ? true : false; - bool recording = (input_st->bsv_movie_state.flags & BSV_FLAG_MOVIE_RECORDING) ? true : false; - bsv_movie_t *movie = input_st->bsv_movie_state_handle; - /* If there is no current replay, ignore this entirely. - TODO/FIXME: Later, consider loading up the replay - and allow the user to continue it? - Or would that be better done from the replay hotkeys? - */ - if (!(playback || recording)) - return true; - - if (!buffer) - { - if (recording) - { - const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_FAILED_INCOMPAT); - runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); - RARCH_ERR("[Replay] %s.\n", _msg); - return false; - } - - if (playback) - { - const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_HALT_INCOMPAT); - runloop_msg_queue_push(_msg, sizeof(_msg), 1, 180, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_WARNING); - RARCH_WARN("[Replay] %s.\n", _msg); - movie_stop(input_st); - } - } - else - { - /* TODO: should factor the next few lines away, magic numbers ahoy */ - uint32_t *header = (uint32_t *)(buffer + sizeof(uint32_t)); - int64_t *ident_spot = (int64_t *)(header + 4); - int64_t ident; - /* avoid unaligned 8-byte read */ - memcpy(&ident, ident_spot, sizeof(int64_t)); - ident = swap_if_big64(ident); - - if (ident == movie->identifier) /* is compatible? */ - { - int64_t _len = (int64_t)swap_if_big32(((uint32_t *)buffer)[0]); - int64_t handle_idx = intfstream_tell(movie->file); - bool same_timeline = replay_check_same_timeline(movie, (uint8_t *)header, _len); - /* If the state is part of this replay, go back to that state - and rewind/fast forward the replay. - - If the savestate movie is after the current replay - length we can replace the current replay data with it, - but if it's earlier we can rewind the replay to the - savestate movie time point. - - This can truncate the current replay if we're in recording mode. - */ - if (playback && _len > handle_idx) - { - const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_FAILED_FUTURE_STATE); - runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); - RARCH_ERR("[Replay] %s.\n", _msg); - return false; - } - else if (playback && !same_timeline) - { - const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_FAILED_WRONG_TIMELINE); - runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); - RARCH_ERR("[Replay] %s.\n", _msg); - return false; - } - else if (recording && (_len > handle_idx || !same_timeline)) - { - if (!same_timeline) - { - const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_OVERWRITING_REPLAY); - runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_WARNING); - RARCH_WARN("[Replay] %s.\n", _msg); - } - intfstream_rewind(movie->file); - intfstream_write(movie->file, buffer+sizeof(int32_t), _len); - } - else - { - intfstream_seek(movie->file, _len, SEEK_SET); - if (recording) - intfstream_truncate(movie->file, _len); - } - } - else - { - /* otherwise, if recording do not allow the load */ - if (recording) - { - const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_FAILED_INCOMPAT); - runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); - RARCH_ERR("[Replay] %s.\n", _msg); - return false; - } - /* if in playback, halt playback and go to that state normally */ - if (playback) - { - const char *_msg = msg_hash_to_str(MSG_REPLAY_LOAD_STATE_HALT_INCOMPAT); - runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_WARNING); - RARCH_WARN("[Replay] %s.\n", _msg); - movie_stop(input_st); - } - } - } - return true; -} -#endif - void input_driver_poll(void) { size_t i, j; @@ -7301,29 +6422,7 @@ void input_driver_poll(void) #endif #ifdef HAVE_BSV_MOVIE if (BSV_MOVIE_IS_PLAYBACK_ON()) - { - runloop_state_t *runloop_st = runloop_state_get_ptr(); - retro_keyboard_event_t *key_event = &runloop_st->key_event; - - if (*key_event && *key_event == runloop_st->frontend_key_event) - { - int i; - bsv_key_data_t k; - for (i = 0; i < input_st->bsv_movie_state_handle->key_event_count; i++) - { -#ifdef HAVE_CHEEVOS - rcheevos_pause_hardcore(); -#endif - k = input_st->bsv_movie_state_handle->key_events[i]; - input_keyboard_event(k.down, swap_if_big32(k.code), - swap_if_big32(k.character), swap_if_big16(k.mod), - RETRO_DEVICE_KEYBOARD); - } - /* Have to clear here so we don't double-apply key events */ - /* Zero out key events when playing back or recording */ - input_st->bsv_movie_state_handle->key_event_count = 0; - } - } + bsv_movie_poll(input_st); #endif } @@ -7335,22 +6434,8 @@ int16_t input_driver_state_wrapper(unsigned port, unsigned device, settings_t *settings = config_get_ptr(); int16_t result = 0; #ifdef HAVE_BSV_MOVIE - /* Load input from BSV record, if enabled */ if (BSV_MOVIE_IS_PLAYBACK_ON()) - { - int16_t bsv_result = 0; - bsv_movie_t *movie = input_st->bsv_movie_state_handle; - if (bsv_movie_handle_read_input_event( - movie, port, device, idx, id, &bsv_result)) - { -#ifdef HAVE_CHEEVOS - rcheevos_pause_hardcore(); -#endif - return bsv_result; - } - - input_st->bsv_movie_state.flags |= BSV_FLAG_MOVIE_END; - } + return bsv_movie_read_state(input_st, port, device, idx, id); #endif /* Read input state */ @@ -7364,12 +6449,11 @@ int16_t input_driver_state_wrapper(unsigned port, unsigned device, input_st->analog_requested[port] = true; #ifdef HAVE_BSV_MOVIE - /* Save input to BSV record, if enabled */ if (BSV_MOVIE_IS_RECORDING()) #ifdef HAVE_REWIND - if (!state_manager_frame_is_reversed()) + if (!state_manager_frame_is_reversed()) #endif - bsv_movie_handle_push_input_event( + bsv_movie_push_input_event( input_st->bsv_movie_state_handle, port, device, @@ -8184,12 +7268,11 @@ void input_keyboard_event(bool down, unsigned code, if (*key_event == runloop_st->frontend_key_event) { #ifdef HAVE_BSV_MOVIE - /* Save input to BSV record, if recording */ if (BSV_MOVIE_IS_RECORDING()) #ifdef HAVE_REWIND if (!state_manager_frame_is_reversed()) #endif - bsv_movie_handle_push_key_event( + bsv_movie_push_key_event( input_st->bsv_movie_state_handle, down, mod, code, character); #endif diff --git a/input/input_driver.h b/input/input_driver.h index 6fb2f4e62b..271f9af141 100644 --- a/input/input_driver.h +++ b/input/input_driver.h @@ -56,6 +56,10 @@ #include "../command.h" #endif +#ifdef HAVE_BSV_MOVIE +#include "bsv/uint32s_index.h" +#endif + #if defined(ANDROID) #define DEFAULT_MAX_PADS 8 #define ANDROID_KEYBOARD_PORT DEFAULT_MAX_PADS @@ -129,8 +133,12 @@ #define REPLAY_CHECKPOINT2_COMPRESSION_ZSTD 2 /* Which encoding to use. - RAW: Just raw checkpoint data, possibly compressed. */ + RAW: Just raw checkpoint data, possibly compressed. + STATESTREAM: Incremental, block-deduplicated encoding per + https://github.com/sumitshetye2/v86_savestreams +*/ #define REPLAY_CHECKPOINT2_ENCODING_RAW 0 +#define REPLAY_CHECKPOINT2_ENCODING_STATESTREAM 1 /** * Takes as input analog key identifiers and converts them to corresponding @@ -209,34 +217,32 @@ struct bsv_state /* These data are always little-endian. */ struct bsv_key_data { - uint8_t down; - uint8_t _padding; - uint16_t mod; - uint32_t code; - uint32_t character; + uint8_t down; + uint8_t _padding; + uint16_t mod; + uint32_t code; + uint32_t character; }; typedef struct bsv_key_data bsv_key_data_t; struct bsv_input_data { - uint8_t port; - uint8_t device; - uint8_t idx; - uint8_t _padding; - /* little-endian numbers */ - uint16_t id; - int16_t value; + uint8_t port; + uint8_t device; + uint8_t idx; + uint8_t _padding; + /* little-endian numbers */ + uint16_t id; + int16_t value; }; typedef struct bsv_input_data bsv_input_data_t; struct bsv_movie { intfstream_t *file; - uint8_t *state; int64_t identifier; uint32_t version; size_t min_file_pos; - size_t state_size; /* A ring buffer keeping track of positions * in the file for each frame. */ @@ -254,6 +260,21 @@ struct bsv_movie bool playback; bool first_rewind; bool did_rewind; + +#ifdef HAVE_STATESTREAM + /* Block index and superblock index for incremental checkpoints */ + uint32s_index_t *superblocks; + uint32s_index_t *blocks; + uint32_t *superblock_seq; + uint8_t commit_interval, commit_threshold; +#endif + + uint8_t checkpoint_compression, checkpoint_encoding; + + uint8_t *last_save, *cur_save; + size_t last_save_size, cur_save_size; + + bool cur_save_valid; }; typedef struct bsv_movie bsv_movie_t; @@ -1067,7 +1088,8 @@ void input_overlay_check_mouse_cursor(void); #ifdef HAVE_BSV_MOVIE void bsv_movie_frame_rewind(void); void bsv_movie_next_frame(input_driver_state_t *input_st); -void bsv_movie_read_next_events(bsv_movie_t*handle); +bool bsv_movie_read_next_events(bsv_movie_t *handle, bool skip_checkpoints); +bool bsv_movie_reset_recording(bsv_movie_t *handle); void bsv_movie_finish_rewind(input_driver_state_t *input_st); void bsv_movie_deinit(input_driver_state_t *input_st); void bsv_movie_deinit_full(input_driver_state_t *input_st); diff --git a/intl/msg_hash_lbl.h b/intl/msg_hash_lbl.h index f2fb36e03d..70bc1d5dc8 100644 --- a/intl/msg_hash_lbl.h +++ b/intl/msg_hash_lbl.h @@ -402,6 +402,10 @@ MSG_HASH( MENU_ENUM_LABEL_REPLAY_CHECKPOINT_INTERVAL, "replay_checkpoint_interval" ) +MSG_HASH( + MENU_ENUM_LABEL_REPLAY_CHECKPOINT_DESERIALIZE, + "replay_checkpoint_deserialize" + ) MSG_HASH( MENU_ENUM_LABEL_AUTO_OVERRIDES_ENABLE, "auto_overrides_enable" diff --git a/intl/msg_hash_us.c b/intl/msg_hash_us.c index df6c8064e2..9257b65c7f 100644 --- a/intl/msg_hash_us.c +++ b/intl/msg_hash_us.c @@ -310,6 +310,9 @@ int msg_hash_get_help_us_enum(enum msg_hash_enums msg, char *s, size_t len) case MENU_ENUM_LABEL_REPLAY_CHECKPOINT_INTERVAL: strlcpy(s, msg_hash_to_str(MENU_ENUM_LABEL_HELP_REPLAY_CHECKPOINT_INTERVAL), len); break; + case MENU_ENUM_LABEL_REPLAY_CHECKPOINT_DESERIALIZE: + strlcpy(s, msg_hash_to_str(MENU_ENUM_LABEL_HELP_REPLAY_CHECKPOINT_DESERIALIZE), len); + break; case MENU_ENUM_LABEL_VALUE_INPUT_ADC_TYPE: strlcpy(s, msg_hash_to_str(MENU_ENUM_LABEL_HELP_INPUT_ADC_TYPE), len); break; diff --git a/intl/msg_hash_us.h b/intl/msg_hash_us.h index 017ac21a18..6c44947053 100644 --- a/intl/msg_hash_us.h +++ b/intl/msg_hash_us.h @@ -4819,6 +4819,18 @@ MSG_HASH( MENU_ENUM_LABEL_HELP_REPLAY_CHECKPOINT_INTERVAL, "Autosaves the game state during replay recording at a regular interval. This is disabled by default unless set otherwise. The interval is measured in seconds. A value of 0 disables checkpoint recording." ) +MSG_HASH( + MENU_ENUM_SUBLABEL_REPLAY_CHECKPOINT_DESERIALIZE, + "Whether to deserialize checkpoints stored in replays during regular playback." + ) +MSG_HASH( + MENU_ENUM_LABEL_VALUE_REPLAY_CHECKPOINT_DESERIALIZE, + "Replay Checkpoint Deserialize" + ) +MSG_HASH( + MENU_ENUM_LABEL_HELP_REPLAY_CHECKPOINT_DESERIALIZE, + "Whether to deserialize checkpoints stored in replays during regular playback. Should be set to true for most cores, but some may exhibit janky behavior when deserializing content." + ) MSG_HASH( MENU_ENUM_LABEL_VALUE_SAVESTATE_AUTO_INDEX, "Increment Save State Index Automatically" diff --git a/libretro-db/rmsgpack.h b/libretro-db/rmsgpack.h index 81b682eec3..adf40583be 100644 --- a/libretro-db/rmsgpack.h +++ b/libretro-db/rmsgpack.h @@ -53,7 +53,7 @@ int rmsgpack_write_bool(intfstream_t *stream, int value); int rmsgpack_write_int(intfstream_t *stream, int64_t value); -int rmsgpack_write_uint(intfstream_t *stream, uint64_t value ); +int rmsgpack_write_uint(intfstream_t *stream, uint64_t value); int rmsgpack_read(intfstream_t *stream, struct rmsgpack_read_callbacks *callbacks, void *data); diff --git a/libretro-db/rmsgpack_dom.c b/libretro-db/rmsgpack_dom.c index 88ad6c2784..be163c1147 100644 --- a/libretro-db/rmsgpack_dom.c +++ b/libretro-db/rmsgpack_dom.c @@ -30,27 +30,44 @@ #include "rmsgpack.h" +/* For the simple stack-based reader state */ #define MAX_DEPTH 128 -struct dom_reader_state +struct rmsgpack_dom_reader_state { int i; - struct rmsgpack_dom_value *stack[MAX_DEPTH]; + bool growable; + int capacity; + struct rmsgpack_dom_value **stack; }; -static struct rmsgpack_dom_value *dom_reader_state_pop( - struct dom_reader_state *s) +static struct rmsgpack_dom_value *rmsgpack_dom_reader_state_pop( + struct rmsgpack_dom_reader_state *s) { struct rmsgpack_dom_value *v = s->stack[s->i]; s->i--; return v; } -static int dom_reader_state_push( - struct dom_reader_state *s, struct rmsgpack_dom_value *v) +static int rmsgpack_dom_reader_state_push( + struct rmsgpack_dom_reader_state *s, struct rmsgpack_dom_value *v) { - if ((s->i + 1) == MAX_DEPTH) - return -1; + if ((s->i + 1) == s->capacity) + { + if(s->growable) + { + s->capacity *= 2; + s->stack = realloc(s->stack, s->capacity * sizeof(struct rmsgpack_dom_value *)); + if(!s->stack) + { + printf("[RMSGPACK_DOM] failed to reallocate stack to %ld\n", + s->capacity*sizeof(struct rmsgpack_dom_value *)); + return -1; + } + } + else + return -1; + } s->i++; s->stack[s->i] = v; return 0; @@ -58,18 +75,18 @@ static int dom_reader_state_push( static int dom_read_nil(void *data) { - struct dom_reader_state *dom_state = (struct dom_reader_state *)data; + struct rmsgpack_dom_reader_state *dom_state = (struct rmsgpack_dom_reader_state *)data; struct rmsgpack_dom_value *v = - (struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state); + (struct rmsgpack_dom_value*)rmsgpack_dom_reader_state_pop(dom_state); v->type = RDT_NULL; return 0; } static int dom_read_bool(int value, void *data) { - struct dom_reader_state *dom_state = (struct dom_reader_state *)data; + struct rmsgpack_dom_reader_state *dom_state = (struct rmsgpack_dom_reader_state *)data; struct rmsgpack_dom_value *v = - (struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state); + (struct rmsgpack_dom_value*)rmsgpack_dom_reader_state_pop(dom_state); v->type = RDT_BOOL; v->val.bool_ = value; return 0; @@ -77,9 +94,9 @@ static int dom_read_bool(int value, void *data) static int dom_read_int(int64_t value, void *data) { - struct dom_reader_state *dom_state = (struct dom_reader_state *)data; + struct rmsgpack_dom_reader_state *dom_state = (struct rmsgpack_dom_reader_state *)data; struct rmsgpack_dom_value *v = - (struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state); + (struct rmsgpack_dom_value*)rmsgpack_dom_reader_state_pop(dom_state); v->type = RDT_INT; v->val.int_ = value; return 0; @@ -87,9 +104,9 @@ static int dom_read_int(int64_t value, void *data) static int dom_read_uint(uint64_t value, void *data) { - struct dom_reader_state *dom_state = (struct dom_reader_state *)data; + struct rmsgpack_dom_reader_state *dom_state = (struct rmsgpack_dom_reader_state *)data; struct rmsgpack_dom_value *v = - (struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state); + (struct rmsgpack_dom_value*)rmsgpack_dom_reader_state_pop(dom_state); v->type = RDT_UINT; v->val.uint_ = value; return 0; @@ -97,9 +114,9 @@ static int dom_read_uint(uint64_t value, void *data) static int dom_read_string(char *value, uint32_t len, void *data) { - struct dom_reader_state *dom_state = (struct dom_reader_state *)data; + struct rmsgpack_dom_reader_state *dom_state = (struct rmsgpack_dom_reader_state *)data; struct rmsgpack_dom_value *v = - (struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state); + (struct rmsgpack_dom_value*)rmsgpack_dom_reader_state_pop(dom_state); v->type = RDT_STRING; v->val.string.len = len; @@ -109,9 +126,9 @@ static int dom_read_string(char *value, uint32_t len, void *data) static int dom_read_bin(void *value, uint32_t len, void *data) { - struct dom_reader_state *dom_state = (struct dom_reader_state *)data; + struct rmsgpack_dom_reader_state *dom_state = (struct rmsgpack_dom_reader_state *)data; struct rmsgpack_dom_value *v = (struct rmsgpack_dom_value*) - dom_reader_state_pop(dom_state); + rmsgpack_dom_reader_state_pop(dom_state); v->type = RDT_BINARY; v->val.binary.len = len; v->val.binary.buff = (char *)value; @@ -122,8 +139,8 @@ static int dom_read_map_start(uint32_t len, void *data) { unsigned i; struct rmsgpack_dom_pair *items = NULL; - struct dom_reader_state *dom_state = (struct dom_reader_state *)data; - struct rmsgpack_dom_value *v = dom_reader_state_pop(dom_state); + struct rmsgpack_dom_reader_state *dom_state = (struct rmsgpack_dom_reader_state *)data; + struct rmsgpack_dom_value *v = rmsgpack_dom_reader_state_pop(dom_state); v->type = RDT_MAP; v->val.map.len = len; @@ -137,9 +154,9 @@ static int dom_read_map_start(uint32_t len, void *data) for (i = 0; i < len; i++) { - if (dom_reader_state_push(dom_state, &items[i].value) < 0) + if (rmsgpack_dom_reader_state_push(dom_state, &items[i].value) < 0) return -1; - if (dom_reader_state_push(dom_state, &items[i].key) < 0) + if (rmsgpack_dom_reader_state_push(dom_state, &items[i].key) < 0) return -1; } @@ -149,8 +166,8 @@ static int dom_read_map_start(uint32_t len, void *data) static int dom_read_array_start(uint32_t len, void *data) { size_t i; - struct dom_reader_state *dom_state = (struct dom_reader_state *)data; - struct rmsgpack_dom_value *v = dom_reader_state_pop(dom_state); + struct rmsgpack_dom_reader_state *dom_state = (struct rmsgpack_dom_reader_state *)data; + struct rmsgpack_dom_value *v = rmsgpack_dom_reader_state_pop(dom_state); struct rmsgpack_dom_value *items = NULL; v->type = RDT_ARRAY; @@ -165,7 +182,7 @@ static int dom_read_array_start(uint32_t len, void *data) for (i = 0; i < len; i++) { - if (dom_reader_state_push(dom_state, &items[i]) < 0) + if (rmsgpack_dom_reader_state_push(dom_state, &items[len-i-1]) < 0) return -1; } @@ -403,20 +420,41 @@ static struct rmsgpack_read_callbacks dom_reader_callbacks = { dom_read_array_start }; -int rmsgpack_dom_read(intfstream_t *fd, struct rmsgpack_dom_value *out) +int rmsgpack_dom_read_with(intfstream_t *fd, struct rmsgpack_dom_value *out, struct rmsgpack_dom_reader_state *s) { int rv; - struct dom_reader_state s; - - s.i = 0; - s.stack[0] = out; - - if ((rv = rmsgpack_read(fd, &dom_reader_callbacks, &s)) < 0) + s->i = 0; + s->stack[0] = out; + if ((rv = rmsgpack_read(fd, &dom_reader_callbacks, s)) < 0) rmsgpack_dom_value_free(out); - return rv; } +struct rmsgpack_dom_reader_state *rmsgpack_dom_reader_state_new() +{ + struct rmsgpack_dom_reader_state *s = calloc(1, sizeof(struct rmsgpack_dom_reader_state)); + s->i = 0; + s->capacity = 1024; + s->growable = true; + s->stack = calloc(1024, sizeof(struct rmsgpack_dom_value *)); + return s; +} +void rmsgpack_dom_reader_state_free(struct rmsgpack_dom_reader_state *state) +{ + free(state->stack); + free(state); +} + +int rmsgpack_dom_read(intfstream_t *fd, struct rmsgpack_dom_value *out) +{ + struct rmsgpack_dom_reader_state s; + s.i = 0; + s.growable = false; + s.capacity = MAX_DEPTH; + s.stack = alloca(MAX_DEPTH); + return rmsgpack_dom_read_with(fd, out, &s); +} + int rmsgpack_dom_read_into(intfstream_t *fd, ...) { int rv; diff --git a/libretro-db/rmsgpack_dom.h b/libretro-db/rmsgpack_dom.h index 27c130ec49..3a41dd0db2 100644 --- a/libretro-db/rmsgpack_dom.h +++ b/libretro-db/rmsgpack_dom.h @@ -79,6 +79,8 @@ struct rmsgpack_dom_pair struct rmsgpack_dom_value value; /* uint64_t alignment */ }; +struct dom_reader_state; + void rmsgpack_dom_value_print(struct rmsgpack_dom_value *obj); void rmsgpack_dom_value_free(struct rmsgpack_dom_value *v); @@ -91,6 +93,10 @@ struct rmsgpack_dom_value *rmsgpack_dom_value_map_value( int rmsgpack_dom_read(intfstream_t *stream, struct rmsgpack_dom_value *out); +struct rmsgpack_dom_reader_state *rmsgpack_dom_reader_state_new(); +int rmsgpack_dom_read_with(intfstream_t *stream, struct rmsgpack_dom_value *out, struct rmsgpack_dom_reader_state *state); +void rmsgpack_dom_reader_state_free(struct rmsgpack_dom_reader_state *state); + int rmsgpack_dom_write(intfstream_t *stream, const struct rmsgpack_dom_value *obj); int rmsgpack_dom_read_into(intfstream_t *stream, ...); diff --git a/menu/cbs/menu_cbs_sublabel.c b/menu/cbs/menu_cbs_sublabel.c index 6ee3feb12d..bc504d1000 100644 --- a/menu/cbs/menu_cbs_sublabel.c +++ b/menu/cbs/menu_cbs_sublabel.c @@ -814,6 +814,7 @@ DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_savestate_max_keep, MENU_ DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_autosave_interval, MENU_ENUM_SUBLABEL_AUTOSAVE_INTERVAL) DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_replay_max_keep, MENU_ENUM_SUBLABEL_REPLAY_MAX_KEEP) DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_replay_checkpoint_interval, MENU_ENUM_SUBLABEL_REPLAY_CHECKPOINT_INTERVAL) +DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_replay_checkpoint_deserialize, MENU_ENUM_SUBLABEL_REPLAY_CHECKPOINT_DESERIALIZE) DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_input_remap_binds_enable, MENU_ENUM_SUBLABEL_INPUT_REMAP_BINDS_ENABLE) DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_input_remap_sort_by_controller_enable, MENU_ENUM_SUBLABEL_INPUT_REMAP_SORT_BY_CONTROLLER_ENABLE) DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_input_autodetect_enable, MENU_ENUM_SUBLABEL_INPUT_AUTODETECT_ENABLE) @@ -4125,6 +4126,9 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs, case MENU_ENUM_LABEL_REPLAY_CHECKPOINT_INTERVAL: BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_replay_checkpoint_interval); break; + case MENU_ENUM_LABEL_REPLAY_CHECKPOINT_DESERIALIZE: + BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_replay_checkpoint_deserialize); + break; case MENU_ENUM_LABEL_SAVESTATE_MAX_KEEP: BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_savestate_max_keep); break; diff --git a/menu/menu_displaylist.c b/menu/menu_displaylist.c index 18a6cc059a..7dabda1f80 100644 --- a/menu/menu_displaylist.c +++ b/menu/menu_displaylist.c @@ -10995,6 +10995,7 @@ unsigned menu_displaylist_build_list( {MENU_ENUM_LABEL_REPLAY_AUTO_INDEX, PARSE_ONLY_BOOL, true}, {MENU_ENUM_LABEL_REPLAY_MAX_KEEP, PARSE_ONLY_UINT, false}, {MENU_ENUM_LABEL_REPLAY_CHECKPOINT_INTERVAL, PARSE_ONLY_UINT, true}, + {MENU_ENUM_LABEL_REPLAY_CHECKPOINT_DESERIALIZE, PARSE_ONLY_BOOL, true}, {MENU_ENUM_LABEL_CONTENT_RUNTIME_LOG, PARSE_ONLY_BOOL, true}, {MENU_ENUM_LABEL_CONTENT_RUNTIME_LOG_AGGREGATE, PARSE_ONLY_BOOL, true}, #if HAVE_CLOUDSYNC diff --git a/menu/menu_setting.c b/menu/menu_setting.c index 54b995b570..9f21f3100f 100644 --- a/menu/menu_setting.c +++ b/menu/menu_setting.c @@ -11607,6 +11607,21 @@ static bool setting_append_list( (*list)[list_info->index - 1].get_string_representation = &setting_get_string_representation_uint_replay_checkpoint_interval; menu_settings_list_current_add_range(list, list_info, 0, 3600, 60, true, true); + + CONFIG_BOOL( + list, list_info, + &settings->bools.replay_checkpoint_deserialize, + MENU_ENUM_LABEL_REPLAY_CHECKPOINT_DESERIALIZE, + MENU_ENUM_LABEL_VALUE_REPLAY_CHECKPOINT_DESERIALIZE, + DEFAULT_REPLAY_CHECKPOINT_DESERIALIZE, + MENU_ENUM_LABEL_VALUE_OFF, + MENU_ENUM_LABEL_VALUE_ON, + &group_info, + &subgroup_info, + parent_group, + general_write_handler, + general_read_handler, + SD_FLAG_NONE); #endif CONFIG_BOOL( diff --git a/msg_hash.h b/msg_hash.h index aab681cfad..9018345bb0 100644 --- a/msg_hash.h +++ b/msg_hash.h @@ -2596,6 +2596,7 @@ enum msg_hash_enums MENU_LBL_H(LIBRETRO_LOG_LEVEL), MENU_LBL_H(AUTOSAVE_INTERVAL), MENU_LBL_H(REPLAY_CHECKPOINT_INTERVAL), + MENU_LBL_H(REPLAY_CHECKPOINT_DESERIALIZE), MENU_LBL_H(CONFIG_SAVE_ON_EXIT), MENU_LABEL(REMAP_SAVE_ON_EXIT), MENU_LABEL(CONFIGURATION_LIST), diff --git a/tasks/task_movie.c b/tasks/task_movie.c index eceec8d322..dd8cc923d6 100644 --- a/tasks/task_movie.c +++ b/tasks/task_movie.c @@ -43,30 +43,41 @@ #include "../runloop.h" #include "tasks_internal.h" #include "../input/input_driver.h" +#include "../input/bsv/bsvmovie.h" -#define MAGIC_INDEX 0 -#define VERSION_INDEX 1 -#define CRC_INDEX 2 -#define STATE_SIZE_INDEX 3 -/* Identifier is int64_t, so takes up two slots */ -#define IDENTIFIER_INDEX 4 -#define HEADER_LEN 6 +#ifdef HAVE_STATESTREAM +#if DEBUG +#include "input/bsv/uint32s_index.h" +#endif -#define REPLAY_FORMAT_VERSION 1 -#define REPLAY_MAGIC 0x42535632 +#define REPLAY_DEFAULT_COMMIT_INTERVAL 4 +#define REPLAY_DEFAULT_COMMIT_THRESHOLD 2 + +/* Superblock and block sizes for incremental savestates. */ +#define DEFAULT_SUPERBLOCK_SIZE 16 /* measured in blocks */ +#define DEFAULT_BLOCK_SIZE 16384 /* measured in bytes */ + +#define SMALL_STATE_THRESHOLD (1<<20) /* states < 1MB are "small" and are tuned differently */ +#define SMALL_SUPERBLOCK_SIZE 16 /* measured in blocks */ +#define SMALL_BLOCK_SIZE 128 /* measured in bytes */ + +#endif /* Forward declaration */ bool content_load_state_in_progress(void* data); /* Private functions */ -static bool bsv_movie_init_playback( - bsv_movie_t *handle, const char *path) +static bool bsv_movie_init_playback(bsv_movie_t *handle, const char *path) { int64_t *identifier_loc; uint32_t state_size = 0; - uint32_t header[HEADER_LEN] = {0}; + uint32_t header[REPLAY_HEADER_LEN] = {0}; uint32_t vsn = 0; +#ifdef HAVE_STATESTREAM + uint32_t superblock_size = DEFAULT_SUPERBLOCK_SIZE, + block_size = DEFAULT_BLOCK_SIZE/4; +#endif intfstream_t *file = intfstream_open_file(path, RETRO_VFS_FILE_ACCESS_READ, RETRO_VFS_FILE_ACCESS_HINT_NONE); @@ -80,46 +91,38 @@ static bool bsv_movie_init_playback( handle->file = file; handle->playback = true; - intfstream_read(handle->file, header, sizeof(uint32_t) * HEADER_LEN); - if (swap_if_big32(header[MAGIC_INDEX]) != REPLAY_MAGIC) + intfstream_read(handle->file, header, sizeof(uint32_t) * REPLAY_HEADER_LEN); + if (swap_if_big32(header[REPLAY_HEADER_MAGIC_INDEX]) != REPLAY_MAGIC) { - RARCH_ERR("[Replay] %s\n", msg_hash_to_str(MSG_MOVIE_FILE_IS_NOT_A_VALID_REPLAY_FILE)); + RARCH_ERR("[Replay] %s : %s : magic %d vs %d \n", msg_hash_to_str(MSG_MOVIE_FILE_IS_NOT_A_VALID_REPLAY_FILE), path, swap_if_big32(header[REPLAY_HEADER_MAGIC_INDEX]), REPLAY_MAGIC); return false; } - vsn = swap_if_big32(header[VERSION_INDEX]); + vsn = swap_if_big32(header[REPLAY_HEADER_VERSION_INDEX]); if (vsn > REPLAY_FORMAT_VERSION) { - RARCH_ERR("[Replay] %s\n", msg_hash_to_str(MSG_MOVIE_FILE_IS_NOT_A_VALID_REPLAY_FILE)); + RARCH_ERR("[Replay] %s : vsn %d vs %d\n", msg_hash_to_str(MSG_MOVIE_FILE_IS_NOT_A_VALID_REPLAY_FILE), vsn, REPLAY_FORMAT_VERSION); return false; } handle->version = vsn; - state_size = swap_if_big32(header[STATE_SIZE_INDEX]); - identifier_loc = (int64_t *)(header+IDENTIFIER_INDEX); + state_size = swap_if_big32(header[REPLAY_HEADER_STATE_SIZE_INDEX]); + identifier_loc = (int64_t *)(header+REPLAY_HEADER_IDENTIFIER_INDEX); handle->identifier = swap_if_big64(*identifier_loc); -#if 0 - RARCH_ERR("----- debug %u -----\n", header[0]); - RARCH_ERR("----- debug %u -----\n", header[1]); - RARCH_ERR("----- debug %u -----\n", header[2]); - RARCH_ERR("----- debug %u -----\n", header[3]); - RARCH_ERR("----- debug %u -----\n", header[4]); - RARCH_ERR("----- debug %u -----\n", header[5]); -#endif - - if (state_size) + handle->min_file_pos = sizeof(header) + state_size; + if (state_size && vsn <= 1) { size_t info_size; retro_ctx_serialize_info_t serial_info; - uint8_t *buf = (uint8_t*)malloc(state_size); + uint8_t *buf = (uint8_t*)malloc(state_size); if (!buf) return false; - handle->state = buf; - handle->state_size = state_size; - if (intfstream_read(handle->file, - handle->state, state_size) != state_size) + /* The header used to be six ints long */ + intfstream_seek(handle->file, REPLAY_HEADER_V0V1_LEN_BYTES, SEEK_SET); + + if (intfstream_read(handle->file, buf, state_size) != state_size) { RARCH_ERR("[Replay] %s\n", msg_hash_to_str(MSG_COULD_NOT_READ_STATE_FROM_MOVIE)); return false; @@ -127,19 +130,37 @@ static bool bsv_movie_init_playback( info_size = core_serialize_size(); /* For cores like dosbox, the reported size is not always correct. So we just give a warning if they don't match up. */ - serial_info.data_const = handle->state; + serial_info.data_const = buf; serial_info.size = state_size; core_unserialize(&serial_info); + free(buf); if (info_size != state_size) - { RARCH_WARN("[Replay] %s\n", msg_hash_to_str(MSG_MOVIE_FORMAT_DIFFERENT_SERIALIZER_VERSION)); - } + } + else if (vsn >= 2) + { + uint8_t compression, encoding; +#ifdef HAVE_STATESTREAM + uint32_t commit_settings = header[REPLAY_HEADER_CHECKPOINT_CONFIG_INDEX]; + superblock_size = swap_if_big32(header[REPLAY_HEADER_SUPERBLOCK_SIZE_INDEX]); + block_size = swap_if_big32(header[REPLAY_HEADER_BLOCK_SIZE_INDEX]); + handle->commit_interval = commit_settings >> 24; + handle->commit_threshold = (commit_settings >> 16) & 0x000000FF; + handle->checkpoint_compression = (commit_settings >> 8) & 0x000000FF; + handle->superblocks = uint32s_index_new(superblock_size,handle->commit_interval,handle->commit_threshold); + handle->blocks = uint32s_index_new(block_size/4,handle->commit_interval,handle->commit_threshold); +#endif + + if (intfstream_read(handle->file, &(compression), sizeof(uint8_t)) != sizeof(uint8_t) || + intfstream_read(handle->file, &(encoding), sizeof(uint8_t)) != sizeof(uint8_t)) + return false; + if (!bsv_movie_load_checkpoint(handle, compression, encoding, false)) + return false; } - handle->min_file_pos = sizeof(header) + state_size; if(vsn > 0) - bsv_movie_read_next_events(handle); + bsv_movie_read_next_events(handle, false); return true; @@ -153,10 +174,16 @@ static bool bsv_movie_init_record( time_t time_lil = swap_if_big64(t); uint32_t state_size = 0; uint32_t content_crc = 0; - uint32_t header[HEADER_LEN] = {0}; + uint32_t header[REPLAY_HEADER_LEN] = {0}; intfstream_t *file = intfstream_open_file(path, RETRO_VFS_FILE_ACCESS_WRITE | RETRO_VFS_FILE_ACCESS_READ, RETRO_VFS_FILE_ACCESS_HINT_NONE); + settings_t *settings = config_get_ptr(); +#ifdef HAVE_STATESTREAM + bool is_small = false; + uint32_t superblock_size; + uint32_t block_size; +#endif if (!file) { @@ -166,43 +193,58 @@ static bool bsv_movie_init_record( handle->file = file; handle->version = REPLAY_FORMAT_VERSION; +#ifdef HAVE_STATESTREAM + handle->commit_interval = REPLAY_DEFAULT_COMMIT_INTERVAL; + handle->commit_threshold = REPLAY_DEFAULT_COMMIT_THRESHOLD; +#endif + handle->checkpoint_compression = REPLAY_CHECKPOINT2_COMPRESSION_NONE; + if (settings->bools.savestate_file_compression) +#if defined(HAVE_ZSTD) + handle->checkpoint_compression = REPLAY_CHECKPOINT2_COMPRESSION_ZSTD; +#elif defined(HAVE_ZLIB) + handle->checkpoint_compression = REPLAY_CHECKPOINT2_COMPRESSION_ZLIB; +#else + {} +#endif content_crc = content_get_crc(); - header[MAGIC_INDEX] = swap_if_big32(REPLAY_MAGIC); - header[VERSION_INDEX] = swap_if_big32(handle->version); - header[CRC_INDEX] = swap_if_big32(content_crc); + header[REPLAY_HEADER_MAGIC_INDEX] = swap_if_big32(REPLAY_MAGIC); + header[REPLAY_HEADER_VERSION_INDEX] = swap_if_big32(handle->version); + header[REPLAY_HEADER_CRC_INDEX] = swap_if_big32(content_crc); info_size = core_serialize_size(); - state_size = (unsigned)info_size; +#ifdef HAVE_STATESTREAM + is_small = info_size < SMALL_STATE_THRESHOLD; + superblock_size = is_small ? SMALL_SUPERBLOCK_SIZE : DEFAULT_SUPERBLOCK_SIZE; + block_size = is_small ? SMALL_BLOCK_SIZE : DEFAULT_BLOCK_SIZE; +#endif + header[REPLAY_HEADER_STATE_SIZE_INDEX] = 0; /* Will fill this in later */ + header[REPLAY_HEADER_FRAME_COUNT_INDEX] = 0; +#ifdef HAVE_STATESTREAM + header[REPLAY_HEADER_BLOCK_SIZE_INDEX] = swap_if_big32(block_size); + header[REPLAY_HEADER_SUPERBLOCK_SIZE_INDEX] = swap_if_big32(superblock_size); + header[REPLAY_HEADER_CHECKPOINT_CONFIG_INDEX] = (((uint32_t)handle->commit_interval) << 24) | + ((uint32_t)handle->commit_threshold << 16) | + (((uint32_t)handle->checkpoint_compression) << 8); +#else + header[REPLAY_HEADER_BLOCK_SIZE_INDEX] = 0; + header[REPLAY_HEADER_SUPERBLOCK_SIZE_INDEX] = 0; + header[REPLAY_HEADER_CHECKPOINT_CONFIG_INDEX] = 0; +#endif + handle->identifier = (int64_t)t; + *((int64_t *)(header+REPLAY_HEADER_IDENTIFIER_INDEX)) = time_lil; + intfstream_write(handle->file, header, REPLAY_HEADER_LEN_BYTES); - header[STATE_SIZE_INDEX] = swap_if_big32(state_size); - handle->identifier = (int64_t)t; - *((int64_t *)(header+IDENTIFIER_INDEX)) = time_lil; - intfstream_write(handle->file, header, HEADER_LEN * sizeof(uint32_t)); - - handle->min_file_pos = sizeof(header) + state_size; - handle->state_size = state_size; - +#ifdef HAVE_STATESTREAM + handle->superblocks = uint32s_index_new(superblock_size, handle->commit_interval, handle->commit_threshold); + handle->blocks = uint32s_index_new(block_size/4, handle->commit_interval, handle->commit_threshold); +#endif if (state_size) - { - retro_ctx_serialize_info_t serial_info; - uint8_t *st = (uint8_t*)malloc(state_size); - - if (!st) - return false; - - handle->state = st; - - serial_info.data = handle->state; - serial_info.size = state_size; - - core_serialize(&serial_info); - - intfstream_write(handle->file, - handle->state, state_size); - } + return bsv_movie_reset_recording(handle); + + handle->min_file_pos = sizeof(header); return true; } @@ -212,8 +254,18 @@ void bsv_movie_free(bsv_movie_t *handle) intfstream_close(handle->file); free(handle->file); - free(handle->state); free(handle->frame_pos); + +#ifdef HAVE_STATESTREAM + uint32s_index_free(handle->superblocks); + uint32s_index_free(handle->blocks); + free(handle->superblock_seq); +#endif + if (handle->last_save) + free(handle->last_save); + if (handle->cur_save) + free(handle->cur_save); + free(handle); } @@ -239,7 +291,6 @@ static bsv_movie_t *bsv_movie_init_internal(const char *path, enum rarch_movie_t goto error; handle->frame_pos = frame_pos; - handle->frame_pos[0] = handle->min_file_pos; handle->frame_mask = (1 << 20) - 1; @@ -378,6 +429,14 @@ bool movie_stop_playback(input_driver_state_t *input_st) /* Checks if movie is being played back. */ if (!(input_st->bsv_movie_state.flags & BSV_FLAG_MOVIE_PLAYBACK)) return false; +#ifdef HAVE_STATESTREAM +#if DEBUG + RARCH_DBG("[Replay] superblock histogram\n"); + uint32s_index_print_count_data(input_st->bsv_movie_state_handle->superblocks); + RARCH_DBG("[Replay] block histogram\n"); + uint32s_index_print_count_data(input_st->bsv_movie_state_handle->blocks); +#endif +#endif _msg = msg_hash_to_str(MSG_MOVIE_PLAYBACK_ENDED); runloop_msg_queue_push(_msg, strlen(_msg), 2, 180, false, NULL, MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); @@ -393,12 +452,25 @@ bool movie_stop_playback(input_driver_state_t *input_st) /* in the future this should probably be a deferred task as well */ bool movie_stop_record(input_driver_state_t *input_st) { + bsv_movie_t *movie = input_st->bsv_movie_state_handle; + uint32_t frame_count; const char *_msg = msg_hash_to_str(MSG_MOVIE_RECORD_STOPPED); - if (!(input_st->bsv_movie_state_handle)) + if (!movie) return false; runloop_msg_queue_push(_msg, strlen(_msg), 2, 180, true, NULL, MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); RARCH_LOG("[Replay] %s\n", _msg); +#ifdef HAVE_STATESTREAM +#if DEBUG + RARCH_DBG("[Replay] superblock histogram\n"); + uint32s_index_print_count_data(movie->superblocks); + RARCH_DBG("[Replay] block histogram\n"); + uint32s_index_print_count_data(movie->blocks); +#endif +#endif + frame_count = swap_if_big32(movie->frame_counter); + intfstream_seek(movie->file, REPLAY_HEADER_FRAME_COUNT_INDEX*sizeof(uint32_t), SEEK_SET); + intfstream_write(movie->file, &frame_count, sizeof(uint32_t)); bsv_movie_deinit_full(input_st); input_st->bsv_movie_state.flags &= ~( BSV_FLAG_MOVIE_END diff --git a/tasks/task_save.c b/tasks/task_save.c index 8185d83fd6..343a7e288b 100644 --- a/tasks/task_save.c +++ b/tasks/task_save.c @@ -860,7 +860,6 @@ static bool content_load_rastate1(unsigned char* input, size_t len) #else bool frame_is_reversed = false; #endif - if (BSV_MOVIE_IS_RECORDING() && !seen_replay && !frame_is_reversed) { /* TODO OSD message */