From c9908403165bd6fe5c9f146ce39cc58366459e24 Mon Sep 17 00:00:00 2001 From: Robert Olsson Date: Sun, 6 Sep 2026 14:34:19 +0200 Subject: [PATCH] Optimize search performance Build search results in /tmp, then swap them onto the card Every result row was a separate INSERT compiled and committed straight to the SD card, so a search across a large collection spent most of its time waiting on the SD card. The old database was deleted up front too, so an interrupted search left you with nothing. The list is now built in /tmp with one prepared statement inside a single transaction, copied over once, and renamed into place. The previous results stay readable until that rename. If anything fails we drop the old database so MainUI rebuilds an empty list instead of showing hits for a keyword that is no longer in active_search. --- src/common/db_cache.hpp | 48 +++++++ src/search/search.hpp | 305 +++++++++++++++++++++++++++++++--------- 2 files changed, 287 insertions(+), 66 deletions(-) diff --git a/src/common/db_cache.hpp b/src/common/db_cache.hpp index 36fed05..92e34f8 100644 --- a/src/common/db_cache.hpp +++ b/src/common/db_cache.hpp @@ -232,6 +232,54 @@ bool insertRom(sqlite3 *db, string name, RomEntry entry) return execSql(db, sql::insert(name, entry)); } +// Search builds can reuse one prepared INSERT for all result rows. +bool prepareRomInsert(sqlite3 *db, string name, sqlite3_stmt **stmt) +{ + string table = TABLE_NAME(name); + char *query = sqlite3_mprintf( + "INSERT INTO %Q (disp, path, imgpath, type, ppath, pinyin, cpinyin) " + "VALUES (?, ?, ?, ?, ?, ?, '');", table.c_str()); + if (query == NULL) + return false; + + int rc = sqlite3_prepare_v2(db, query, -1, stmt, NULL); + if (rc != SQLITE_OK) { + std::cerr << "ERROR: while compiling prepared insert: " + << sqlite3_errmsg(db) << std::endl; + sqlite3_free(query); + return false; + } + + sqlite3_free(query); + return true; +} + +bool insertRomPrepared(sqlite3 *db, sqlite3_stmt *stmt, const RomEntry &entry) +{ + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + + if (sqlite3_bind_text(stmt, 1, entry.label.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK || + sqlite3_bind_text(stmt, 2, entry.path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK || + sqlite3_bind_text(stmt, 3, entry.imgpath.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK || + sqlite3_bind_int(stmt, 4, entry.type) != SQLITE_OK || + sqlite3_bind_text(stmt, 5, entry.ppath.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK || + sqlite3_bind_text(stmt, 6, entry.label.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) { + std::cerr << "ERROR: while binding prepared insert: " + << sqlite3_errmsg(db) << std::endl; + return false; + } + + int rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + std::cerr << "ERROR: while performing prepared insert: " + << sqlite3_errmsg(db) << std::endl; + return false; + } + + return true; +} + bool duplicateResults(sqlite3 *db, string name, string ppath) { return execSql(db, sql::dupChangePpath(name, ppath)); diff --git a/src/search/search.hpp b/src/search/search.hpp index 362690b..4602636 100644 --- a/src/search/search.hpp +++ b/src/search/search.hpp @@ -19,9 +19,129 @@ using std::vector; const string DB_NAME = "data"; const string DB_DIR = fullpath("/mnt/SDCARD/App/Search/" + DB_NAME); const string DB_PATH = DB_DIR + "/" + CACHE_NAME(DB_NAME); +const string DB_TMP_PATH = "/tmp/onion-search-" + CACHE_NAME(DB_NAME); +const string DB_STAGE_PATH = DB_PATH + ".new"; static SDL_Surface* search_icon = NULL; +bool execSearchSql(sqlite3 *db, const string &sql) +{ + char *error = NULL; + int rc = sqlite3_exec(db, sql.c_str(), NULL, NULL, &error); + + if (rc != SQLITE_OK) { + std::cerr << "Search DB SQL error: " + << (error ? error : sqlite3_errmsg(db)) << std::endl; + if (error) + sqlite3_free(error); + return false; + } + + return true; +} + +void abortSearchDatabase(sqlite3 *db, sqlite3_stmt *insert_stmt) +{ + if (insert_stmt) + sqlite3_finalize(insert_stmt); + + // No ROLLBACK: sqlite documents it as undefined with journal_mode=OFF. + // Removing the file below is what actually discards the build. + if (db) + sqlite3_close(db); + + remove(DB_TMP_PATH.c_str()); + remove(DB_STAGE_PATH.c_str()); + + // main.cpp has already stored the new keyword in `active_search`, so + // keeping the previous database would list results for a keyword the user + // can no longer see. Dropping it makes MainUI rebuild an empty list. + remove(DB_PATH.c_str()); +} + +bool publishSearchDatabase() +{ + remove(DB_STAGE_PATH.c_str()); + + FILE *src = fopen(DB_TMP_PATH.c_str(), "rb"); + if (!src) { + std::cerr << "Couldn't open temporary search database for reading" << std::endl; + return false; + } + + FILE *dst = fopen(DB_STAGE_PATH.c_str(), "wb"); + if (!dst) { + std::cerr << "Couldn't open staged search database for writing" << std::endl; + fclose(src); + return false; + } + + // Static, not on the stack: 64 KiB is already more than the SD card takes + // per write, and performSearch's frame stays small. + static char buffer[64 * 1024]; + bool ok = true; + size_t count; + + while ((count = fread(buffer, 1, sizeof(buffer), src)) > 0) { + if (fwrite(buffer, 1, count, dst) != count) { + ok = false; + break; + } + } + + if (ferror(src)) + ok = false; + + if (ok && fflush(dst) != 0) + ok = false; + if (ok && fsync(fileno(dst)) != 0) + ok = false; + + if (fclose(src) != 0) + ok = false; + if (fclose(dst) != 0) + ok = false; + + if (!ok) { + std::cerr << "Couldn't copy completed search database to SD" << std::endl; + remove(DB_STAGE_PATH.c_str()); + return false; + } + + // POSIX rename replaces the old result DB in one filesystem operation. + if (rename(DB_STAGE_PATH.c_str(), DB_PATH.c_str()) != 0) { + std::cerr << "Couldn't install completed search database" << std::endl; + remove(DB_STAGE_PATH.c_str()); + return false; + } + + remove(DB_TMP_PATH.c_str()); + return true; +} + +bool finishSearchDatabase(sqlite3 *db, sqlite3_stmt *insert_stmt) +{ + sqlite3_finalize(insert_stmt); + + if (!execSearchSql(db, "COMMIT;")) { + abortSearchDatabase(db, NULL); + return false; + } + + if (sqlite3_close(db) != SQLITE_OK) { + abortSearchDatabase(NULL, NULL); + return false; + } + + if (!publishSearchDatabase()) { + // Nothing was installed, so don't leave the old results behind. + remove(DB_PATH.c_str()); + return false; + } + + return true; +} + void updateDisplay(Display *display, string msg, string submsg = "") { display->clear(); @@ -77,35 +197,59 @@ string totalTextMessage(int total) void performSearch(Display* display, string keyword) { - if (exists(DB_PATH)) - remove(DB_PATH.c_str()); - - string path = DB_PATH; + // Build the complete result database in /tmp. Keep the previous SD-card + // database intact until the replacement has been committed and closed. + remove(DB_TMP_PATH.c_str()); + remove(DB_STAGE_PATH.c_str()); - if (!db::create(DB_PATH, DB_NAME)) { - std::cerr << "Couldn't create database" << std::endl; + if (!db::create(DB_TMP_PATH, DB_NAME)) { + std::cerr << "Couldn't create temporary search database" << std::endl; return; } - sqlite3* db; + sqlite3* db = NULL; + sqlite3_stmt* insert_stmt = NULL; int total_lines = 0; // Open the database file - if (!db::open(&db, DB_PATH)) + if (!db::open(&db, DB_TMP_PATH)) { + remove(DB_TMP_PATH.c_str()); return; + } + + // The build database is disposable and RAM-backed, so durability work is + // unnecessary until the one final SD-card copy. + if (!execSearchSql(db, + "PRAGMA journal_mode=OFF;" + "PRAGMA synchronous=OFF;" + "PRAGMA temp_store=MEMORY;") || + !execSearchSql(db, "BEGIN TRANSACTION;") || + !db::prepareRomInsert(db, DB_NAME, &insert_stmt)) { + abortSearchDatabase(db, insert_stmt); + return; + } + + auto insert = [db, insert_stmt](const RomEntry &entry) { + return db::insertRomPrepared(db, insert_stmt, entry); + }; keyword = trim(keyword); if (keyword.length() == 0) { - db::insertRom(db, DB_NAME, { - .label = "Enter search term...", - .path = "search", - .imgpath = DB_DIR + "/Imgs/Enter search term....png" - }); + if (!insert({ + .label = "Enter search term...", + .path = "search", + .imgpath = DB_DIR + "/Imgs/Enter search term....png" + })) { + abortSearchDatabase(db, insert_stmt); + return; + } + // addTools(db); total_lines += 2; // db::addEmptyLines(db, DB_NAME, total_lines); - sqlite3_close(db); + if (!finishSearchDatabase(db, insert_stmt)) + std::cerr << "Couldn't publish search database" << std::endl; return; } @@ -160,23 +304,29 @@ void performSearch(Display* display, string keyword) status_main = totalTextMessage(total); updateDisplay(display, status_main, status_sub); - db::insertRom(db, DB_NAME, { - .label = label, - .path = rom_path, - .imgpath = rom_path, - .type = 1 - }); + if (!insert({ + .label = label, + .path = rom_path, + .imgpath = rom_path, + .type = 1 + })) { + abortSearchDatabase(db, insert_stmt); + return; + } total_lines++; for (auto &entry : result) { string path = launch_cmd + ":" + entry.path; - db::insertRom(db, DB_NAME, { - .label = entry.label, - .path = path, - .imgpath = entry.imgpath, - .type = 0, - .ppath = label - }); + if (!insert({ + .label = entry.label, + .path = path, + .imgpath = entry.imgpath, + .type = 0, + .ppath = label + })) { + abortSearchDatabase(db, insert_stmt); + return; + } } } @@ -184,37 +334,50 @@ void performSearch(Display* display, string keyword) string all_label = "All systems (" + to_string(total) + ")"; - db::insertRom(db, DB_NAME, { - .label = all_label, - .path = "", - .imgpath = "", - .type = 1 - }); + if (!insert({ + .label = all_label, + .path = "", + .imgpath = "", + .type = 1 + })) { + abortSearchDatabase(db, insert_stmt); + return; + } total_lines++; if (total == 0) { - db::insertRom(db, DB_NAME, { - .label = "No results", - .path = "clear", - .imgpath = "/mnt/SDCARD/.tmp_update/res/help_clear_search.png", - .type = 0, - .ppath = all_label - }); + if (!insert({ + .label = "No results", + .path = "clear", + .imgpath = "/mnt/SDCARD/.tmp_update/res/help_clear_search.png", + .type = 0, + .ppath = all_label + })) { + abortSearchDatabase(db, insert_stmt); + return; + } } else { - // Copy all rows of type=0 and change ppath -> all_label - db::duplicateResults(db, DB_NAME, all_label); + // Copy all rows of type=0 and change ppath -> all_label. This stays + // inside the same transaction as the prepared row inserts. + if (!execSearchSql(db, db::sql::dupChangePpath(DB_NAME, all_label))) { + abortSearchDatabase(db, insert_stmt); + return; + } } if (missing_caches.size() > 0) { string cache_missing_label = "~Missing caches (" + to_string(missing_caches.size()) + ")"; - db::insertRom(db, DB_NAME, { - .label = cache_missing_label, - .path = "", - .imgpath = "", - .type = 1 - }); + if (!insert({ + .label = cache_missing_label, + .path = "", + .imgpath = "", + .type = 1 + })) { + abortSearchDatabase(db, insert_stmt); + return; + } total_lines++; for (auto &config : missing_caches) { @@ -223,36 +386,46 @@ void performSearch(Display* display, string keyword) if (dirname(emu_path) == "/mnt/SDCARD/RApp") label += " [Expert]"; - db::insertRom(db, DB_NAME, { - .label = label, - .path = "setstate:" + emu_path + ":" + config.label, - .imgpath = "/mnt/SDCARD/.tmp_update/res/help_unavailable.png", - .type = 0, - .ppath = cache_missing_label - }); + if (!insert({ + .label = label, + .path = "setstate:" + emu_path + ":" + config.label, + .imgpath = "/mnt/SDCARD/.tmp_update/res/help_unavailable.png", + .type = 0, + .ppath = cache_missing_label + })) { + abortSearchDatabase(db, insert_stmt); + return; + } } } // addTools(db); total_lines++; - db::insertRom(db, DB_NAME, { - .label = "Clear search", - .path = "clear", - .imgpath = "/mnt/SDCARD/.tmp_update/res/help_clear_search.png" - }); + if (!insert({ + .label = "Clear search", + .path = "clear", + .imgpath = "/mnt/SDCARD/.tmp_update/res/help_clear_search.png" + })) { + abortSearchDatabase(db, insert_stmt); + return; + } total_lines++; - db::insertRom(db, DB_NAME, { - .label = "Search: " + keyword, - .path = "search", - .imgpath = DB_DIR + "/Imgs/Enter search term....png" - }); + if (!insert({ + .label = "Search: " + keyword, + .path = "search", + .imgpath = DB_DIR + "/Imgs/Enter search term....png" + })) { + abortSearchDatabase(db, insert_stmt); + return; + } total_lines++; // db::addEmptyLines(db, DB_NAME, total_lines); - sqlite3_close(db); + if (!finishSearchDatabase(db, insert_stmt)) + std::cerr << "Couldn't publish search database" << std::endl; } #endif // SEARCH_HPP__