From a4e626777f4d97148e49bf36a9de9d9f89cda7a7 Mon Sep 17 00:00:00 2001 From: wowwheaties <298423159+wowwheaties@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:13:43 -0400 Subject: [PATCH] =?UTF-8?q?=E2=98=80=EF=B8=8F=2067%=20=E2=80=93=20score=20?= =?UTF-8?q?registered=20owners=20inside=20retained=20modules=20from=20lift?= =?UTF-8?q?er=20batch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `lifter batch` scores each retained module at its extent, but many modules hold more than one registered owner (regions.json manual_regions). Scored at the owner's own span, the unchanged draft often links exact already. - `--owner-spans`: after the module pass, score every registered owner whose start lies inside the module at that owner's span, one row per owner, with an `owner_spans=` summary line (`list --owner-spans` lists them). - `--sweep`: when the owner span is not exact, retry with the span widened to the rest of the module and report both, for owners registered short of the pool island their function really runs to. - trailing pad: a reference one halfword longer than the candidate with no wrong instruction is ranked exact and marked `trailing_pad=1` with the raw count kept in the row; `adopt` still decides on the real bytes. Default output is unchanged without the flags. On this tree `batch --all --owner-spans --sweep` reports 273 owner spans in 187 modules, 47 of them exact from the unchanged draft. Co-Authored-By: Claude Fable 5.1 --- tools/lifter/src/main.rs | 209 ++++++++++++++++++++++++++++++++++++- tools/lifter/src/owners.rs | 50 +++++++++ 2 files changed, 255 insertions(+), 4 deletions(-) diff --git a/tools/lifter/src/main.rs b/tools/lifter/src/main.rs index e271d456b..e006761c8 100644 --- a/tools/lifter/src/main.rs +++ b/tools/lifter/src/main.rs @@ -1,4 +1,4 @@ -use lifter::owners::{self, first_error, score, score_extending, Module}; +use lifter::owners::{self, first_error, score, score_extending, Module, RegisteredOwner}; use lifter::{lift_owner, tune}; use std::path::{Path, PathBuf}; use std::process::ExitCode; @@ -10,11 +10,21 @@ const USAGE: &str = "usage: lifter [args]\n\ lift, compile, and compare against the ROM bytes\n\ tune : [--span BYTES] [--name NAME]\n\ score, then try call respellings line by line and keep improvements\n\ - batch [--kind WORD] [--limit N] [--all]\n\ + batch [--kind WORD] [--limit N] [--all] [--owner-spans] [--sweep]\n\ score every unregistered retained module (scene and script kinds)\n\ - list [--kind WORD] [--all]\n\ + list [--kind WORD] [--all] [--owner-spans]\n\ print the retained modules the batch would visit\n\ -Candidates are written under out/lifter/."; +Candidates are written under out/lifter/.\n\n\ + --owner-spans also score (list: also list) every registered owner\n\ + (regions.json manual_regions) whose start lies inside the\n\ + module, at that owner's own span, in addition to the\n\ + module-extent score. Default behaviour is unchanged when\n\ + this is absent. batch only.\n\ + --sweep with --owner-spans on batch: also retry each owner-span\n\ + score with the span widened to the rest of the module, and\n\ + report the better of the two (a short registered span can\n\ + cut a function that really runs to the next owner or the\n\ + module end)."; struct Options { positional: Vec, @@ -29,6 +39,8 @@ struct Options { source: Option, only: Option, jobs: Option, + owner_spans: bool, + sweep: bool, } fn parse(arguments: &[String]) -> Result { @@ -45,6 +57,8 @@ fn parse(arguments: &[String]) -> Result { source: None, only: None, jobs: None, + owner_spans: false, + sweep: false, }; let mut iter = arguments.iter(); while let Some(argument) = iter.next() { @@ -77,6 +91,8 @@ fn parse(arguments: &[String]) -> Result { ) } "--all" => options.all = true, + "--owner-spans" => options.owner_spans = true, + "--sweep" => options.sweep = true, "--path" => options.path = Some(value("--path")?), "--source" => options.source = Some(PathBuf::from(value("--source")?)), "--only" => options.only = Some(PathBuf::from(value("--only")?)), @@ -223,9 +239,25 @@ fn selected(root: &Path, options: &Options) -> Result, String> { Ok(modules) } +/// The registered owners (from `regions.json`) whose start lies inside `m`. +fn owners_inside<'a>(m: &Module, registered: &'a [RegisteredOwner]) -> Vec<&'a RegisteredOwner> { + registered + .iter() + .filter(|o| o.overlay == m.overlay && o.entry >= m.entry && o.entry < m.entry + m.span) + .collect() +} + fn list(root: &Path, options: &Options) -> Result<(), String> { + let registered = if options.owner_spans { + owners::registered_owners(root)? + } else { + Vec::new() + }; for m in selected(root, options)? { println!("{} {} {}", m.key(), m.span, m.kind); + for owner in owners_inside(&m, ®istered) { + println!(" {} span={}", owner.key(), owner.span); + } } Ok(()) } @@ -314,6 +346,172 @@ fn batch_one(root: &Path, options: &Options, m: &Module) -> Outcome { } } +/// True when the reference is exactly one halfword longer than the candidate +/// and the scorer reports no wrong instruction: every instruction matches and +/// the registered span merely includes the pad word after the final return +/// that ELF `.size` drops. The raw count stays visible in the row; `adopt` +/// still decides on the real bytes. +fn trailing_pad(result: &owners::Score) -> bool { + result.differing == 1 + && result.reference == result.candidate + 2 + && result.report.contains("wrong_instructions=0") +} + +/// One span's score, with the trailing-pad tolerance already applied. +struct Scored { + span: u32, + raw: u32, + ranked: u32, + pad: bool, +} + +fn describe(span: u32, result: &owners::Score) -> Scored { + let pad = trailing_pad(result); + Scored { + span, + raw: result.differing, + ranked: if pad { 0 } else { result.differing }, + pad, + } +} + +/// Renders one `Scored` as `{prefix}differing_halfwords=N[ {prefix}trailing_pad=1 +/// {prefix}raw_differing_halfwords=N]`, `prefix` distinguishing an owner-span +/// score from its `--sweep` retry on the same line. +fn format_scored(prefix: &str, s: &Scored) -> String { + let mut out = format!("{prefix}differing_halfwords={}", s.ranked); + if s.pad { + out.push_str(&format!( + " {prefix}trailing_pad=1 {prefix}raw_differing_halfwords={}", + s.raw + )); + } + out +} + +/// One registered-owner span's batch result, ranking already applying the +/// trailing-pad tolerance. +enum OwnerOutcome { + Scored { ranked: u32 }, + Failed, +} + +/// Scores one registered owner against a module's draft at the owner's own +/// span (`--owner-spans`), and optionally again at the rest of the module +/// (`--sweep`) when the owner-span score is not exact — a short registered +/// span can cut a function that really runs to the next owner or the module +/// end (see HANDOFF's "reverse case": a pool-island split registered as two +/// owners). +fn owner_span_one( + root: &Path, + options: &Options, + m: &Module, + path: &Path, + owner: &RegisteredOwner, +) -> OwnerOutcome { + let owner_key = owner.key(); + let result = match score(root, path, &owner_key, owner.span) { + Ok(result) => result, + Err(error) => { + let detail = first_error(&error).unwrap_or(error); + println!("{owner_key} span={} failed {}", owner.span, detail.trim()); + return OwnerOutcome::Failed; + } + }; + let scored = describe(owner.span, &result); + let module_note = format!("(module {:08x} extent {})", m.entry, m.span); + let swept_span = m.entry + m.span - owner.entry; + let sweep = options.sweep && scored.ranked > 0 && swept_span != owner.span; + if !sweep { + println!( + "{owner_key} span={} {} {module_note}", + owner.span, + format_scored("", &scored) + ); + return OwnerOutcome::Scored { + ranked: scored.ranked, + }; + } + match score(root, path, &owner_key, swept_span) { + Ok(swept_result) => { + let swept = describe(swept_span, &swept_result); + let best = if swept.ranked < scored.ranked { + "sweep" + } else { + "owner" + }; + println!( + "{owner_key} span={} {} {module_note} sweep_span={} {} best={best}", + scored.span, + format_scored("", &scored), + swept.span, + format_scored("sweep_", &swept) + ); + OwnerOutcome::Scored { + ranked: scored.ranked.min(swept.ranked), + } + } + Err(error) => { + let detail = first_error(&error).unwrap_or(error); + println!( + "{owner_key} span={} {} {module_note} sweep_span={swept_span} sweep_failed {}", + scored.span, + format_scored("", &scored), + detail.trim() + ); + OwnerOutcome::Scored { + ranked: scored.ranked, + } + } + } +} + +/// Runs the `--owner-spans` pass over every registered owner inside an +/// already-scored (not failed) module, printing one row per owner and +/// returning a summary tally. +fn owner_spans_pass(root: &Path, options: &Options, modules: &[Module], outcomes: &[Outcome]) { + let registered = match owners::registered_owners(root) { + Ok(registered) => registered, + Err(error) => { + println!("owner-spans failed {error}"); + return; + } + }; + let mut jobs_list: Vec<(Module, RegisteredOwner)> = Vec::new(); + for (m, outcome) in modules.iter().zip(outcomes) { + if matches!(outcome, Outcome::Failed) { + continue; + } + for owner in owners_inside(m, ®istered) { + jobs_list.push((m.clone(), owner.clone())); + } + } + if jobs_list.is_empty() { + return; + } + let results = parallel(&jobs_list, jobs(options), |(m, owner)| { + match scratch_path(root, &m.key()) { + Ok(path) => owner_span_one(root, options, m, &path, owner), + Err(error) => { + println!("{} span={} failed {error}", owner.key(), owner.span); + OwnerOutcome::Failed + } + } + }); + let (mut owner_exact, mut owner_differing, mut owner_failed) = (0, 0, 0); + for result in &results { + match result { + OwnerOutcome::Scored { ranked: 0 } => owner_exact += 1, + OwnerOutcome::Scored { .. } => owner_differing += 1, + OwnerOutcome::Failed => owner_failed += 1, + } + } + println!( + "owner_spans={} owner_exact={owner_exact} owner_differing={owner_differing} owner_failed={owner_failed}", + jobs_list.len() + ); +} + fn batch(root: &Path, options: &Options) -> Result<(), String> { let modules = selected(root, options)?; let outcomes = parallel(&modules, jobs(options), |m| batch_one(root, options, m)); @@ -345,6 +543,9 @@ fn batch(root: &Path, options: &Options) -> Result<(), String> { .join(" ") ); } + if options.owner_spans { + owner_spans_pass(root, options, &modules, &outcomes); + } Ok(()) } diff --git a/tools/lifter/src/owners.rs b/tools/lifter/src/owners.rs index 1d0a13708..a1b1a926c 100644 --- a/tools/lifter/src/owners.rs +++ b/tools/lifter/src/owners.rs @@ -69,6 +69,56 @@ pub fn modules(root: &Path) -> Result, String> { Ok(modules) } +#[derive(Deserialize)] +struct Regions { + manual_regions: Vec, +} + +#[derive(Deserialize)] +struct ManualRegion { + overlay: String, + entry: String, + span_bytes: u32, +} + +/// One entry from `regions.json`'s `manual_regions`: a function-sized owner +/// inside (or coinciding with) a retained module, whether or not it has been +/// adopted yet. +#[derive(Debug, Clone)] +pub struct RegisteredOwner { + pub overlay: String, + pub entry: u32, + pub span: u32, +} + +impl RegisteredOwner { + pub fn key(&self) -> String { + format!("{}:{:08x}", self.overlay, self.entry) + } +} + +/// Every registered owner in `games/gs1/semantic/regions.json`, regardless +/// of adoption state. Unlike `study`/`bench`, which only look at owners that +/// already have mapped source, this is for finding owners still sitting +/// inside a retained module's draft. +pub fn registered_owners(root: &Path) -> Result, String> { + let path = root.join("games/gs1/semantic/regions.json"); + let text = std::fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; + let regions: Regions = + serde_json::from_slice(&text).map_err(|error| format!("{}: {error}", path.display()))?; + regions + .manual_regions + .into_iter() + .map(|region| { + Ok(RegisteredOwner { + entry: parse_hex(®ion.entry)?, + overlay: region.overlay, + span: region.span_bytes, + }) + }) + .collect() +} + /// Parses `:` into its parts. pub fn parse_owner(owner: &str) -> Result<(String, u32), String> { let (overlay, address) = owner