Analyzer improvements for ProDG (WIP)

Fixes #19

src/analysis/cfa.rs

@@ -531,3 +531,50 @@ pub fn locate_sda_bases(obj: &mut ObjInfo) -> Result<bool> {
         None => Ok(false),
     }
 }
+
+/// ProDG hardcodes .bss and .sbss section initialization in `entry`
+/// This function locates the memset calls and returns a list of
+/// (address, size) pairs for the .bss sections.
+pub fn locate_bss_memsets(obj: &mut ObjInfo) -> Result<Vec<(u32, u32)>> {
+    let mut bss_sections: Vec<(u32, u32)> = Vec::new();
+    let Some(entry) = obj.entry else {
+        return Ok(bss_sections);
+    };
+    let (section_index, _) = obj
+        .sections
+        .at_address(entry as u32)
+        .context(format!("Entry point {:#010X} outside of any section", entry))?;
+    let entry_addr = SectionAddress::new(section_index, entry as u32);
+
+    let mut executor = Executor::new(obj);
+    executor.push(entry_addr, VM::new(), false);
+    executor.run(
+        obj,
+        |ExecCbData { executor: _, vm, result, ins_addr: _, section: _, ins, block_start: _ }| {
+            match result {
+                StepResult::Continue | StepResult::LoadStore { .. } => Ok(ExecCbResult::Continue),
+                StepResult::Illegal => bail!("Illegal instruction @ {:#010X}", ins.addr),
+                StepResult::Jump(_target) => Ok(ExecCbResult::End(())),
+                StepResult::Branch(branches) => {
+                    for branch in branches {
+                        if branch.link {
+                            // ProDG bug? Registers are supposed to start at r3
+                            if let (
+                                GprValue::Constant(addr),
+                                GprValue::Constant(value),
+                                GprValue::Constant(size),
+                            ) = (vm.gpr_value(4), vm.gpr_value(5), vm.gpr_value(6))
+                            {
+                                if value == 0 && size > 0 {
+                                    bss_sections.push((addr, size));
+                                }
+                            }
+                        }
+                    }
+                    Ok(ExecCbResult::Continue)
+                }
+            }
+        },
+    )?;
+    Ok(bss_sections)
+}

src/analysis/slices.rs

@@ -12,7 +12,7 @@ use crate::{
         disassemble,
         executor::{ExecCbData, ExecCbResult, Executor},
         uniq_jump_table_entries,
-        vm::{section_address_for, BranchTarget, StepResult, VM},
+        vm::{section_address_for, BranchTarget, GprValue, StepResult, VM},
         RelocationTarget,
     },
     obj::{ObjInfo, ObjKind, ObjSection},
@@ -212,8 +212,13 @@ impl FunctionSlices {
         let ExecCbData { executor, vm, result, ins_addr, section, ins, block_start } = data;
 
         // Track discovered prologue(s) and epilogue(s)
-        self.check_prologue(section, ins_addr, ins)
+        // HACK: ProDG sometimes uses LR as a storage register for int-to-float conversions
-            .with_context(|| format!("While processing {:#010X}: {:#?}", function_start, self))?;
+        // To our heuristic, this looks like a prologue, so first check LR for the magic number.
+        if vm.lr != GprValue::Constant(0x43300000) {
+            self.check_prologue(section, ins_addr, ins).with_context(|| {
+                format!("While processing {:#010X}: {:#?} {:#?}", function_start, self, vm.gpr)
+            })?;
+        }
         self.check_epilogue(section, ins_addr, ins)
             .with_context(|| format!("While processing {:#010X}: {:#?}", function_start, self))?;
         if !self.has_conditional_blr && is_conditional_blr(ins) {

src/analysis/vm.rs

@@ -63,13 +63,13 @@ impl Gpr {
 }
 
 #[derive(Default, Debug, Clone, Eq, PartialEq)]
-struct Cr {
+pub struct Cr {
     /// The left-hand value of this comparison
-    left: GprValue,
+    pub left: GprValue,
     /// The right-hand value of this comparison
-    right: GprValue,
+    pub right: GprValue,
     /// Whether this comparison is signed
-    signed: bool,
+    pub signed: bool,
 }
 
 #[derive(Default, Debug, Clone, Eq, PartialEq)]
@@ -77,9 +77,11 @@ pub struct VM {
     /// General purpose registers
     pub gpr: [Gpr; 32],
     /// Condition registers
-    cr: [Cr; 8],
+    pub cr: [Cr; 8],
+    /// Link register
+    pub lr: GprValue,
     /// Count register
-    ctr: GprValue,
+    pub ctr: GprValue,
 }
 
 impl VM {
@@ -277,6 +279,27 @@ impl VM {
                     }
                 }
             }
+            // subf rD, rA, rB
+            // subfc rD, rA, rB
+            Opcode::Subf | Opcode::Subfc => {
+                self.gpr[ins.field_rD()].set_direct(
+                    match (self.gpr[ins.field_rA()].value, self.gpr[ins.field_rB()].value) {
+                        (GprValue::Constant(left), GprValue::Constant(right)) => {
+                            GprValue::Constant((!left).wrapping_add(right).wrapping_add(1))
+                        }
+                        _ => GprValue::Unknown,
+                    },
+                );
+            }
+            // subfic rD, rA, SIMM
+            Opcode::Subfic => {
+                self.gpr[ins.field_rD()].set_direct(match self.gpr[ins.field_rA()].value {
+                    GprValue::Constant(value) => GprValue::Constant(
+                        (!value).wrapping_add(ins.field_simm() as u32).wrapping_add(1),
+                    ),
+                    _ => GprValue::Unknown,
+                });
+            }
             // ori rA, rS, UIMM
             Opcode::Ori => {
                 if let Some(target) =
@@ -472,19 +495,17 @@ impl VM {
                 self.gpr[ins.field_rD()].set_direct(value);
             }
             // mtspr SPR, rS
-            Opcode::Mtspr => {
+            Opcode::Mtspr => match ins.field_spr() {
-                if ins.field_spr() == 9 {
+                8 => self.lr = self.gpr[ins.field_rS()].value,
-                    // CTR
+                9 => self.ctr = self.gpr[ins.field_rS()].value,
-                    self.ctr = self.gpr[ins.field_rS()].value;
+                _ => {}
-                }
+            },
-            }
             // mfspr rD, SPR
             Opcode::Mfspr => {
-                let value = if ins.field_spr() == 9 {
+                let value = match ins.field_spr() {
-                    // CTR
+                    8 => self.lr,
-                    self.ctr
+                    9 => self.ctr,
-                } else {
+                    _ => GprValue::Unknown,
-                    GprValue::Unknown
                 };
                 self.gpr[ins.field_rD()].set_direct(value);
             }

src/util/dol.rs

@@ -7,7 +7,7 @@ use std::{
 use anyhow::{anyhow, bail, ensure, Result};
 
 use crate::{
-    analysis::cfa::{locate_sda_bases, SectionAddress},
+    analysis::cfa::{locate_bss_memsets, locate_sda_bases, SectionAddress},
     array_ref,
     obj::{
         ObjArchitecture, ObjInfo, ObjKind, ObjSection, ObjSectionKind, ObjSymbol, ObjSymbolFlagSet,
@@ -467,6 +467,70 @@ pub fn process_dol(buf: &[u8], name: &str) -> Result<ObjInfo> {
             });
         }
 
+        // ProDG: Locate BSS sections by analyzing the entry point
+        if bss_sections.is_empty() {
+            // Create temporary object
+            let mut temp_sections = sections.clone();
+            temp_sections.push(ObjSection {
+                name: ".bss".to_string(),
+                kind: ObjSectionKind::Bss,
+                address: bss_section.address as u64,
+                size: bss_section.size as u64,
+                data: vec![],
+                align: 0,
+                elf_index: 0,
+                relocations: Default::default(),
+                original_address: 0,
+                file_offset: 0,
+                section_known: false,
+                splits: Default::default(),
+            });
+            let mut obj = ObjInfo::new(
+                ObjKind::Executable,
+                ObjArchitecture::PowerPc,
+                name.to_string(),
+                vec![],
+                temp_sections,
+            );
+            obj.entry = Some(dol.entry_point() as u64);
+            let bss_sections = locate_bss_memsets(&mut obj)?;
+            match bss_sections.len() {
+                0 => log::warn!("Failed to locate BSS sections"),
+                2 => {
+                    // .bss and .sbss
+                    sections.push(ObjSection {
+                        name: ".bss".to_string(),
+                        kind: ObjSectionKind::Bss,
+                        address: bss_sections[0].0 as u64,
+                        size: bss_sections[0].1 as u64,
+                        data: vec![],
+                        align: 0,
+                        elf_index: 0,
+                        relocations: Default::default(),
+                        original_address: 0,
+                        file_offset: 0,
+                        section_known: false,
+                        splits: Default::default(),
+                    });
+                    sections.push(ObjSection {
+                        name: ".sbss".to_string(),
+                        kind: ObjSectionKind::Bss,
+                        address: bss_sections[1].0 as u64,
+                        size: bss_sections[1].1 as u64,
+                        data: vec![],
+                        align: 0,
+                        elf_index: 0,
+                        relocations: Default::default(),
+                        original_address: 0,
+                        file_offset: 0,
+                        section_known: false,
+                        splits: Default::default(),
+                    });
+                }
+                n => bail!("Invalid number of BSS sections: {}", n),
+            }
+        }
+
         // Sort sections by address ascending
         sections.sort_by_key(|s| s.address);
     }