diff --git a/objdiff-core/src/arch/mips.rs b/objdiff-core/src/arch/mips.rs
index f0355bb..fa69c64 100644
--- a/objdiff-core/src/arch/mips.rs
+++ b/objdiff-core/src/arch/mips.rs
@@ -1,7 +1,7 @@
 use std::borrow::Cow;
 
 use anyhow::{bail, Result};
-use object::{elf, Endian, Endianness, File, Object, Relocation, RelocationFlags, SectionIndex};
+use object::{elf, Endian, Endianness, File, Object, Relocation, RelocationFlags};
 use rabbitizer::{config, Abi, InstrCategory, Instruction, OperandType};
 
 use crate::{
@@ -38,9 +38,6 @@ impl ObjArch for ObjArchMips {
         let code = &section.data
             [symbol.section_address as usize..(symbol.section_address + symbol.size) as usize];
 
-        let line_info =
-            obj.line_info.as_ref().and_then(|map| map.get(&SectionIndex(section.orig_index)));
-
         let start_address = symbol.address;
         let end_address = symbol.address + symbol.size;
         let ins_count = code.len() / 4;
@@ -114,8 +111,10 @@ impl ObjArch for ObjArchMips {
                     }
                 }
             }
-            let line =
-                line_info.and_then(|map| map.range(..=cur_addr as u64).last().map(|(_, &b)| b));
+            let line = section
+                .line_info
+                .as_ref()
+                .and_then(|map| map.range(..=cur_addr as u64).last().map(|(_, &b)| b));
             insts.push(ObjIns {
                 address: cur_addr as u64,
                 size: 4,
diff --git a/objdiff-core/src/arch/ppc.rs b/objdiff-core/src/arch/ppc.rs
index f657787..6b21ea4 100644
--- a/objdiff-core/src/arch/ppc.rs
+++ b/objdiff-core/src/arch/ppc.rs
@@ -1,7 +1,7 @@
 use std::borrow::Cow;
 
 use anyhow::{bail, Result};
-use object::{elf, File, Relocation, RelocationFlags, SectionIndex};
+use object::{elf, File, Relocation, RelocationFlags};
 use ppc750cl::{Argument, InsIter, GPR};
 
 use crate::{
@@ -39,9 +39,6 @@ impl ObjArch for ObjArchPpc {
         let code = &section.data
             [symbol.section_address as usize..(symbol.section_address + symbol.size) as usize];
 
-        let line_info =
-            obj.line_info.as_ref().and_then(|map| map.get(&SectionIndex(section.orig_index)));
-
         let ins_count = code.len() / 4;
         let mut ops = Vec::<u16>::with_capacity(ins_count);
         let mut insts = Vec::<ObjIns>::with_capacity(ins_count);
@@ -135,8 +132,10 @@ impl ObjArch for ObjArchPpc {
             }
 
             ops.push(ins.op as u16);
-            let line =
-                line_info.and_then(|map| map.range(..=cur_addr as u64).last().map(|(_, &b)| b));
+            let line = section
+                .line_info
+                .as_ref()
+                .and_then(|map| map.range(..=cur_addr as u64).last().map(|(_, &b)| b));
             insts.push(ObjIns {
                 address: cur_addr as u64,
                 size: 4,
diff --git a/objdiff-core/src/arch/x86.rs b/objdiff-core/src/arch/x86.rs
index db961e0..cd65690 100644
--- a/objdiff-core/src/arch/x86.rs
+++ b/objdiff-core/src/arch/x86.rs
@@ -6,7 +6,7 @@ use iced_x86::{
     GasFormatter, Instruction, IntelFormatter, MasmFormatter, NasmFormatter, NumberKind, OpKind,
     PrefixKind, Register,
 };
-use object::{pe, Endian, Endianness, File, Object, Relocation, RelocationFlags, SectionIndex};
+use object::{pe, Endian, Endianness, File, Object, Relocation, RelocationFlags};
 
 use crate::{
     arch::{ObjArch, ProcessCodeResult},
@@ -36,9 +36,6 @@ impl ObjArch for ObjArchX86 {
         let code = &section.data
             [symbol.section_address as usize..(symbol.section_address + symbol.size) as usize];
 
-        let line_info =
-            obj.line_info.as_ref().and_then(|map| map.get(&SectionIndex(section.orig_index)));
-
         let mut result = ProcessCodeResult { ops: Vec::new(), insts: Vec::new() };
         let mut decoder = Decoder::with_ip(self.bits, code, symbol.address, DecoderOptions::NONE);
         let mut formatter: Box<dyn Formatter> = match config.x86_formatter {
@@ -76,6 +73,7 @@ impl ObjArch for ObjArchX86 {
                 .relocations
                 .iter()
                 .find(|r| r.address >= address && r.address < address + instruction.len() as u64);
+            let line = section.line_info.as_ref().and_then(|m| m.get(&address).cloned());
             output.ins = ObjIns {
                 address,
                 size: instruction.len() as u8,
@@ -84,7 +82,7 @@ impl ObjArch for ObjArchX86 {
                 args: vec![],
                 reloc: reloc.cloned(),
                 branch_dest: None,
-                line: line_info.and_then(|m| m.get(&address).cloned()),
+                line,
                 formatted: String::new(),
                 orig: None,
             };
diff --git a/objdiff-core/src/obj/mod.rs b/objdiff-core/src/obj/mod.rs
index df93d81..c8534c9 100644
--- a/objdiff-core/src/obj/mod.rs
+++ b/objdiff-core/src/obj/mod.rs
@@ -1,16 +1,11 @@
 pub mod read;
 pub mod split_meta;
 
-use std::{
-    borrow::Cow,
-    collections::{BTreeMap, HashMap},
-    fmt,
-    path::PathBuf,
-};
+use std::{borrow::Cow, collections::BTreeMap, fmt, path::PathBuf};
 
 use filetime::FileTime;
 use flagset::{flags, FlagSet};
-use object::{RelocationFlags, SectionIndex};
+use object::RelocationFlags;
 use split_meta::SplitMeta;
 
 use crate::{arch::ObjArch, util::ReallySigned};
@@ -44,6 +39,8 @@ pub struct ObjSection {
     pub symbols: Vec<ObjSymbol>,
     pub relocations: Vec<ObjReloc>,
     pub virtual_address: Option<u64>,
+    /// Line number info (.line or .debug_line section)
+    pub line_info: Option<BTreeMap<u64, u64>>,
 }
 
 #[derive(Debug, Clone, Eq, PartialEq)]
@@ -133,8 +130,6 @@ pub struct ObjInfo {
     pub sections: Vec<ObjSection>,
     /// Common BSS symbols
     pub common: Vec<ObjSymbol>,
-    /// Line number info (.line or .debug_line section)
-    pub line_info: Option<HashMap<SectionIndex, BTreeMap<u64, u64>>>,
     /// Split object metadata (.note.split section)
     pub split_meta: Option<SplitMeta>,
 }
diff --git a/objdiff-core/src/obj/read.rs b/objdiff-core/src/obj/read.rs
index 6bc3450..c89c03e 100644
--- a/objdiff-core/src/obj/read.rs
+++ b/objdiff-core/src/obj/read.rs
@@ -1,9 +1,4 @@
-use std::{
-    collections::{BTreeMap, HashMap},
-    fs,
-    io::Cursor,
-    path::Path,
-};
+use std::{fs, io::Cursor, path::Path};
 
 use anyhow::{anyhow, bail, ensure, Context, Result};
 use byteorder::{BigEndian, ReadBytesExt};
@@ -116,6 +111,7 @@ fn filter_sections(obj_file: &File<'_>, split_meta: Option<&SplitMeta>) -> Resul
             symbols: Vec::new(),
             relocations: Vec::new(),
             virtual_address,
+            line_info: None,
         });
     }
     result.sort_by(|a, b| a.name.cmp(&b.name));
@@ -273,46 +269,47 @@ fn relocations_by_section(
     Ok(relocations)
 }
 
-fn line_info(obj_file: &File<'_>) -> Result<Option<HashMap<SectionIndex, BTreeMap<u64, u64>>>> {
-    let mut map = HashMap::new();
-
+fn line_info(obj_file: &File<'_>, sections: &mut [ObjSection]) -> Result<()> {
     // DWARF 1.1
     if let Some(section) = obj_file.section_by_name(".line") {
-        if section.size() == 0 {
-            return Ok(None);
-        }
-        let text_section = obj_file
-            .sections()
-            .find(|s| s.kind() == SectionKind::Text)
-            .context("No text section found for line info")?;
-        let mut lines = BTreeMap::new();
-
         let data = section.uncompressed_data()?;
         let mut reader = Cursor::new(data.as_ref());
 
-        let size = reader.read_u32::<BigEndian>()?;
-        let base_address = reader.read_u32::<BigEndian>()? as u64;
-        while reader.position() < size as u64 {
-            let line_number = reader.read_u32::<BigEndian>()? as u64;
-            let statement_pos = reader.read_u16::<BigEndian>()?;
-            if statement_pos != 0xFFFF {
-                log::warn!("Unhandled statement pos {}", statement_pos);
+        let mut text_sections = obj_file.sections().filter(|s| s.kind() == SectionKind::Text);
+        while reader.position() < data.len() as u64 {
+            let text_section_index = text_sections
+                .next()
+                .ok_or_else(|| anyhow!("Next text section not found for line info"))?
+                .index()
+                .0;
+            let start = reader.position();
+            let size = reader.read_u32::<BigEndian>()?;
+            let base_address = reader.read_u32::<BigEndian>()? as u64;
+            let Some(out_section) =
+                sections.iter_mut().find(|s| s.orig_index == text_section_index)
+            else {
+                // Skip line info for sections we filtered out
+                reader.set_position(start + size as u64);
+                continue;
+            };
+            let lines = out_section.line_info.get_or_insert_with(Default::default);
+            let end = start + size as u64;
+            while reader.position() < end {
+                let line_number = reader.read_u32::<BigEndian>()? as u64;
+                let statement_pos = reader.read_u16::<BigEndian>()?;
+                if statement_pos != 0xFFFF {
+                    log::warn!("Unhandled statement pos {}", statement_pos);
+                }
+                let address_delta = reader.read_u32::<BigEndian>()? as u64;
+                lines.insert(base_address + address_delta, line_number);
+                log::debug!("Line: {:#x} -> {}", base_address + address_delta, line_number);
             }
-            let address_delta = reader.read_u32::<BigEndian>()? as u64;
-            lines.insert(base_address + address_delta, line_number);
         }
-
-        map.insert(text_section.index(), lines);
     }
 
     // DWARF 2+
     #[cfg(feature = "dwarf")]
     {
-        let mut text_sections = obj_file.sections().filter(|s| s.kind() == SectionKind::Text);
-        let first_section = text_sections.next().context("No text section found for line info")?;
-        map.insert(first_section.index(), BTreeMap::new());
-        let mut lines = map.get_mut(&first_section.index()).unwrap();
-
         let dwarf_cow = gimli::DwarfSections::load(|id| {
             Ok::<_, gimli::Error>(
                 obj_file
@@ -327,32 +324,48 @@ fn line_info(obj_file: &File<'_>) -> Result<Option<HashMap<SectionIndex, BTreeMa
         };
         let dwarf = dwarf_cow.borrow(|section| gimli::EndianSlice::new(section, endian));
         let mut iter = dwarf.units();
-        'outer: while let Some(header) = iter.next()? {
+        if let Some(header) = iter.next()? {
             let unit = dwarf.unit(header)?;
             if let Some(program) = unit.line_program.clone() {
+                let mut text_sections =
+                    obj_file.sections().filter(|s| s.kind() == SectionKind::Text);
+                let section_index = text_sections
+                    .next()
+                    .ok_or_else(|| anyhow!("Next text section not found for line info"))?
+                    .index()
+                    .0;
+                let mut lines = sections
+                    .iter_mut()
+                    .find(|s| s.orig_index == section_index)
+                    .map(|s| s.line_info.get_or_insert_with(Default::default));
+
                 let mut rows = program.rows();
                 while let Some((_header, row)) = rows.next_row()? {
-                    if let Some(line) = row.line() {
+                    if let (Some(line), Some(lines)) = (row.line(), &mut lines) {
                         lines.insert(row.address(), line.get());
                     }
                     if row.end_sequence() {
                         // The next row is the start of a new sequence, which means we must
                         // advance to the next .text section.
-                        if let Some(next_section) = text_sections.next() {
-                            map.insert(next_section.index(), BTreeMap::new());
-                            lines = map.get_mut(&next_section.index()).unwrap();
-                        } else {
-                            break 'outer;
-                        }
+                        let section_index = text_sections
+                            .next()
+                            .ok_or_else(|| anyhow!("Next text section not found for line info"))?
+                            .index()
+                            .0;
+                        lines = sections
+                            .iter_mut()
+                            .find(|s| s.orig_index == section_index)
+                            .map(|s| s.line_info.get_or_insert_with(Default::default));
                     }
                 }
             }
         }
+        if iter.next()?.is_some() {
+            log::warn!("Multiple units found in DWARF data, only processing the first");
+        }
     }
-    if map.is_empty() {
-        return Ok(None);
-    }
-    Ok(Some(map))
+
+    Ok(())
 }
 
 pub fn read(obj_path: &Path) -> Result<ObjInfo> {
@@ -371,16 +384,9 @@ pub fn read(obj_path: &Path) -> Result<ObjInfo> {
         section.relocations =
             relocations_by_section(arch.as_ref(), &obj_file, section, split_meta.as_ref())?;
     }
+    line_info(&obj_file, &mut sections)?;
     let common = common_symbols(arch.as_ref(), &obj_file, split_meta.as_ref())?;
-    Ok(ObjInfo {
-        arch,
-        path: obj_path.to_owned(),
-        timestamp,
-        sections,
-        common,
-        line_info: line_info(&obj_file)?,
-        split_meta,
-    })
+    Ok(ObjInfo { arch, path: obj_path.to_owned(), timestamp, sections, common, split_meta })
 }
 
 pub fn has_function(obj_path: &Path, symbol_name: &str) -> Result<bool> {