diff --git a/objdiff-core/src/arch/mips.rs b/objdiff-core/src/arch/mips.rs
index fa69c64..109d0bb 100644
--- a/objdiff-core/src/arch/mips.rs
+++ b/objdiff-core/src/arch/mips.rs
@@ -111,10 +111,7 @@ impl ObjArch for ObjArchMips {
                     }
                 }
             }
-            let line = section
-                .line_info
-                .as_ref()
-                .and_then(|map| map.range(..=cur_addr as u64).last().map(|(_, &b)| b));
+            let line = section.line_info.range(..=cur_addr as u64).last().map(|(_, &b)| b);
             insts.push(ObjIns {
                 address: cur_addr as u64,
                 size: 4,
diff --git a/objdiff-core/src/arch/ppc.rs b/objdiff-core/src/arch/ppc.rs
index 6b21ea4..d167c59 100644
--- a/objdiff-core/src/arch/ppc.rs
+++ b/objdiff-core/src/arch/ppc.rs
@@ -132,10 +132,7 @@ impl ObjArch for ObjArchPpc {
             }
 
             ops.push(ins.op as u16);
-            let line = section
-                .line_info
-                .as_ref()
-                .and_then(|map| map.range(..=cur_addr as u64).last().map(|(_, &b)| b));
+            let line = section.line_info.range(..=cur_addr as u64).last().map(|(_, &b)| b);
             insts.push(ObjIns {
                 address: cur_addr as u64,
                 size: 4,
diff --git a/objdiff-core/src/arch/x86.rs b/objdiff-core/src/arch/x86.rs
index cd65690..f2fb028 100644
--- a/objdiff-core/src/arch/x86.rs
+++ b/objdiff-core/src/arch/x86.rs
@@ -73,7 +73,7 @@ impl ObjArch for ObjArchX86 {
                 .relocations
                 .iter()
                 .find(|r| r.address >= address && r.address < address + instruction.len() as u64);
-            let line = section.line_info.as_ref().and_then(|m| m.get(&address).cloned());
+            let line = section.line_info.range(..=address).last().map(|(_, &b)| b);
             output.ins = ObjIns {
                 address,
                 size: instruction.len() as u8,
diff --git a/objdiff-core/src/obj/mod.rs b/objdiff-core/src/obj/mod.rs
index c8534c9..59255ba 100644
--- a/objdiff-core/src/obj/mod.rs
+++ b/objdiff-core/src/obj/mod.rs
@@ -40,7 +40,7 @@ pub struct ObjSection {
     pub relocations: Vec<ObjReloc>,
     pub virtual_address: Option<u64>,
     /// Line number info (.line or .debug_line section)
-    pub line_info: Option<BTreeMap<u64, u64>>,
+    pub line_info: BTreeMap<u64, u64>,
 }
 
 #[derive(Debug, Clone, Eq, PartialEq)]
diff --git a/objdiff-core/src/obj/read.rs b/objdiff-core/src/obj/read.rs
index c89c03e..8462da8 100644
--- a/objdiff-core/src/obj/read.rs
+++ b/objdiff-core/src/obj/read.rs
@@ -111,7 +111,7 @@ fn filter_sections(obj_file: &File<'_>, split_meta: Option<&SplitMeta>) -> Resul
             symbols: Vec::new(),
             relocations: Vec::new(),
             virtual_address,
-            line_info: None,
+            line_info: Default::default(),
         });
     }
     result.sort_by(|a, b| a.name.cmp(&b.name));
@@ -292,7 +292,6 @@ fn line_info(obj_file: &File<'_>, sections: &mut [ObjSection]) -> Result<()> {
                 reader.set_position(start + size as u64);
                 continue;
             };
-            let lines = out_section.line_info.get_or_insert_with(Default::default);
             let end = start + size as u64;
             while reader.position() < end {
                 let line_number = reader.read_u32::<BigEndian>()? as u64;
@@ -301,7 +300,7 @@ fn line_info(obj_file: &File<'_>, sections: &mut [ObjSection]) -> Result<()> {
                     log::warn!("Unhandled statement pos {}", statement_pos);
                 }
                 let address_delta = reader.read_u32::<BigEndian>()? as u64;
-                lines.insert(base_address + address_delta, line_number);
+                out_section.line_info.insert(base_address + address_delta, line_number);
                 log::debug!("Line: {:#x} -> {}", base_address + address_delta, line_number);
             }
         }
@@ -337,7 +336,7 @@ fn line_info(obj_file: &File<'_>, sections: &mut [ObjSection]) -> Result<()> {
                 let mut lines = sections
                     .iter_mut()
                     .find(|s| s.orig_index == section_index)
-                    .map(|s| s.line_info.get_or_insert_with(Default::default));
+                    .map(|s| &mut s.line_info);
 
                 let mut rows = program.rows();
                 while let Some((_header, row)) = rows.next_row()? {
@@ -355,7 +354,7 @@ fn line_info(obj_file: &File<'_>, sections: &mut [ObjSection]) -> Result<()> {
                         lines = sections
                             .iter_mut()
                             .find(|s| s.orig_index == section_index)
-                            .map(|s| s.line_info.get_or_insert_with(Default::default));
+                            .map(|s| &mut s.line_info);
                     }
                 }
             }