Stripped file and gnu_debuglink support take 2 (#596)

* Stripped file and gnu_debuglink support take 2

* Docstring for _file_crc32

* Removed pointless FIXME

* _file_crc32 does not rewind

Author: sevaa

elftools/dwarf/dwarf_util.py

@@ -7,7 +7,7 @@
 # This code is in the public domain
 #-------------------------------------------------------------------------------
 
-import os
+import os, binascii
 from ..construct.macros import UBInt32, UBInt64, ULInt32, ULInt64, Array
 from ..common.exceptions import DWARFError
 from ..common.utils import preserve_stream_pos, struct_parse
@@ -60,3 +60,14 @@ def _iter_CUs_in_section(stream, structs, parser):
         yield header
         offset = header.offset_after_length + header.unit_length   
 
+def _file_crc32(file):
+    """ Provided a readable binary stream, reads the stream to the end
+        and computes the CRC32 checksum of its contents,
+        with the initial value of 0.
+    """
+    d = file.read(4096)
+    checksum = 0
+    while len(d):
+        checksum = binascii.crc32(d, checksum)
+        d = file.read(4096)
+    return checksum

elftools/dwarf/dwarfinfo.py

@@ -83,7 +83,6 @@ def __init__(self,
             debug_rnglists_sec,
             debug_sup_sec,
             gnu_debugaltlink_sec,
-            gnu_debuglink_sec,
             debug_types_sec
             ):
         """ config:
@@ -113,7 +112,6 @@ def __init__(self,
         self.debug_rnglists_sec = debug_rnglists_sec
         self.debug_sup_sec = debug_sup_sec
         self.gnu_debugaltlink_sec = gnu_debugaltlink_sec
-        self.gnu_debuglink_sec = gnu_debuglink_sec
         self.debug_types_sec = debug_types_sec
 
         # Sets the supplementary_dwarfinfo to None. Client code can set this
@@ -716,7 +714,7 @@ def replace_value(data, content_type, replacer):
 
     def parse_debugsupinfo(self):
         """
-        Extract a filename from .debug_sup, .gnu_debualtlink sections, or .gnu_debuglink.
+        Extract a filename from .debug_sup, .gnu_debualtlink sections.
         """
         if self.debug_sup_sec is not None:
             self.debug_sup_sec.stream.seek(0)
@@ -727,9 +725,9 @@ def parse_debugsupinfo(self):
             self.gnu_debugaltlink_sec.stream.seek(0)
             suplink = self.structs.Dwarf_debugaltlink.parse_stream(self.gnu_debugaltlink_sec.stream)
             return suplink.sup_filename
-        if self.gnu_debuglink_sec is not None:
-            self.gnu_debuglink_sec.stream.seek(0)
-            suplink = self.structs.Dwarf_debuglink.parse_stream(self.gnu_debuglink_sec.stream)
-            return suplink.sup_filename
+        # The section .gnu_debuglink with similarly looking contents
+        # has a different meaning - it doesn't point at supplementary DWARF,
+        # which is meant to be referenced from primary DWARF,
+        # it points at DWARF proper.
         return None
 

elftools/dwarf/structs.py

@@ -177,7 +177,6 @@ def _create_structs(self):
 
         self._create_debugsup()
         self._create_gnu_debugaltlink()
-        self._create_gnu_debuglink()
 
     def _create_initial_length(self):
         def _InitialLength(name):
@@ -275,18 +274,6 @@ def _create_gnu_debugaltlink(self):
             CString("sup_filename"),
             String("sup_checksum", length=20))
 
-    def _create_gnu_debuglink(self):
-        self.Dwarf_debuglink = Struct('Elf_debuglink',
-            CString("sup_filename"),
-            Switch('', lambda ctx: (len(ctx.sup_filename) % 4),
-                {
-                    0: String("sup_padding", length=3),
-                    1: String("sup_padding", length=2),
-                    2: String("sup_padding", length=1),
-                    3: String("sup_padding", length=0),
-                }),
-            String("sup_checksum", length=4))
-
     def _create_dw_form(self):
         self.Dwarf_dw_form = dict(
             DW_FORM_addr=self.the_Dwarf_target_addr,

elftools/elf/elffile.py

@@ -30,6 +30,7 @@
 from ..ehabi.ehabiinfo import EHABIInfo
 from .hash import ELFHashSection, GNUHashSection
 from .constants import SHN_INDICES
+from ..dwarf.dwarf_util import _file_crc32
 
 class ELFFile(object):
     """ Creation: the constructor accepts a stream (file-like object) with the
@@ -94,15 +95,24 @@ def load_from_path(cls, path):
         ELFFile from it, setting up a correct stream_loader relative to the
         original file.
         """
-        base_directory = os.path.dirname(path)
-        def loader(elf_path):
-            # FIXME: use actual path instead of str/bytes
-            if not os.path.isabs(elf_path):
-                elf_path = os.path.join(base_directory,
-                                        elf_path)
-            return open(elf_path, 'rb')
         stream = open(path, 'rb')
-        return ELFFile(stream, loader)
+        return ELFFile(stream, ELFFile.make_relative_loader(path))
+    
+    @staticmethod
+    def make_relative_loader(base_path):
+        """ Return a function that takes a potentially relative path,
+            resolves it against base_path (bytes or str), and opens a file at that.
+
+            ELFFile uses functions like that for resolving DWARF links.
+        """
+        if isinstance(base_path, str):
+            base_path = base_path.encode('UTF-8') # resolver takes a bytes path
+        base_directory = os.path.dirname(base_path)
+        def loader(rel_path):
+            if not os.path.isabs(rel_path):
+                rel_path = os.path.join(base_directory, rel_path)
+            return open(rel_path, 'rb')
+        return loader
 
     def num_sections(self):
         """ Number of sections in the file
@@ -172,6 +182,13 @@ def get_section_index(self, section_name):
         if self._section_name_map is None:
             self._make_section_name_map()
         return self._section_name_map.get(section_name, None)
+    
+    def has_section(self, section_name):
+        """ Section existence check by name, without the overhead of parsing if found.
+        """
+        if self._section_name_map is None:
+            self._make_section_name_map()
+        return section_name in self._section_name_map
 
     def iter_sections(self, type=None):
         """ Yield all the sections in the file. If the optional |type|
@@ -231,14 +248,18 @@ def address_offsets(self, start, size=1):
                 end <= seg['p_vaddr'] + seg['p_filesz']):
                 yield start - seg['p_vaddr'] + seg['p_offset']
 
-    def has_dwarf_info(self):
+    def has_dwarf_info(self, strict=False):
         """ Check whether this file appears to have debugging information.
             We assume that if it has the .debug_info or .zdebug_info section, it
             has all the other required sections as well.
+
+            Unless you pass strict=True, the presence of .eh_frame section,
+            which is DWARF adjacent but hardly DWARF proper, will count as debug info.
+            Stripped files contain .eh_frame but none of the .[z]debug_xxx sections.
         """
-        return bool(self.get_section_by_name('.debug_info') or
-            self.get_section_by_name('.zdebug_info') or
-            self.get_section_by_name('.eh_frame'))
+        return (self.has_section('.debug_info') or
+            self.has_section('.zdebug_info') or
+            (not strict and self.has_section('.eh_frame')))
 
     def get_dwarf_info(self, relocate_dwarf_sections=True, follow_links=True):
         """ Return a DWARFInfo object representing the debugging information in
@@ -247,23 +268,39 @@ def get_dwarf_info(self, relocate_dwarf_sections=True, follow_links=True):
             If relocate_dwarf_sections is True, relocations for DWARF sections
             are looked up and applied.
 
-            If follow_links is True, we will try to load the supplementary
+            If follow_links is True, we will try to load the external and/or supplementary
             object file (if any), and use it to resolve references and imports.
         """
-        # Expect that has_dwarf_info was called, so at least .debug_info is
+        # Expect that has_dwarf_info() was called, so at least .debug_info is
         # present.
         # Sections that aren't found will be passed as None to DWARFInfo.
 
+        # TODO: support linking by build ID
+        # https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html
+
+        # A file may contain a debug link but not be stripped, so check for debug_info just in case
+        debuglink_section = self.get_section_by_name('.gnu_debuglink')
+        if debuglink_section and not self.has_dwarf_info(True) and follow_links and self.stream_loader:
+            debuglink = struct_parse(self.structs.Gnu_debuglink, debuglink_section.stream, debuglink_section.header.sh_offset)
+            with self.stream_loader(debuglink.filename) as ext_file:
+                # Validate checksum...
+                if _file_crc32(ext_file) != debuglink.checksum:
+                    raise ELFError('The linked DWARF file does not match the checksum in the link.')
+                ext_file.seek(0, os.SEEK_SET)
+                ext_elffile = ELFFile(ext_file, self.stream_loader)
+                # Inheriting the stream loader like that might be wrong if the supplementary DWARF link in the other file
+                # is relative to the other file's directory as opposed to this file's directory.
+                return ext_elffile.get_dwarf_info(relocate_dwarf_sections=relocate_dwarf_sections, follow_links=True)
+       
         section_names = ('.debug_info', '.debug_aranges', '.debug_abbrev',
                          '.debug_str', '.debug_line', '.debug_frame',
                          '.debug_loc', '.debug_ranges', '.debug_pubtypes',
                          '.debug_pubnames', '.debug_addr',
                          '.debug_str_offsets', '.debug_line_str',
                          '.debug_loclists', '.debug_rnglists',
-                         '.debug_sup', '.gnu_debugaltlink', '.gnu_debuglink',
-                         '.debug_types')
+                         '.debug_sup', '.gnu_debugaltlink', '.debug_types')
 
-        compressed = bool(self.get_section_by_name('.zdebug_info'))
+        compressed = self.has_section('.zdebug_info')
         if compressed:
             section_names = tuple(map(lambda x: '.z' + x[1:], section_names))
 
@@ -275,7 +312,7 @@ def get_dwarf_info(self, relocate_dwarf_sections=True, follow_links=True):
          debug_loc_sec_name, debug_ranges_sec_name, debug_pubtypes_name,
          debug_pubnames_name, debug_addr_name, debug_str_offsets_name,
          debug_line_str_name, debug_loclists_sec_name, debug_rnglists_sec_name,
-         debug_sup_name, gnu_debugaltlink_name, gnu_debuglink, debug_types_sec_name,
+         debug_sup_name, gnu_debugaltlink_name, debug_types_sec_name,
          eh_frame_sec_name) = section_names
 
         debug_sections = {}
@@ -318,13 +355,23 @@ def get_dwarf_info(self, relocate_dwarf_sections=True, follow_links=True):
                 debug_rnglists_sec=debug_sections[debug_rnglists_sec_name],
                 debug_sup_sec=debug_sections[debug_sup_name],
                 gnu_debugaltlink_sec=debug_sections[gnu_debugaltlink_name],
-                gnu_debuglink_sec=debug_sections[gnu_debuglink],
                 debug_types_sec=debug_sections[debug_types_sec_name]
                 )
         if follow_links:
             dwarfinfo.supplementary_dwarfinfo = self.get_supplementary_dwarfinfo(dwarfinfo)
         return dwarfinfo
-
+    
+    def has_dwarf_link(self):
+        """ Whether the binary's debug info is in an
+            external file. Use get_dwarf_link to retrieve the path to it.
+        """
+        return self.has_section('.gnu_debuglink')
+    
+    def get_dwarf_link(self):
+        """ Read the .gnu_debuglink section, return an object with filename (as bytes) and checksum (as number) in it.
+        """
+        section = self.get_section_by_name('.gnu_debuglink')
+        return struct_parse(self.structs.Gnu_debuglink, section.stream, section.header.sh_offset) if section else None
 
     def get_supplementary_dwarfinfo(self, dwarfinfo):
         """

elftools/elf/structs.py

@@ -112,6 +112,7 @@ def create_advanced_structs(self, e_type=None, e_machine=None, e_ident_osabi=Non
         self._create_riscv_attributes()
         self._create_elf_hash()
         self._create_gnu_hash()
+        self._create_gnu_debuglink()
 
     #-------------------------------- PRIVATE --------------------------------#
 
@@ -562,3 +563,9 @@ def _create_gnu_hash(self):
                                self.Elf_word('bloom_shift'),
                                Array(lambda ctx: ctx['bloom_size'], self.Elf_xword('bloom')),
                                Array(lambda ctx: ctx['nbuckets'], self.Elf_word('buckets')))
+        
+    def _create_gnu_debuglink(self):
+        self.Gnu_debuglink = Struct('Gnu_debuglink',
+            CString("filename"),
+            Padding(lambda ctx: 3 - len(ctx.filename) % 4, strict=True),
+            self.Elf_word("checksum"))

test/test_debuglink.py

@@ -44,8 +44,8 @@ def subprograms_from_debuglink(self, elf: ELFFile) -> dict[str, (int, int)]:
         # Retrieve the subprograms from the DWARF info
         dwarf_info = elf.get_dwarf_info(follow_links=True, relocate_dwarf_sections=True)
 
-        if dwarf_info.supplementary_dwarfinfo:
-            for CU in dwarf_info.supplementary_dwarfinfo.iter_CUs():
+        if dwarf_info:
+            for CU in dwarf_info.iter_CUs():
                 for DIE in CU.iter_DIEs():
                     if DIE.tag == 'DW_TAG_subprogram':
                         attributes = DIE.attributes
@@ -70,9 +70,24 @@ def subprograms_from_debuglink(self, elf: ELFFile) -> dict[str, (int, int)]:
     def test_debuglink(self):
         with open('test/testfiles_for_unittests/debuglink', "rb") as elf_file:
             elf = ELFFile(elf_file, stream_loader=self.stream_loader)
+            # Contains eh_frame and gnu_debuglink, but no DWARF
+            self.assertTrue(elf.has_dwarf_info(False))
+            self.assertFalse(elf.has_dwarf_info(True))
+            self.assertTrue(elf.has_dwarf_link())
+
+            link = elf.get_dwarf_link()
+            self.assertIsNotNone(link)
+            self.assertEqual(link.filename, b'debuglink.debug')
+            self.assertEqual(link.checksum, 0x29b7c5f1)
+
             subprograms = self.subprograms_from_debuglink(elf)
             self.assertEqual(subprograms, {b'main': (0x1161, 0x52), b'addNumbers': (0x1149, 0x18)})
 
+        # Test the filesystem aware ELFFile loading
+        elf = ELFFile.load_from_path('test/testfiles_for_unittests/debuglink')
+        subprograms = self.subprograms_from_debuglink(elf)
+        self.assertEqual(subprograms, {b'main': (0x1161, 0x52), b'addNumbers': (0x1149, 0x18)})
+
 if __name__ == '__main__':
     unittest.main()
 

test/test_refaddr_bitness.py

@@ -51,7 +51,6 @@ def test_main(self):
             debug_rnglists_sec = None,
             debug_sup_sec = None,
             gnu_debugaltlink_sec = None,
-            gnu_debuglink_sec = None,
             debug_types_sec=None
         )