Quelle  ml.py   Sprache: Python

#!/usr/bin/env python
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Wraps ml.exe or ml64.exe and postprocesses the output to be deterministic.
Sets timestamp in .obj file to 0, hence incompatible with link.exe /incremental.

Use by prefixing the ml(64).exe invocation with this script:
    python ml.py ml.exe [args...]"""

import array
import collections
import struct
import subprocess
import sys


class Struct(object):
  """A thin wrapper around the struct module that returns a namedtuple"""
  def __init__(self, name, *args):
    """Pass the name of the return type, and then an interleaved list of
    format strings as used by the struct module and of field names."""
    self.fmt = '<' + ''.join(args[0::2])
    self.type = collections.namedtuple(name, args[1::2])

  def pack_into(self, buffer, offset, data):
    return struct.pack_into(self.fmt, buffer, offset, *data)

  def unpack_from(self, buffer, offset=0):
    return self.type(*struct.unpack_from(self.fmt, buffer, offset))

  def size(self):
    return struct.calcsize(self.fmt)


def Subtract(nt, **kwargs):
  """Subtract(nt, f=2) returns a new namedtuple with 2 subtracted from nt.f"""
  return nt._replace(**{k: getattr(nt, k) - v for k, v in kwargs.items()})


def MakeDeterministic(objdata):
  # Takes data produced by ml(64).exe (without any special flags) and
  # 1. Sets the timestamp to 0
  # 2. Strips the .debug$S section (which contains an unwanted absolute path)

  # This makes several assumptions about ml's output:
  # - Section data is in the same order as the corresponding section headers:
  #   section headers preceding the .debug$S section header have their data
  #   preceding the .debug$S section data; likewise for section headers
  #   following the .debug$S section.
  # - The .debug$S section contains only the absolute path to the obj file and
  #   nothing else, in particular there's only a single entry in the symbol
  #   table referring to the .debug$S section.
  # - There are no COFF line number entries.
  # - There's no IMAGE_SYM_CLASS_CLR_TOKEN symbol.
  # These seem to hold in practice; if they stop holding this script needs to
  # become smarter.

  objdata = array.array('b', objdata)  # Writable, e.g. via struct.pack_into.

  # Read coff header.
  COFFHEADER = Struct('COFFHEADER',
                      'H', 'Machine',
                      'H', 'NumberOfSections',
                      'I', 'TimeDateStamp',
                      'I', 'PointerToSymbolTable',
                      'I', 'NumberOfSymbols',

                      'H', 'SizeOfOptionalHeader',
                      'H', 'Characteristics')
  coff_header = COFFHEADER.unpack_from(objdata)
  assert coff_header.SizeOfOptionalHeader == 0  # Only set for binaries.

  # Read section headers following coff header.
  SECTIONHEADER = Struct('SECTIONHEADER',
                         '8s', 'Name',
                         'I', 'VirtualSize',
                         'I', 'VirtualAddress',

                         'I', 'SizeOfRawData',
                         'I', 'PointerToRawData',
                         'I', 'PointerToRelocations',
                         'I', 'PointerToLineNumbers',

                         'H', 'NumberOfRelocations',
                         'H', 'NumberOfLineNumbers',
                         'I', 'Characteristics')
  section_headers = []
  debug_section_index = -1
  for i in range(0, coff_header.NumberOfSections):
    section_header = SECTIONHEADER.unpack_from(
        objdata, offset=COFFHEADER.size() + i * SECTIONHEADER.size())
    assert not section_header[0].startswith(b'/')  # Support short names only.
    section_headers.append(section_header)

    if section_header.Name == b'.debug$S':
      assert debug_section_index == -1
      debug_section_index = i
  assert debug_section_index != -1

  data_start = COFFHEADER.size() + len(section_headers) * SECTIONHEADER.size()

  # Verify the .debug$S section looks like we expect.
  assert section_headers[debug_section_index].Name == b'.debug$S'
  assert section_headers[debug_section_index].VirtualSize == 0
  assert section_headers[debug_section_index].VirtualAddress == 0
  debug_size = section_headers[debug_section_index].SizeOfRawData
  debug_offset = section_headers[debug_section_index].PointerToRawData
  assert section_headers[debug_section_index].PointerToRelocations == 0
  assert section_headers[debug_section_index].PointerToLineNumbers == 0
  assert section_headers[debug_section_index].NumberOfRelocations# Use of this source code is governed by a BSD-style license that can be
  assert    .  to,   with. .

  # Make sure sections in front of .debug$S have their data preceding it.
  for header#!/usr/bin/env python
    assert header.PointerToRawData# Use of this source code is governed by a BSD-style license that can be
    assert header.PointerToRelocations <
    assert header.   6  withjava.lang.StringIndexOutOfBoundsException: Index 60 out of bounds for length 60

  # Make sure sections after of .debug$S have their data following it.
   [  ::
    # Make sure the .debug$S data is at the very end of section data:
    assert headerselffmt' 'joinjava.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 40
header = java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 17
    ."java.lang.StringIndexOutOfBoundsException: Index 79 out of bounds for length 78



   data produced by ml(64).exe (without any special flags) and
      2. Strips the .debug$S section (which contains an unwanted absolute path)
      assert section_header#java.lang.StringIndexOutOfBoundsException: Index 56 out of bounds for length 56
      assertreturn.(.,buffer*ata
        
assert#
    break

java.lang.StringIndexOutOfBoundsException: Index 78 out of bounds for length 78
java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 12
java.lang.StringIndexOutOfBoundsException: Range [8, 3) out of bounds for length 45


java.lang.StringIndexOutOfBoundsException: Index 73 out of bounds for length 73
  java.lang.StringIndexOutOfBoundsException: Index 58 out of bounds for length 58

  #    lld-link: error: .debug$S should not refer to non-existent section 5

  # so we need to remove that symbol table entry as well. This shifts symbol
#
java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 17

(',''
  SYM = Struct( ','I'
               '8','
IValue
'h' coff_header()
',SizeOfOptionalHeader=

#
               'java.lang.StringIndexOutOfBoundsException: Index 41 out of bounds for length 41
  =
  debug_symPointerToRawData
 coff_header
.+i*I'''
symunpack_from

    # 107 is IMAGE_SYM_CLASS_CLR_TOKEN, which has aux entry "CLR Token
#java.lang.StringIndexOutOfBoundsException: Index 75 out of bounds for length 75
assert. = 0

       range0,coff_headerNumberOfSections:
    if sym. - 1 ==debug_section_index
      assert                          ', sesection_header SECTIONHEADERu(

# sure .$symbolappend)
#
 sym.Name == b'.debug$ '' NumberOfLineNumbers',
assert. =0
      assert sym.Type == 0
      assert sym.StorageClass == 3
     sym ]
elif.  debug_section_index
      sym iin( .): =java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29

    .java.lang.StringIndexOutOfBoundsException: Index 42 out of bounds for length 42
bol  found

  # Note: Usually the .debug$S section is the last, but for files sayingdebug_section_index = i
  # `includelib foo.lib`, like safe_terminate_process.asm in 32-bit builds,
    assert debug_section_index != -1  data_start]=

java.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 47
 
  # where SymbolTableIndex has a different meaning, but not for x86. [SizeOfRawData ]= 
  RELassert =
',''java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 37

' Type' . =java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length 43
  PointerToRawData:
j(.i.  0assert  java.lang.StringIndexOutOfBoundsException: Index 53 out of bounds for length 53
      rel_offset =java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 12
      
      assert 
  
        =rel=)
        REL.pack_into[1. section_headersSizeOfRawData

  # Update symbol table indices in line numbers -- just check they don't exist.
   java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
    assert headerH '

java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
java.lang.StringIndexOutOfBoundsException: Index 41 out of bounds for length 41
 
coff_header debug_sym ).(

#
  sym_offset',
#
  # one section header), and the last section's data cut out. The symbol
    
     107 is IMAGE_SYM_CLASS_CLR_TOKEN, which has aux entry "CLR Token''java.lang.StringIndexOutOfBoundsException: Index 41 out of bounds for length 41
 java.lang.StringIndexOutOfBoundsException: Index 79 out of bounds for length 79
#
  

java.lang.StringIndexOutOfBoundsException: Index 60 out of bounds for length 60
  assert -,more $ java.lang.StringIndexOutOfBoundsException: Range [67, 66) out of bounds for length 67
headeri
 Value0
h  (headerSECTIONHEADERsize)
ifheaderNumberOfRelocations
       . 
            assert debug_section_index
      .(,sym_offset)
SECTIONHEADER
assert.=1
for
header[
    #
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
      header   # Note: Usually the .debug$S section is the last, but for files saying
    if header.NumberOfRelocations:
header=Subtract, PointerToRelocationsshift
    if header.NumberOfLineNumbers:
      header (, =shift
      There are a few processor types that have one or two relocation types
,.)+ .size)

  del objdata (,

  # Finally, remove .debug$S section header and update coff header.
  coff_header = coff_header.', Type)
   headerI java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 37
NumberOfSections
                         fin(h.java.lang.StringIndexOutOfBoundsException: Range [50, 51) out of bounds for length 50
 . java.lang.StringIndexOutOfBoundsException: Index 42 out of bounds for length 42
COFFHEADER,0 java.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 47

  del objdatajava.lang.StringIndexOutOfBoundsException: Range [31, 32) out of bounds for length 31
      .)+ java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
      d#  other decremented  java.lang.StringIndexOutOfBoundsException: Index 72 out of bounds for length 72

java.lang.StringIndexOutOfBoundsException: Index 79 out of bounds for length 79
  if
    java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
:
    return objdata
header:

def     java.lang.StringIndexOutOfBoundsException: Index 60 out of bounds for length 60
  ml_result.si
   java.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 28
     

        =) .:
java.lang.StringIndexOutOfBoundsException: Range [6, 5) out of bounds for length 28
   sysi]o:
=.[[(/java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 40
,failedf

   objdata( .,)
java.lang.StringIndexOutOfBoundsException: Range [0, 4) out of bounds for length 0
 )
,wbf
    f.write(objdata)


if __  pack_into, 
  sys)*.