diff options
| author | Tom Rini <trini@konsulko.com> | 2018-06-08 07:20:08 -0400 |
|---|---|---|
| committer | Tom Rini <trini@konsulko.com> | 2018-06-08 07:20:08 -0400 |
| commit | 8f48cf9f1758a2c5cef05381401184959af64a69 (patch) | |
| tree | b0d4b76cc974dc25a6e6f6b0869cb07ce4a42024 | |
| parent | 71002b508a1bc0986023764f155f0db26f548db2 (diff) | |
| parent | 844e5b20f2691eccce9ac8f7e3732bbb5d0ac757 (diff) | |
Merge git://git.denx.de/u-boot-dm
46 files changed, 977 insertions, 391 deletions
@@ -11,8 +11,7 @@ result=0 run_test ./test/py/test.py --bd sandbox --build # Run tests which require sandbox_spl -run_test ./test/py/test.py --bd sandbox_spl --build -k \ - test/py/tests/test_ofplatdata.py +run_test ./test/py/test.py --bd sandbox_spl --build -k test_ofplatdata.py # Run tests for the flat DT version of sandbox ./test/py/test.py --bd sandbox_flattree --build diff --git a/tools/binman/README b/tools/binman/README index b20098177ee..22f21bc5b44 100644 --- a/tools/binman/README +++ b/tools/binman/README @@ -302,9 +302,9 @@ align-size: align-end: This sets the alignment of the end of an entry. Some entries require that they end on an alignment boundary, regardless of where they - start. If 'align-end' is not provided, no alignment is performed. - - Note: This is not yet implemented in binman. + start. This does not move the start of the entry, so the contents of + the entry will still start at the beginning. But there may be padding + at the end. If 'align-end' is not provided, no alignment is performed. filename: For 'blob' types this provides the filename containing the binary to @@ -316,6 +316,13 @@ type: possible to use any name, and then add (for example) 'type = "u-boot"' to specify the type. +pos-unset: + Indicates that the position of this entry should not be set by placing + it immediately after the entry before. Instead, is set by another + entry which knows where this entry should go. When this boolean + property is present, binman will give an error if another entry does + not set the position (with the GetPositions() method). + The attributes supported for images are described below. Several are similar to those for entries. @@ -387,6 +394,57 @@ end-at-4gb: Examples of the above options can be found in the tests. See the tools/binman/test directory. +It is possible to have the same binary appear multiple times in the image, +either by using a unit number suffix (u-boot@0, u-boot@1) or by using a +different name for each and specifying the type with the 'type' attribute. + + +Sections and hiearchical images +------------------------------- + +Sometimes it is convenient to split an image into several pieces, each of which +contains its own set of binaries. An example is a flash device where part of +the image is read-only and part is read-write. We can set up sections for each +of these, and place binaries in them independently. The image is still produced +as a single output file. + +This feature provides a way of creating hierarchical images. For example here +is an example image with two copies of U-Boot. One is read-only (ro), intended +to be written only in the factory. Another is read-write (rw), so that it can be +upgraded in the field. The sizes are fixed so that the ro/rw boundary is known +and can be programmed: + + binman { + section@0 { + read-only; + name-prefix = "ro-"; + size = <0x100000>; + u-boot { + }; + }; + section@1 { + name-prefix = "rw-"; + size = <0x100000>; + u-boot { + }; + }; + }; + +This image could be placed into a SPI flash chip, with the protection boundary +set at 1MB. + +A few special properties are provided for sections: + +read-only: + Indicates that this section is read-only. This has no impact on binman's + operation, but his property can be read at run time. + +name-prefix: + This string is prepended to all the names of the binaries in the + section. In the example above, the 'u-boot' binaries which actually be + renamed to 'ro-u-boot' and 'rw-u-boot'. This can be useful to + distinguish binaries with otherwise identical names. + Special properties ------------------ @@ -498,6 +556,25 @@ At present this feature is only supported in SPL. In principle it is possible to fill in such symbols in U-Boot proper, as well. +Map files +--------- + +The -m option causes binman to output a .map file for each image that it +generates. This shows the position and size of each entry. For example: + + Position Size Name + 00000000 00000010 section@0 + 00000000 00000004 u-boot + 00000010 00000010 section@1 + 00000000 00000004 u-boot + +This shows a hierarchical image with two sections, each with a single entry. The +positions of the sections are absolute hex byte offsets within the image. The +positions of the entries are relative to their respective sections. The size of +each entry is also shown, in bytes (hex). The indentation shows the entries +nested inside their sections. + + Code coverage ------------- @@ -547,7 +624,7 @@ Binman takes a lot of inspiration from a Chrome OS tool called a reasonably simple and sound design but has expanded greatly over the years. In particular its handling of x86 images is convoluted. -Quite a few lessons have been learned which are hopefully be applied here. +Quite a few lessons have been learned which are hopefully applied here. Design notes @@ -578,17 +655,13 @@ Some ideas: 'Access to binman entry positions at run time' above - Use of-platdata to make the information available to code that is unable to use device tree (such as a very small SPL image) -- Write an image map to a text file - Allow easy building of images by specifying just the board name - Produce a full Python binding for libfdt (for upstream) - Add an option to decode an image into the constituent binaries -- Suppoort hierarchical images (packing of binaries into another binary - which is then placed in the image) - Support building an image for a board (-b) more completely, with a configurable build directory - Consider making binman work with buildman, although if it is used in the Makefile, this will be automatic -- Implement align-end -- Simon Glass <sjg@chromium.org> diff --git a/tools/binman/binman.py b/tools/binman/binman.py index fa2f551f554..31b045337d2 100755 --- a/tools/binman/binman.py +++ b/tools/binman/binman.py @@ -23,15 +23,18 @@ for dirname in ['../patman', '../dtoc', '..']: # Bring in the libfdt module sys.path.insert(0, 'scripts/dtc/pylibfdt') -# Also allow entry-type modules to be brought in from the etype directory. -sys.path.insert(0, os.path.join(our_path, 'etype')) - import cmdline import command import control -def RunTests(debug): - """Run the functional tests and any embedded doctests""" +def RunTests(debug, args): + """Run the functional tests and any embedded doctests + + Args: + debug: True to enable debugging, which shows a full stack trace on error + args: List of positional args provided to binman. This can hold a test + name to execute (as in 'binman -t testSections', for example) + """ import elf_test import entry_test import fdt_test @@ -53,9 +56,16 @@ def RunTests(debug): # 'entry' module. suite = unittest.TestLoader().loadTestsFromTestCase(entry_test.TestEntry) suite.run(result) + test_name = args and args[0] or None for module in (ftest.TestFunctional, fdt_test.TestFdt, elf_test.TestElf, image_test.TestImage): - suite = unittest.TestLoader().loadTestsFromTestCase(module) + if test_name: + try: + suite = unittest.TestLoader().loadTestsFromName(args[0], module) + except AttributeError: + continue + else: + suite = unittest.TestLoader().loadTestsFromTestCase(module) suite.run(result) print result @@ -114,7 +124,7 @@ def RunBinman(options, args): sys.tracebacklimit = 0 if options.test: - ret_code = RunTests(options.debug) + ret_code = RunTests(options.debug, args[1:]) elif options.test_coverage: RunTestCoverage() diff --git a/tools/binman/bsection.py b/tools/binman/bsection.py new file mode 100644 index 00000000000..3f30f6e4fe7 --- /dev/null +++ b/tools/binman/bsection.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2018 Google, Inc +# Written by Simon Glass <sjg@chromium.org> +# +# Base class for sections (collections of entries) +# + +from __future__ import print_function + +from collections import OrderedDict +import sys + +import fdt_util +import re +import tools + +class Section(object): + """A section which contains multiple entries + + A section represents a collection of entries. There must be one or more + sections in an image. Sections are used to group entries together. + + Attributes: + _node: Node object that contains the section definition in device tree + _size: Section size in bytes, or None if not known yet + _align_size: Section size alignment, or None + _pad_before: Number of bytes before the first entry starts. This + effectively changes the place where entry position 0 starts + _pad_after: Number of bytes after the last entry ends. The last + entry will finish on or before this boundary + _pad_byte: Byte to use to pad the section where there is no entry + _sort: True if entries should be sorted by position, False if they + must be in-order in the device tree description + _skip_at_start: Number of bytes before the first entry starts. These + effectively adjust the starting position of entries. For example, + if _pad_before is 16, then the first entry would start at 16. + An entry with pos = 20 would in fact be written at position 4 + in the image file. + _end_4gb: Indicates that the section ends at the 4GB boundary. This is + used for x86 images, which want to use positions such that a + memory address (like 0xff800000) is the first entry position. + This causes _skip_at_start to be set to the starting memory + address. + _name_prefix: Prefix to add to the name of all entries within this + section + _entries: OrderedDict() of entries + """ + def __init__(self, name, node, test=False): + global entry + global Entry + import entry + from entry import Entry + + self._node = node + self._size = None + self._align_size = None + self._pad_before = 0 + self._pad_after = 0 + self._pad_byte = 0 + self._sort = False + self._skip_at_start = 0 + self._end_4gb = False + self._name_prefix = '' + self._entries = OrderedDict() + if not test: + self._ReadNode() + self._ReadEntries() + + def _ReadNode(self): + """Read properties from the section node""" + self._size = fdt_util.GetInt(self._node, 'size') + self._align_size = fdt_util.GetInt(self._node, 'align-size') + if tools.NotPowerOfTwo(self._align_size): + self._Raise("Alignment size %s must be a power of two" % + self._align_size) + self._pad_before = fdt_util.GetInt(self._node, 'pad-before', 0) + self._pad_after = fdt_util.GetInt(self._node, 'pad-after', 0) + self._pad_byte = fdt_util.GetInt(self._node, 'pad-byte', 0) + self._sort = fdt_util.GetBool(self._node, 'sort-by-pos') + self._end_4gb = fdt_util.GetBool(self._node, 'end-at-4gb') + if self._end_4gb and not self._size: + self._Raise("Section size must be provided when using end-at-4gb") + if self._end_4gb: + self._skip_at_start = 0x100000000 - self._size + self._name_prefix = fdt_util.GetString(self._node, 'name-prefix') + + def _ReadEntries(self): + for node in self._node.subnodes: + entry = Entry.Create(self, node) + entry.SetPrefix(self._name_prefix) + self._entries[node.name] = entry + + def CheckSize(self): + """Check that the section contents does not exceed its size, etc.""" + contents_size = 0 + for entry in self._entries.values(): + contents_size = max(contents_size, entry.pos + entry.size) + + contents_size -= self._skip_at_start + + size = self._size + if not size: + size = self._pad_before + contents_size + self._pad_after + size = tools.Align(size, self._align_size) + + if self._size and contents_size > self._size: + self._Raise("contents size %#x (%d) exceeds section size %#x (%d)" % + (contents_size, contents_size, self._size, self._size)) + if not self._size: + self._size = size + if self._size != tools.Align(self._size, self._align_size): + self._Raise("Size %#x (%d) does not match align-size %#x (%d)" % + (self._size, self._size, self._align_size, self._align_size)) + return size + + def _Raise(self, msg): + """Raises an error for this section + + Args: + msg: Error message to use in the raise string + Raises: + ValueError() + """ + raise ValueError("Section '%s': %s" % (self._node.path, msg)) + + def GetPath(self): + """Get the path of an image (in the FDT) + + Returns: + Full path of the node for this image + """ + return self._node.path + + def FindEntryType(self, etype): + """Find an entry type in the section + + Args: + etype: Entry type to find + Returns: + entry matching that type, or None if not found + """ + for entry in self._entries.values(): + if entry.etype == etype: + return entry + return None + + def GetEntryContents(self): + """Call ObtainContents() for each entry + + This calls each entry's ObtainContents() a few times until they all + return True. We stop calling an entry's function once it returns + True. This allows the contents of one entry to depend on another. + + After 3 rounds we give up since it's likely an error. + """ + todo = self._entries.values() + for passnum in range(3): + next_todo = [] + for entry in todo: + if not entry.ObtainContents(): + next_todo.append(entry) + todo = next_todo + if not todo: + break + + def _SetEntryPosSize(self, name, pos, size): + """Set the position and size of an entry + + Args: + name: Entry name to update + pos: New position + size: New size + """ + entry = self._entries.get(name) + if not entry: + self._Raise("Unable to set pos/size for unknown entry '%s'" % name) + entry.SetPositionSize(self._skip_at_start + pos, size) + + def GetEntryPositions(self): + """Handle entries that want to set the position/size of other entries + + This calls each entry's GetPositions() method. If it returns a list + of entries to update, it updates them. + """ + for entry in self._entries.values(): + pos_dict = entry.GetPositions() + for name, info in pos_dict.iteritems(): + self._SetEntryPosSize(name, *info) + + def PackEntries(self): + """Pack all entries into the section""" + pos = self._skip_at_start + for entry in self._entries.values(): + pos = entry.Pack(pos) + + def _SortEntries(self): + """Sort entries by position""" + entries = sorted(self._entries.values(), key=lambda entry: entry.pos) + self._entries.clear() + for entry in entries: + self._entries[entry._node.name] = entry + + def CheckEntries(self): + """Check that entries do not overlap or extend outside the section""" + if self._sort: + self._SortEntries() + pos = 0 + prev_name = 'None' + for entry in self._entries.values(): + entry.CheckPosition() + if (entry.pos < self._skip_at_start or + entry.pos >= self._skip_at_start + self._size): + entry.Raise("Position %#x (%d) is outside the section starting " + "at %#x (%d)" % + (entry.pos, entry.pos, self._skip_at_start, + self._skip_at_start)) + if entry.pos < pos: + entry.Raise("Position %#x (%d) overlaps with previous entry '%s' " + "ending at %#x (%d)" % + (entry.pos, entry.pos, prev_name, pos, pos)) + pos = entry.pos + entry.size + prev_name = entry.GetPath() + + def ProcessEntryContents(self): + """Call the ProcessContents() method for each entry + + This is intended to adjust the contents as needed by the entry type. + """ + for entry in self._entries.values(): + entry.ProcessContents() + + def WriteSymbols(self): + """Write symbol values into binary files for access at run time""" + for entry in self._entries.values(): + entry.WriteSymbols(self) + + def BuildSection(self, fd, base_pos): + """Write the section to a file""" + fd.seek(base_pos) + fd.write(self.GetData()) + + def GetData(self): + """Write the section to a file""" + section_data = chr(self._pad_byte) * self._size + + for entry in self._entries.values(): + data = entry.GetData() + base = self._pad_before + entry.pos - self._skip_at_start + section_data = (section_data[:base] + data + + section_data[base + len(data):]) + return section_data + + def LookupSymbol(self, sym_name, optional, msg): + """Look up a symbol in an ELF file + + Looks up a symbol in an ELF file. Only entry types which come from an + ELF image can be used by this function. + + At present the only entry property supported is pos. + + Args: + sym_name: Symbol name in the ELF file to look up in the format + _binman_<entry>_prop_<property> where <entry> is the name of + the entry and <property> is the property to find (e.g. + _binman_u_boot_prop_pos). As a special case, you can append + _any to <entry> to have it search for any matching entry. E.g. + _binman_u_boot_any_prop_pos will match entries called u-boot, + u-boot-img and u-boot-nodtb) + optional: True if the symbol is optional. If False this function + will raise if the symbol is not found + msg: Message to display if an error occurs + + Returns: + Value that should be assigned to that symbol, or None if it was + optional and not found + + Raises: + ValueError if the symbol is invalid or not found, or references a + property which is not supported + """ + m = re.match(r'^_binman_(\w+)_prop_(\w+)$', sym_name) + if not m: + raise ValueError("%s: Symbol '%s' has invalid format" % + (msg, sym_name)) + entry_name, prop_name = m.groups() + entry_name = entry_name.replace('_', '-') + entry = self._entries.get(entry_name) + if not entry: + if entry_name.endswith('-any'): + root = entry_name[:-4] + for name in self._entries: + if name.startswith(root): + rest = name[len(root):] + if rest in ['', '-img', '-nodtb']: + entry = self._entries[name] + if not entry: + err = ("%s: Entry '%s' not found in list (%s)" % + (msg, entry_name, ','.join(self._entries.keys()))) + if optional: + print('Warning: %s' % err, file=sys.stderr) + return None + raise ValueError(err) + if prop_name == 'pos': + return entry.pos + else: + raise ValueError("%s: No such property '%s'" % (msg, prop_name)) + + def GetEntries(self): + return self._entries + + def WriteMap(self, fd, indent): + """Write a map of the section to a .map file + + Args: + fd: File to write the map to + """ + for entry in self._entries.values(): + entry.WriteMap(fd, indent) diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py index e9e0434ab36..bf63919eb79 100644 --- a/tools/binman/cmdline.py +++ b/tools/binman/cmdline.py @@ -30,6 +30,8 @@ def ParseArgs(argv): help='Add a path to a directory to use for input files') parser.add_option('-H', '--full-help', action='store_true', default=False, help='Display the README file') + parser.add_option('-m', '--map', action='store_true', + default=False, help='Output a map file for each image') parser.add_option('-O', '--outdir', type='string', action='store', help='Path to directory to use for intermediate and ' 'output files') diff --git a/tools/binman/control.py b/tools/binman/control.py index bc8ed8e37ae..92434729061 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -112,6 +112,8 @@ def Binman(options, args): image.ProcessEntryContents() image.WriteSymbols() image.BuildImage() + if options.map: + image.WriteMap() finally: tools.FinaliseOutputDir() finally: diff --git a/tools/binman/elf.py b/tools/binman/elf.py index 50085a3893a..0ae3b611bae 100644 --- a/tools/binman/elf.py +++ b/tools/binman/elf.py @@ -75,7 +75,7 @@ def GetSymbolAddress(fname, sym_name): return None return sym.address -def LookupAndWriteSymbols(elf_fname, entry, image): +def LookupAndWriteSymbols(elf_fname, entry, section): """Replace all symbols in an entry with their correct values The entry contents is updated so that values for referenced symbols will be @@ -87,7 +87,7 @@ def LookupAndWriteSymbols(elf_fname, entry, image): elf_fname: Filename of ELF image containing the symbol information for entry entry: Entry to process - image: Image which can be used to lookup symbol values + section: Section which can be used to lookup symbol values """ fname = tools.GetInputFilename(elf_fname) syms = GetSymbols(fname, ['image', 'binman']) @@ -98,8 +98,8 @@ def LookupAndWriteSymbols(elf_fname, entry, image): return for name, sym in syms.iteritems(): if name.startswith('_binman'): - msg = ("Image '%s': Symbol '%s'\n in entry '%s'" % - (image.GetPath(), name, entry.GetPath())) + msg = ("Section '%s': Symbol '%s'\n in entry '%s'" % + (section.GetPath(), name, entry.GetPath())) offset = sym.address - base.address if offset < 0 or offset + sym.size > entry.contents_size: raise ValueError('%s has offset %x (size %x) but the contents ' @@ -114,7 +114,7 @@ def LookupAndWriteSymbols(elf_fname, entry, image): (msg, sym.size)) # Look up the symbol in our entry tables. - value = image.LookupSymbol(name, sym.weak, msg) + value = section.LookupSymbol(name, sym.weak, msg) if value is not None: value += base.address else: diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index 4abde121bd1..fb6e451cf0e 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -40,12 +40,12 @@ class FakeEntry: def GetPath(self): return 'entry_path' -class FakeImage: +class FakeSection: def __init__(self, sym_value=1): self.sym_value = sym_value def GetPath(self): - return 'image_path' + return 'section_path' def LookupSymbol(self, name, weak, msg): return self.sym_value @@ -67,51 +67,51 @@ class TestElf(unittest.TestCase): def testMissingFile(self): entry = FakeEntry(10) - image = FakeImage() + section = FakeSection() with self.assertRaises(ValueError) as e: - syms = elf.LookupAndWriteSymbols('missing-file', entry, image) + syms = elf.LookupAndWriteSymbols('missing-file', entry, section) self.assertIn("Filename 'missing-file' not found in input path", str(e.exception)) def testOutsideFile(self): entry = FakeEntry(10) - image = FakeImage() + section = FakeSection() elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms') with self.assertRaises(ValueError) as e: - syms = elf.LookupAndWriteSymbols(elf_fname, entry, image) + syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) self.assertIn('entry_path has offset 4 (size 8) but the contents size ' 'is a', str(e.exception)) def testMissingImageStart(self): entry = FakeEntry(10) - image = FakeImage() + section = FakeSection() elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms_bad') - self.assertEqual(elf.LookupAndWriteSymbols(elf_fname, entry, image), + self.assertEqual(elf.LookupAndWriteSymbols(elf_fname, entry, section), None) def testBadSymbolSize(self): entry = FakeEntry(10) - image = FakeImage() + section = FakeSection() elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms_size') with self.assertRaises(ValueError) as e: - syms = elf.LookupAndWriteSymbols(elf_fname, entry, image) + syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) self.assertIn('has size 1: only 4 and 8 are supported', str(e.exception)) def testNoValue(self): entry = FakeEntry(20) - image = FakeImage(sym_value=None) + section = FakeSection(sym_value=None) elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms') - syms = elf.LookupAndWriteSymbols(elf_fname, entry, image) + syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) self.assertEqual(chr(255) * 16 + 'a' * 4, entry.data) def testDebug(self): elf.debug = True entry = FakeEntry(20) - image = FakeImage() + section = FakeSection() elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms') with capture_sys_output() as (stdout, stderr): - syms = elf.LookupAndWriteSymbols(elf_fname, entry, image) + syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) elf.debug = False self.assertTrue(len(stdout.getvalue()) > 0) diff --git a/tools/binman/etype/entry.py b/tools/binman/entry.py index c331312c491..e4d688c91f9 100644 --- a/tools/binman/etype/entry.py +++ b/tools/binman/entry.py @@ -4,6 +4,8 @@ # Base class for all entries # +from __future__ import print_function + # importlib was introduced in Python 2.7 but there was a report of it not # working in 2.7.12, so we work around this: # http://lists.denx.de/pipermail/u-boot/2016-October/269729.html @@ -14,15 +16,19 @@ except: have_importlib = False import fdt_util +import os +import sys import tools modules = {} +our_path = os.path.dirname(os.path.realpath(__file__)) + class Entry(object): - """An Entry in the image + """An Entry in the section An entry corresponds to a single node in the device-tree description - of the image. Each entry ends up being a part of the final image. + of the section. Each entry ends up being a part of the final section. Entries can be placed either right next to each other, or with padding between them. The type of the entry determines the data that is in it. @@ -30,9 +36,9 @@ class Entry(object): Entry. Attributes: - image: The image containing this entry + section: The section containing this entry node: The node that created this entry - pos: Absolute position of entry within the image, None if not known + pos: Absolute position of entry within the section, None if not known size: Entry size in bytes, None if not known contents_size: Size of contents in bytes, 0 by default align: Entry start position alignment, or None @@ -42,10 +48,11 @@ class Entry(object): pad_after: Number of pad bytes after the contents, 0 if none data: Contents of entry (string of bytes) """ - def __init__(self, image, etype, node, read_node=True): - self.image = image + def __init__(self, section, etype, node, read_node=True, name_prefix=''): + self.section = section self.etype = etype self._node = node + self.name = node and (name_prefix + node.name) or 'none' self.pos = None self.size = None self.contents_size = 0 @@ -59,11 +66,11 @@ class Entry(object): self.ReadNode() @staticmethod - def Create(image, node, etype=None): + def Create(section, node, etype=None): """Create a new entry for a node. Args: - image: Image object containing this node + section: Image object containing this node node: Node object containing information about the entry to create etype: Entry type to use, or None to work it out (used for tests) @@ -72,11 +79,20 @@ class Entry(object): """ if not etype: etype = fdt_util.GetString(node, 'type', node.name) + + # Convert something like 'u-boot@0' to 'u_boot' since we are only + # interested in the type. module_name = etype.replace('-', '_') + if '@' in module_name: + module_name = module_name.split('@')[0] module = modules.get(module_name) + # Also allow entry-type modules to be brought in from the etype directory. + # Import the module if we have not already done so. if not module: + old_path = sys.path + sys.path.insert(0, os.path.join(our_path, 'etype')) try: if have_importlib: module = importlib.import_module(module_name) @@ -85,11 +101,13 @@ class Entry(object): except ImportError: raise ValueError("Unknown entry type '%s' in node '%s'" % (etype, node.path)) + finally: + sys.path = old_path modules[module_name] = module # Call its constructor to get the object we want. obj = getattr(module, 'Entry_%s' % module_name) - return obj(image, etype, node) + return obj(section, etype, node) def ReadNode(self): """Read entry information from the node @@ -111,6 +129,15 @@ class Entry(object): self.align_end = fdt_util.GetInt(self._node, 'align-end') self.pos_unset = fdt_util.GetBool(self._node, 'pos-unset') + def SetPrefix(self, prefix): + """Set the name prefix for a node + + Args: + prefix: Prefix to set, or '' to not use a prefix + """ + if prefix: + self.name = prefix + self.name + def ObtainContents(self): """Figure out the contents of an entry. @@ -122,7 +149,7 @@ class Entry(object): return True def Pack(self, pos): - """Figure out how to pack the entry into the image + """Figure out how to pack the entry into the section Most of the time the entries are not fully specified. There may be an alignment but no size. In that case we take the size from the @@ -134,10 +161,10 @@ class Entry(object): entry will be know. Args: - Current image position pointer + Current section position pointer Returns: - New image position pointer (after this entry) + New section position pointer (after this entry) """ if self.pos is None: if self.pos_unset: @@ -198,10 +225,29 @@ class Entry(object): def ProcessContents(self): pass - def WriteSymbols(self, image): + def WriteSymbols(self, section): """Write symbol values into binary files for access at run time Args: - image: Image containing the entry + section: Section containing the entry """ pass + + def CheckPosition(self): + """Check that the entry positions are correct + + This is used for entries which have extra position requirements (other + than having to be fully inside their section). Sub-classes can implement + this function and raise if there is a problem. + """ + pass + + def WriteMap(self, fd, indent): + """Write a map of the entry to a .map file + + Args: + fd: File to write the map to + indent: Curent indent level of map (0=none, 1=one level, etc.) + """ + print('%s%08x %08x %s' % (' ' * indent, self.pos, self.size, + self.name), file=fd) diff --git a/tools/binman/etype/_testing.py b/tools/binman/etype/_testing.py index b166a71c4a9..c376dd5c9ca 100644 --- a/tools/binman/etype/_testing.py +++ b/tools/binman/etype/_testing.py @@ -10,8 +10,8 @@ import fdt_util import tools class Entry__testing(Entry): - def __init__(self, image, etype, node): - Entry.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry.__init__(self, section, etype, node) def ObtainContents(self): self.data = 'a' diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index 10e59e980d6..16b1e5f64d9 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -10,8 +10,8 @@ import fdt_util import tools class Entry_blob(Entry): - def __init__(self, image, etype, node): - Entry.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry.__init__(self, section, etype, node) self._filename = fdt_util.GetString(self._node, "filename", self.etype) def ObtainContents(self): diff --git a/tools/binman/etype/intel_cmc.py b/tools/binman/etype/intel_cmc.py index 07cad2eecce..b8621e0cc5e 100644 --- a/tools/binman/etype/intel_cmc.py +++ b/tools/binman/etype/intel_cmc.py @@ -9,5 +9,5 @@ from entry import Entry from blob import Entry_blob class Entry_intel_cmc(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) diff --git a/tools/binman/etype/intel_descriptor.py b/tools/binman/etype/intel_descriptor.py index 4e0c85b92ae..0e7865521e1 100644 --- a/tools/binman/etype/intel_descriptor.py +++ b/tools/binman/etype/intel_descriptor.py @@ -32,8 +32,8 @@ class Entry_intel_descriptor(Entry_blob): size of the ME region, allowing us to place the ME binary in the right place. """ - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) self._regions = [] def GetPositions(self): diff --git a/tools/binman/etype/intel_fsp.py b/tools/binman/etype/intel_fsp.py index 827bd22ca07..cb80a617c32 100644 --- a/tools/binman/etype/intel_fsp.py +++ b/tools/binman/etype/intel_fsp.py @@ -9,5 +9,5 @@ from entry import Entry from blob import Entry_blob class Entry_intel_fsp(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) diff --git a/tools/binman/etype/intel_me.py b/tools/binman/etype/intel_me.py index e02e4859877..0682ead9439 100644 --- a/tools/binman/etype/intel_me.py +++ b/tools/binman/etype/intel_me.py @@ -9,5 +9,5 @@ from entry import Entry from blob import Entry_blob class Entry_intel_me(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) diff --git a/tools/binman/etype/intel_mrc.py b/tools/binman/etype/intel_mrc.py index 7c01b77a6f7..305ac98888b 100644 --- a/tools/binman/etype/intel_mrc.py +++ b/tools/binman/etype/intel_mrc.py @@ -9,8 +9,8 @@ from entry import Entry from blob import Entry_blob class Entry_intel_mrc(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def GetDefaultFilename(self): return 'mrc.bin' diff --git a/tools/binman/etype/intel_vbt.py b/tools/binman/etype/intel_vbt.py index 4f082c3c2c9..d4e8c3f797d 100644 --- a/tools/binman/etype/intel_vbt.py +++ b/tools/binman/etype/intel_vbt.py @@ -8,5 +8,5 @@ from entry import Entry from blob import Entry_blob class Entry_intel_vbt(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) diff --git a/tools/binman/etype/intel_vga.py b/tools/binman/etype/intel_vga.py index 277fff15fdd..140dd40dda6 100644 --- a/tools/binman/etype/intel_vga.py +++ b/tools/binman/etype/intel_vga.py @@ -9,5 +9,5 @@ from entry import Entry from blob import Entry_blob class Entry_intel_vga(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py new file mode 100644 index 00000000000..139fcad51af --- /dev/null +++ b/tools/binman/etype/section.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2018 Google, Inc +# Written by Simon Glass <sjg@chromium.org> +# +# Entry-type module for sections, which are entries which can contain other +# entries. +# + +from entry import Entry +import fdt_util +import tools + +import bsection + +class Entry_section(Entry): + def __init__(self, image, etype, node): + Entry.__init__(self, image, etype, node) + self._section = bsection.Section(node.name, node) + + def ObtainContents(self): + self._section.GetEntryContents() + + def GetData(self): + return self._section.GetData() + + def GetPositions(self): + """Handle entries that want to set the position/size of other entries + + This calls each entry's GetPositions() method. If it returns a list + of entries to update, it updates them. + """ + self._section.GetEntryPositions() + return {} + + def Pack(self, pos): + """Pack all entries into the section""" + self._section.PackEntries() + self.size = self._section.CheckSize() + return super(Entry_section, self).Pack(pos) + + def WriteSymbols(self, section): + """Write symbol values into binary files for access at run time""" + self._section.WriteSymbols() + + def ProcessContents(self): + self._section.ProcessEntryContents() + super(Entry_section, self).ProcessContents() + + def CheckPosition(self): + self._section.CheckEntries() + + def WriteMap(self, fd, indent): + """Write a map of the section to a .map file + + Args: + fd: File to write the map to + """ + super(Entry_section, self).WriteMap(fd, indent) + self._section.WriteMap(fd, indent + 1) diff --git a/tools/binman/etype/u_boot.py b/tools/binman/etype/u_boot.py index fc212d03009..b6058bf6b52 100644 --- a/tools/binman/etype/u_boot.py +++ b/tools/binman/etype/u_boot.py @@ -9,8 +9,8 @@ from entry import Entry from blob import Entry_blob class Entry_u_boot(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def GetDefaultFilename(self): return 'u-boot.bin' diff --git a/tools/binman/etype/u_boot_dtb.py b/tools/binman/etype/u_boot_dtb.py index f6704db742b..dd3c5b27901 100644 --- a/tools/binman/etype/u_boot_dtb.py +++ b/tools/binman/etype/u_boot_dtb.py @@ -9,8 +9,8 @@ from entry import Entry from blob import Entry_blob class Entry_u_boot_dtb(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def GetDefaultFilename(self): return 'u-boot.dtb' diff --git a/tools/binman/etype/u_boot_dtb_with_ucode.py b/tools/binman/etype/u_boot_dtb_with_ucode.py index bedc39805df..1e530d6570a 100644 --- a/tools/binman/etype/u_boot_dtb_with_ucode.py +++ b/tools/binman/etype/u_boot_dtb_with_ucode.py @@ -16,8 +16,8 @@ class Entry_u_boot_dtb_with_ucode(Entry_blob): See Entry_u_boot_ucode for full details of the 3 entries involved in this process. """ - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) self.ucode_data = '' self.collate = False self.ucode_offset = None @@ -29,10 +29,12 @@ class Entry_u_boot_dtb_with_ucode(Entry_blob): def ObtainContents(self): Entry_blob.ObtainContents(self) - # If the image does not need microcode, there is nothing to do - ucode_dest_entry = self.image.FindEntryType('u-boot-spl-with-ucode-ptr') + # If the section does not need microcode, there is nothing to do + ucode_dest_entry = self.section.FindEntryType( + 'u-boot-spl-with-ucode-ptr') if not ucode_dest_entry or not ucode_dest_entry.target_pos: - ucode_dest_entry = self.image.FindEntryType('u-boot-with-ucode-ptr') + ucode_dest_entry = self.section.FindEntryType( + 'u-boot-with-ucode-ptr') if not ucode_dest_entry or not ucode_dest_entry.target_pos: return True diff --git a/tools/binman/etype/u_boot_img.py b/tools/binman/etype/u_boot_img.py index d5f1eb30570..6e0b736af14 100644 --- a/tools/binman/etype/u_boot_img.py +++ b/tools/binman/etype/u_boot_img.py @@ -9,8 +9,8 @@ from entry import Entry from blob import Entry_blob class Entry_u_boot_img(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def GetDefaultFilename(self): return 'u-boot.img' diff --git a/tools/binman/etype/u_boot_nodtb.py b/tools/binman/etype/u_boot_nodtb.py index 183b897bee6..ca9e53a0947 100644 --- a/tools/binman/etype/u_boot_nodtb.py +++ b/tools/binman/etype/u_boot_nodtb.py @@ -9,8 +9,8 @@ from entry import Entry from blob import Entry_blob class Entry_u_boot_nodtb(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def GetDefaultFilename(self): return 'u-boot-nodtb.bin' diff --git a/tools/binman/etype/u_boot_spl.py b/tools/binman/etype/u_boot_spl.py index 6a1c1234678..9edd2dad030 100644 --- a/tools/binman/etype/u_boot_spl.py +++ b/tools/binman/etype/u_boot_spl.py @@ -11,12 +11,12 @@ from entry import Entry from blob import Entry_blob class Entry_u_boot_spl(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) self.elf_fname = 'spl/u-boot-spl' def GetDefaultFilename(self): return 'spl/u-boot-spl.bin' - def WriteSymbols(self, image): - elf.LookupAndWriteSymbols(self.elf_fname, self, image) + def WriteSymbols(self, section): + elf.LookupAndWriteSymbols(self.elf_fname, self, section) diff --git a/tools/binman/etype/u_boot_spl_bss_pad.py b/tools/binman/etype/u_boot_spl_bss_pad.py index d14122b4357..3d2dea2e0d8 100644 --- a/tools/binman/etype/u_boot_spl_bss_pad.py +++ b/tools/binman/etype/u_boot_spl_bss_pad.py @@ -14,8 +14,8 @@ from blob import Entry_blob import tools class Entry_u_boot_spl_bss_pad(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def ObtainContents(self): fname = tools.GetInputFilename('spl/u-boot-spl') diff --git a/tools/binman/etype/u_boot_spl_dtb.py b/tools/binman/etype/u_boot_spl_dtb.py index 43d23778ecf..eefa1ff53a4 100644 --- a/tools/binman/etype/u_boot_spl_dtb.py +++ b/tools/binman/etype/u_boot_spl_dtb.py @@ -9,8 +9,8 @@ from entry import Entry from blob import Entry_blob class Entry_u_boot_spl_dtb(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def GetDefaultFilename(self): return 'spl/u-boot-spl.dtb' diff --git a/tools/binman/etype/u_boot_spl_nodtb.py b/tools/binman/etype/u_boot_spl_nodtb.py index 5b058b4c721..99e56ebe3bf 100644 --- a/tools/binman/etype/u_boot_spl_nodtb.py +++ b/tools/binman/etype/u_boot_spl_nodtb.py @@ -9,8 +9,8 @@ from entry import Entry from blob import Entry_blob class Entry_u_boot_spl_nodtb(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def GetDefaultFilename(self): return 'spl/u-boot-spl-nodtb.bin' diff --git a/tools/binman/etype/u_boot_spl_with_ucode_ptr.py b/tools/binman/etype/u_boot_spl_with_ucode_ptr.py index 1e3181a944c..0dfe268a568 100644 --- a/tools/binman/etype/u_boot_spl_with_ucode_ptr.py +++ b/tools/binman/etype/u_boot_spl_with_ucode_ptr.py @@ -19,8 +19,8 @@ class Entry_u_boot_spl_with_ucode_ptr(Entry_u_boot_with_ucode_ptr): See Entry_u_boot_ucode for full details of the entries involved in this process. """ - def __init__(self, image, etype, node): - Entry_u_boot_with_ucode_ptr.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_u_boot_with_ucode_ptr.__init__(self, section, etype, node) self.elf_fname = 'spl/u-boot-spl' def GetDefaultFilename(self): diff --git a/tools/binman/etype/u_boot_ucode.py b/tools/binman/etype/u_boot_ucode.py index 10130a28114..3a0cff7c3a3 100644 --- a/tools/binman/etype/u_boot_ucode.py +++ b/tools/binman/etype/u_boot_ucode.py @@ -51,13 +51,13 @@ class Entry_u_boot_ucode(Entry_blob): the Entry_u_boot_dtb_with_ucode entry, and uses it as the contents of this entry. """ - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def ObtainContents(self): - # If the image does not need microcode, there is nothing to do - ucode_dest_entry = self.image.FindEntryType('u-boot-with-ucode-ptr') - ucode_dest_entry_spl = self.image.FindEntryType( + # If the section does not need microcode, there is nothing to do + ucode_dest_entry = self.section.FindEntryType('u-boot-with-ucode-ptr') + ucode_dest_entry_spl = self.section.FindEntryType( 'u-boot-spl-with-ucode-ptr') if ((not ucode_dest_entry or not ucode_dest_entry.target_pos) and (not ucode_dest_entry_spl or not ucode_dest_entry_spl.target_pos)): @@ -65,12 +65,12 @@ class Entry_u_boot_ucode(Entry_blob): return True # Get the microcode from the device tree entry - fdt_entry = self.image.FindEntryType('u-boot-dtb-with-ucode') + fdt_entry = self.section.FindEntryType('u-boot-dtb-with-ucode') if not fdt_entry or not fdt_entry.ucode_data: return False if not fdt_entry.collate: - # This section can be empty + # This binary can be empty self.data = '' return True diff --git a/tools/binman/etype/u_boot_with_ucode_ptr.py b/tools/binman/etype/u_boot_with_ucode_ptr.py index 04b9f7cf7f8..41c2ded2fe8 100644 --- a/tools/binman/etype/u_boot_with_ucode_ptr.py +++ b/tools/binman/etype/u_boot_with_ucode_ptr.py @@ -20,8 +20,8 @@ class Entry_u_boot_with_ucode_ptr(Entry_blob): See Entry_u_boot_ucode for full details of the 3 entries involved in this process. """ - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) self.elf_fname = 'u-boot' self.target_pos = None @@ -45,24 +45,24 @@ class Entry_u_boot_with_ucode_ptr(Entry_blob): return # Get the position of the microcode - ucode_entry = self.image.FindEntryType('u-boot-ucode') + ucode_entry = self.section.FindEntryType('u-boot-ucode') if not ucode_entry: self.Raise('Cannot find microcode region u-boot-ucode') - # Check the target pos is in the image. If it is not, then U-Boot is + # Check the target pos is in the section. If it is not, then U-Boot is # being linked incorrectly, or is being placed at the wrong position - # in the image. + # in the section. # - # The image must be set up so that U-Boot is placed at the + # The section must be set up so that U-Boot is placed at the # flash address to which it is linked. For example, if # CONFIG_SYS_TEXT_BASE is 0xfff00000, and the ROM is 8MB, then - # the U-Boot region must start at position 7MB in the image. In this + # the U-Boot region must start at position 7MB in the section. In this # case the ROM starts at 0xff800000, so the position of the first - # entry in the image corresponds to that. + # entry in the section corresponds to that. if (self.target_pos < self.pos or self.target_pos >= self.pos + self.size): self.Raise('Microcode pointer _dt_ucode_base_size at %08x is ' - 'outside the image ranging from %08x to %08x' % + 'outside the section ranging from %08x to %08x' % (self.target_pos, self.pos, self.pos + self.size)) # Get the microcode, either from u-boot-ucode or u-boot-dtb-with-ucode. @@ -72,7 +72,7 @@ class Entry_u_boot_with_ucode_ptr(Entry_blob): if ucode_entry.size: pos, size = ucode_entry.pos, ucode_entry.size else: - dtb_entry = self.image.FindEntryType('u-boot-dtb-with-ucode') + dtb_entry = self.section.FindEntryType('u-boot-dtb-with-ucode') if not dtb_entry: self.Raise('Cannot find microcode region u-boot-dtb-with-ucode') pos = dtb_entry.pos + dtb_entry.ucode_offset diff --git a/tools/binman/etype/x86_start16.py b/tools/binman/etype/x86_start16.py index 01e5b8bc38a..23d27f07040 100644 --- a/tools/binman/etype/x86_start16.py +++ b/tools/binman/etype/x86_start16.py @@ -9,8 +9,8 @@ from entry import Entry from blob import Entry_blob class Entry_x86_start16(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def GetDefaultFilename(self): return 'u-boot-x86-16bit.bin' diff --git a/tools/binman/etype/x86_start16_spl.py b/tools/binman/etype/x86_start16_spl.py index f0abbf038bd..176420ba88c 100644 --- a/tools/binman/etype/x86_start16_spl.py +++ b/tools/binman/etype/x86_start16_spl.py @@ -9,8 +9,8 @@ from entry import Entry from blob import Entry_blob class Entry_x86_start16_spl(Entry_blob): - def __init__(self, image, etype, node): - Entry_blob.__init__(self, image, etype, node) + def __init__(self, section, etype, node): + Entry_blob.__init__(self, section, etype, node) def GetDefaultFilename(self): return 'spl/u-boot-x86-16bit-spl.bin' diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index a3abbc4b84b..eb8a0793cbe 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -146,15 +146,19 @@ class TestFunctional(unittest.TestCase): # options.verbosity = tout.DEBUG return control.Binman(options, args) - def _DoTestFile(self, fname, debug=False): + def _DoTestFile(self, fname, debug=False, map=False): """Run binman with a given test file Args: - fname: Device tree source filename to use (e.g. 05_simple.dts) + fname: Device-tree source filename to use (e.g. 05_simple.dts) + debug: True to enable debugging output + map: True to output map files for the images """ args = ['-p', '-I', self._indir, '-d', self.TestFile(fname)] if debug: args.append('-D') + if map: + args.append('-m') return self._DoBinman(*args) def _SetupDtb(self, fname, outfile='u-boot.dtb'): @@ -165,10 +169,10 @@ class TestFunctional(unittest.TestCase): Args: fname: Filename of .dts file to read - outfile: Output filename for compiled device tree binary + outfile: Output filename for compiled device-tree binary Returns: - Contents of device tree binary + Contents of device-tree binary """ if not self._output_setup: tools.PrepareOutputDir(self._indir, True) @@ -179,7 +183,7 @@ class TestFunctional(unittest.TestCase): TestFunctional._MakeInputFile(outfile, data) return data - def _DoReadFileDtb(self, fname, use_real_dtb=False): + def _DoReadFileDtb(self, fname, use_real_dtb=False, map=False): """Run binman and return the resulting image This runs binman with a given test file and then reads the resulting @@ -189,16 +193,18 @@ class TestFunctional(unittest.TestCase): Raises an assertion failure if binman returns a non-zero exit code. Args: - fname: Device tree source filename to use (e.g. 05_simple.dts) + fname: Device-tree source filename to use (e.g. 05_simple.dts) use_real_dtb: True to use the test file as the contents of the u-boot-dtb entry. Normally this is not needed and the test contents (the U_BOOT_DTB_DATA string) can be used. But in some test we need the real contents. + map: True to output map files for the images Returns: Tuple: Resulting image contents Device tree contents + Map data showing contents of image (or None if none) """ dtb_data = None # Use the compiled test file as the u-boot-dtb input @@ -206,22 +212,36 @@ class TestFunctional(unittest.TestCase): dtb_data = self._SetupDtb(fname) try: - retcode = self._DoTestFile(fname) + retcode = self._DoTestFile(fname, map=map) self.assertEqual(0, retcode) # Find the (only) image, read it and return its contents image = control.images['image'] fname = tools.GetOutputFilename('image.bin') self.assertTrue(os.path.exists(fname)) + if map: + map_fname = tools.GetOutputFilename('image.map') + with open(map_fname) as fd: + map_data = fd.read() + else: + map_data = None with open(fname) as fd: - return fd.read(), dtb_data + return fd.read(), dtb_data, map_data finally: # Put the test file back if use_real_dtb: TestFunctional._MakeInputFile('u-boot.dtb', U_BOOT_DTB_DATA) def _DoReadFile(self, fname, use_real_dtb=False): - """Helper function which discards the device-tree binary""" + """Helper function which discards the device-tree binary + + Args: + fname: Device-tree source filename to use (e.g. 05_simple.dts) + use_real_dtb: True to use the test file as the contents of + the u-boot-dtb entry. Normally this is not needed and the + test contents (the U_BOOT_DTB_DATA string) can be used. + But in some test we need the real contents. + """ return self._DoReadFileDtb(fname, use_real_dtb)[0] @classmethod @@ -270,13 +290,13 @@ class TestFunctional(unittest.TestCase): pos += entry.size def GetFdtLen(self, dtb): - """Get the totalsize field from a device tree binary + """Get the totalsize field from a device-tree binary Args: - dtb: Device tree binary contents + dtb: Device-tree binary contents Returns: - Total size of device tree binary, from the header + Total size of device-tree binary, from the header """ return struct.unpack('>L', dtb[4:8])[0] @@ -326,7 +346,7 @@ class TestFunctional(unittest.TestCase): str(e.exception)) def testMissingDt(self): - """Test that an invalid device tree file generates an error""" + """Test that an invalid device-tree file generates an error""" with self.assertRaises(Exception) as e: self._RunBinman('-d', 'missing_file') # We get one error from libfdt, and a different one from fdtget. @@ -334,7 +354,7 @@ class TestFunctional(unittest.TestCase): 'No such file or directory'], str(e.exception)) def testBrokenDt(self): - """Test that an invalid device tree source file generates an error + """Test that an invalid device-tree source file generates an error Since this is a source file it should be compiled and the error will come from the device-tree compiler (dtc). @@ -413,7 +433,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(0, retcode) self.assertIn('image', control.images) image = control.images['image'] - entries = image._entries + entries = image.GetEntries() self.assertEqual(5, len(entries)) # First u-boot @@ -456,7 +476,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(0, retcode) self.assertIn('image', control.images) image = control.images['image'] - entries = image._entries + entries = image.GetEntries() self.assertEqual(5, len(entries)) # First u-boot with padding before and after @@ -540,7 +560,7 @@ class TestFunctional(unittest.TestCase): """Test that entries which overflow the image size are detected""" with self.assertRaises(ValueError) as e: self._DoTestFile('16_pack_image_overflow.dts') - self.assertIn("Image '/binman': contents size 0x4 (4) exceeds image " + self.assertIn("Section '/binman': contents size 0x4 (4) exceeds section " "size 0x3 (3)", str(e.exception)) def testPackImageSize(self): @@ -563,14 +583,14 @@ class TestFunctional(unittest.TestCase): """Test that invalid image alignment is detected""" with self.assertRaises(ValueError) as e: self._DoTestFile('19_pack_inv_image_align.dts') - self.assertIn("Image '/binman': Size 0x7 (7) does not match " + self.assertIn("Section '/binman': Size 0x7 (7) does not match " "align-size 0x8 (8)", str(e.exception)) def testPackAlignPowerOf2(self): """Test that invalid image alignment is detected""" with self.assertRaises(ValueError) as e: self._DoTestFile('20_pack_inv_image_align_power2.dts') - self.assertIn("Image '/binman': Alignment size 131 must be a power of " + self.assertIn("Section '/binman': Alignment size 131 must be a power of " "two", str(e.exception)) def testImagePadByte(self): @@ -620,7 +640,7 @@ class TestFunctional(unittest.TestCase): """Test that the end-at-4gb property requires a size property""" with self.assertRaises(ValueError) as e: self._DoTestFile('27_pack_4gb_no_size.dts') - self.assertIn("Image '/binman': Image size must be provided when " + self.assertIn("Section '/binman': Section size must be provided when " "using end-at-4gb", str(e.exception)) def testPackX86RomOutside(self): @@ -628,7 +648,7 @@ class TestFunctional(unittest.TestCase): with self.assertRaises(ValueError) as e: self._DoTestFile('28_pack_4gb_outside.dts') self.assertIn("Node '/binman/u-boot': Position 0x0 (0) is outside " - "the image starting at 0xffffffe0 (4294967264)", + "the section starting at 0xffffffe0 (4294967264)", str(e.exception)) def testPackX86Rom(self): @@ -800,13 +820,13 @@ class TestFunctional(unittest.TestCase): self._DoReadFile('40_x86_ucode_not_in_image.dts', True) self.assertIn("Node '/binman/u-boot-with-ucode-ptr': Microcode " "pointer _dt_ucode_base_size at fffffe14 is outside the " - "image ranging from 00000000 to 0000002e", str(e.exception)) + "section ranging from 00000000 to 0000002e", str(e.exception)) def testWithoutMicrocode(self): """Test that we can cope with an image without microcode (e.g. qemu)""" with open(self.TestFile('u_boot_no_ucode_ptr')) as fd: TestFunctional._MakeInputFile('u-boot', fd.read()) - data, dtb = self._DoReadFileDtb('44_x86_optional_ucode.dts', True) + data, dtb, _ = self._DoReadFileDtb('44_x86_optional_ucode.dts', True) # Now check the device tree has no microcode self.assertEqual(U_BOOT_NODTB_DATA, data[:len(U_BOOT_NODTB_DATA)]) @@ -823,7 +843,7 @@ class TestFunctional(unittest.TestCase): """Test that microcode must be placed within the image""" with self.assertRaises(ValueError) as e: self._DoReadFile('41_unknown_pos_size.dts', True) - self.assertIn("Image '/binman': Unable to set pos/size for unknown " + self.assertIn("Section '/binman': Unable to set pos/size for unknown " "entry 'invalid-entry'", str(e.exception)) def testPackFsp(self): @@ -909,6 +929,36 @@ class TestFunctional(unittest.TestCase): sym_values + U_BOOT_SPL_DATA[16:]) self.assertEqual(expected, data) + def testPackUnitAddress(self): + """Test that we support multiple binaries with the same name""" + data = self._DoReadFile('54_unit_address.dts') + self.assertEqual(U_BOOT_DATA + U_BOOT_DATA, data) + + def testSections(self): + """Basic test of sections""" + data = self._DoReadFile('55_sections.dts') + expected = U_BOOT_DATA + '!' * 12 + U_BOOT_DATA + 'a' * 12 + '&' * 8 + self.assertEqual(expected, data) + + def testMap(self): + """Tests outputting a map of the images""" + _, _, map_data = self._DoReadFileDtb('55_sections.dts', map=True) + self.assertEqual('''Position Size Name +00000000 00000010 section@0 + 00000000 00000004 u-boot +00000010 00000010 section@1 + 00000000 00000004 u-boot +''', map_data) + + def testNamePrefix(self): + """Tests that name prefixes are used""" + _, _, map_data = self._DoReadFileDtb('56_name_prefix.dts', map=True) + self.assertEqual('''Position Size Name +00000000 00000010 section@0 + 00000000 00000004 ro-u-boot +00000010 00000010 section@1 + 00000000 00000004 rw-u-boot +''', map_data) if __name__ == "__main__": unittest.main() diff --git a/tools/binman/image.py b/tools/binman/image.py index b10b1887957..835b66c99f5 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -13,6 +13,7 @@ import re import sys import fdt_util +import bsection import tools class Image: @@ -27,158 +28,36 @@ class Image: _node: Node object that contains the image definition in device tree _name: Image name _size: Image size in bytes, or None if not known yet - _align_size: Image size alignment, or None - _pad_before: Number of bytes before the first entry starts. This - effectively changes the place where entry position 0 starts - _pad_after: Number of bytes after the last entry ends. The last - entry will finish on or before this boundary - _pad_byte: Byte to use to pad the image where there is no entry _filename: Output filename for image - _sort: True if entries should be sorted by position, False if they - must be in-order in the device tree description - _skip_at_start: Number of bytes before the first entry starts. These - effecively adjust the starting position of entries. For example, - if _pad_before is 16, then the first entry would start at 16. - An entry with pos = 20 would in fact be written at position 4 - in the image file. - _end_4gb: Indicates that the image ends at the 4GB boundary. This is - used for x86 images, which want to use positions such that a - memory address (like 0xff800000) is the first entry position. - This causes _skip_at_start to be set to the starting memory - address. - _entries: OrderedDict() of entries + _sections: Sections present in this image (may be one or more) + + Args: + test: True if this is being called from a test of Images. This this case + there is no device tree defining the structure of the section, so + we create a section manually. """ def __init__(self, name, node, test=False): - global entry - global Entry - import entry - from entry import Entry - self._node = node self._name = name self._size = None - self._align_size = None - self._pad_before = 0 - self._pad_after = 0 - self._pad_byte = 0 self._filename = '%s.bin' % self._name - self._sort = False - self._skip_at_start = 0 - self._end_4gb = False - self._entries = OrderedDict() - - if not test: + if test: + self._section = bsection.Section('main-section', self._node, True) + else: self._ReadNode() - self._ReadEntries() def _ReadNode(self): """Read properties from the image node""" self._size = fdt_util.GetInt(self._node, 'size') - self._align_size = fdt_util.GetInt(self._node, 'align-size') - if tools.NotPowerOfTwo(self._align_size): - self._Raise("Alignment size %s must be a power of two" % - self._align_size) - self._pad_before = fdt_util.GetInt(self._node, 'pad-before', 0) - self._pad_after = fdt_util.GetInt(self._node, 'pad-after', 0) - self._pad_byte = fdt_util.GetInt(self._node, 'pad-byte', 0) filename = fdt_util.GetString(self._node, 'filename') if filename: self._filename = filename - self._sort = fdt_util.GetBool(self._node, 'sort-by-pos') - self._end_4gb = fdt_util.GetBool(self._node, 'end-at-4gb') - if self._end_4gb and not self._size: - self._Raise("Image size must be provided when using end-at-4gb") - if self._end_4gb: - self._skip_at_start = 0x100000000 - self._size - - def CheckSize(self): - """Check that the image contents does not exceed its size, etc.""" - contents_size = 0 - for entry in self._entries.values(): - contents_size = max(contents_size, entry.pos + entry.size) - - contents_size -= self._skip_at_start - - size = self._size - if not size: - size = self._pad_before + contents_size + self._pad_after - size = tools.Align(size, self._align_size) - - if self._size and contents_size > self._size: - self._Raise("contents size %#x (%d) exceeds image size %#x (%d)" % - (contents_size, contents_size, self._size, self._size)) - if not self._size: - self._size = size - if self._size != tools.Align(self._size, self._align_size): - self._Raise("Size %#x (%d) does not match align-size %#x (%d)" % - (self._size, self._size, self._align_size, self._align_size)) - - def _Raise(self, msg): - """Raises an error for this image - - Args: - msg: Error message to use in the raise string - Raises: - ValueError() - """ - raise ValueError("Image '%s': %s" % (self._node.path, msg)) - - def GetPath(self): - """Get the path of an image (in the FDT) - - Returns: - Full path of the node for this image - """ - return self._node.path - - def _ReadEntries(self): - for node in self._node.subnodes: - self._entries[node.name] = Entry.Create(self, node) - - def FindEntryType(self, etype): - """Find an entry type in the image - - Args: - etype: Entry type to find - Returns: - entry matching that type, or None if not found - """ - for entry in self._entries.values(): - if entry.etype == etype: - return entry - return None + self._section = bsection.Section('main-section', self._node) def GetEntryContents(self): - """Call ObtainContents() for each entry - - This calls each entry's ObtainContents() a few times until they all - return True. We stop calling an entry's function once it returns - True. This allows the contents of one entry to depend on another. - - After 3 rounds we give up since it's likely an error. + """Call ObtainContents() for the section """ - todo = self._entries.values() - for passnum in range(3): - next_todo = [] - for entry in todo: - if not entry.ObtainContents(): - next_todo.append(entry) - todo = next_todo - if not todo: - break - - def _SetEntryPosSize(self, name, pos, size): - """Set the position and size of an entry - - Args: - name: Entry name to update - pos: New position - size: New size - """ - entry = self._entries.get(name) - if not entry: - self._Raise("Unable to set pos/size for unknown entry '%s'" % name) - entry.SetPositionSize(self._skip_at_start + pos, size) + self._section.GetEntryContents() def GetEntryPositions(self): """Handle entries that want to set the position/size of other entries @@ -186,119 +65,44 @@ class Image: This calls each entry's GetPositions() method. If it returns a list of entries to update, it updates them. """ - for entry in self._entries.values(): - pos_dict = entry.GetPositions() - for name, info in pos_dict.iteritems(): - self._SetEntryPosSize(name, *info) + self._section.GetEntryPositions() def PackEntries(self): """Pack all entries into the image""" - pos = self._skip_at_start - for entry in self._entries.values(): - pos = entry.Pack(pos) + self._section.PackEntries() - def _SortEntries(self): - """Sort entries by position""" - entries = sorted(self._entries.values(), key=lambda entry: entry.pos) - self._entries.clear() - for entry in entries: - self._entries[entry._node.name] = entry + def CheckSize(self): + """Check that the image contents does not exceed its size, etc.""" + self._size = self._section.CheckSize() def CheckEntries(self): """Check that entries do not overlap or extend outside the image""" - if self._sort: - self._SortEntries() - pos = 0 - prev_name = 'None' - for entry in self._entries.values(): - if (entry.pos < self._skip_at_start or - entry.pos >= self._skip_at_start + self._size): - entry.Raise("Position %#x (%d) is outside the image starting " - "at %#x (%d)" % - (entry.pos, entry.pos, self._skip_at_start, - self._skip_at_start)) - if entry.pos < pos: - entry.Raise("Position %#x (%d) overlaps with previous entry '%s' " - "ending at %#x (%d)" % - (entry.pos, entry.pos, prev_name, pos, pos)) - pos = entry.pos + entry.size - prev_name = entry.GetPath() + self._section.CheckEntries() def ProcessEntryContents(self): """Call the ProcessContents() method for each entry This is intended to adjust the contents as needed by the entry type. """ - for entry in self._entries.values(): - entry.ProcessContents() + self._section.ProcessEntryContents() def WriteSymbols(self): """Write symbol values into binary files for access at run time""" - for entry in self._entries.values(): - entry.WriteSymbols(self) + self._section.WriteSymbols() def BuildImage(self): """Write the image to a file""" fname = tools.GetOutputFilename(self._filename) with open(fname, 'wb') as fd: - fd.write(chr(self._pad_byte) * self._size) - - for entry in self._entries.values(): - data = entry.GetData() - fd.seek(self._pad_before + entry.pos - self._skip_at_start) - fd.write(data) - - def LookupSymbol(self, sym_name, optional, msg): - """Look up a symbol in an ELF file - - Looks up a symbol in an ELF file. Only entry types which come from an - ELF image can be used by this function. - - At present the only entry property supported is pos. - - Args: - sym_name: Symbol name in the ELF file to look up in the format - _binman_<entry>_prop_<property> where <entry> is the name of - the entry and <property> is the property to find (e.g. - _binman_u_boot_prop_pos). As a special case, you can append - _any to <entry> to have it search for any matching entry. E.g. - _binman_u_boot_any_prop_pos will match entries called u-boot, - u-boot-img and u-boot-nodtb) - optional: True if the symbol is optional. If False this function - will raise if the symbol is not found - msg: Message to display if an error occurs - - Returns: - Value that should be assigned to that symbol, or None if it was - optional and not found - - Raises: - ValueError if the symbol is invalid or not found, or references a - property which is not supported - """ - m = re.match(r'^_binman_(\w+)_prop_(\w+)$', sym_name) - if not m: - raise ValueError("%s: Symbol '%s' has invalid format" % - (msg, sym_name)) - entry_name, prop_name = m.groups() - entry_name = entry_name.replace('_', '-') - entry = self._entries.get(entry_name) - if not entry: - if entry_name.endswith('-any'): - root = entry_name[:-4] - for name in self._entries: - if name.startswith(root): - rest = name[len(root):] - if rest in ['', '-img', '-nodtb']: - entry = self._entries[name] - if not entry: - err = ("%s: Entry '%s' not found in list (%s)" % - (msg, entry_name, ','.join(self._entries.keys()))) - if optional: - print('Warning: %s' % err, file=sys.stderr) - return None - raise ValueError(err) - if prop_name == 'pos': - return entry.pos - else: - raise ValueError("%s: No such property '%s'" % (msg, prop_name)) + self._section.BuildSection(fd, 0) + + def GetEntries(self): + return self._section.GetEntries() + + def WriteMap(self): + """Write a map of the image to a .map file""" + filename = '%s.map' % self._name + fname = tools.GetOutputFilename(filename) + with open(fname, 'w') as fd: + print('%8s %8s %s' % ('Position', 'Size', 'Name'), file=fd) + self._section.WriteMap(fd, 0) diff --git a/tools/binman/image_test.py b/tools/binman/image_test.py index 44a5a2c010f..45dd2378c8a 100644 --- a/tools/binman/image_test.py +++ b/tools/binman/image_test.py @@ -12,25 +12,28 @@ from elf_test import capture_sys_output class TestImage(unittest.TestCase): def testInvalidFormat(self): image = Image('name', 'node', test=True) + section = image._section with self.assertRaises(ValueError) as e: - image.LookupSymbol('_binman_something_prop_', False, 'msg') + section.LookupSymbol('_binman_something_prop_', False, 'msg') self.assertIn( "msg: Symbol '_binman_something_prop_' has invalid format", str(e.exception)) def testMissingSymbol(self): image = Image('name', 'node', test=True) - image._entries = {} + section = image._section + section._entries = {} with self.assertRaises(ValueError) as e: - image.LookupSymbol('_binman_type_prop_pname', False, 'msg') + section.LookupSymbol('_binman_type_prop_pname', False, 'msg') self.assertIn("msg: Entry 'type' not found in list ()", str(e.exception)) def testMissingSymbolOptional(self): image = Image('name', 'node', test=True) - image._entries = {} + section = image._section + section._entries = {} with capture_sys_output() as (stdout, stderr): - val = image.LookupSymbol('_binman_type_prop_pname', True, 'msg') + val = section.LookupSymbol('_binman_type_prop_pname', True, 'msg') self.assertEqual(val, None) self.assertEqual("Warning: msg: Entry 'type' not found in list ()\n", stderr.getvalue()) @@ -38,7 +41,8 @@ class TestImage(unittest.TestCase): def testBadProperty(self): image = Image('name', 'node', test=True) - image._entries = {'u-boot': 1} + section = image._section + section._entries = {'u-boot': 1} with self.assertRaises(ValueError) as e: - image.LookupSymbol('_binman_u_boot_prop_bad', False, 'msg') + section.LookupSymbol('_binman_u_boot_prop_bad', False, 'msg') self.assertIn("msg: No such property 'bad", str(e.exception)) diff --git a/tools/binman/test/54_unit_address.dts b/tools/binman/test/54_unit_address.dts new file mode 100644 index 00000000000..3216dbbcc19 --- /dev/null +++ b/tools/binman/test/54_unit_address.dts @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot@0 { + }; + u-boot@1 { + }; + }; +}; diff --git a/tools/binman/test/55_sections.dts b/tools/binman/test/55_sections.dts new file mode 100644 index 00000000000..2ada395b036 --- /dev/null +++ b/tools/binman/test/55_sections.dts @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + pad-byte = <0x26>; + size = <0x28>; + section@0 { + read-only; + size = <0x10>; + pad-byte = <0x21>; + + u-boot { + }; + }; + section@1 { + size = <0x10>; + pad-byte = <0x61>; + + u-boot { + }; + }; + }; +}; diff --git a/tools/binman/test/56_name_prefix.dts b/tools/binman/test/56_name_prefix.dts new file mode 100644 index 00000000000..f38c80eb183 --- /dev/null +++ b/tools/binman/test/56_name_prefix.dts @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + pad-byte = <0x26>; + size = <0x28>; + section@0 { + read-only; + name-prefix = "ro-"; + size = <0x10>; + pad-byte = <0x21>; + + u-boot { + }; + }; + section@1 { + name-prefix = "rw-"; + size = <0x10>; + pad-byte = <0x61>; + + u-boot { + }; + }; + }; +}; diff --git a/tools/buildman/README b/tools/buildman/README index 6c43c546f87..76601902cb2 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -1008,6 +1008,34 @@ variables. This avoids lots of useless output when converting a CONFIG option to Kconfig. To disable this behaviour, use --squash-config-y. +Checking the environment +======================== + +When converting CONFIG options which manipulate the default environment, +a common requirement is to check that the default environment has not +changed due to the conversion. Buildman supports this with the -U option, +used after a build. This shows differences in the default environment +between one commit and the next. + +For example: + +$ buildman -b squash brppt1 -sU +boards.cfg is up to date. Nothing to do. +Summary of 2 commits for 3 boards (3 threads, 3 jobs per thread) +01: Migrate bootlimit to Kconfig +02: Squashed commit of the following: + c brppt1_mmc: altbootcmd=mmc dev 1; run mmcboot0; -> mmc dev 1; run mmcboot0 + c brppt1_spi: altbootcmd=mmc dev 1; run mmcboot0; -> mmc dev 1; run mmcboot0 + + brppt1_nand: altbootcmd=run usbscript + - brppt1_nand: altbootcmd=run usbscript +(no errors to report) + +This shows that commit 2 modified the value of 'altbootcmd' for 'brppt1_mmc' +and 'brppt1_spi', removing a trailing semicolon. 'brppt1_nand' gained an a +value for 'altbootcmd', but lost one for ' altbootcmd'. + +The -U option uses the u-boot.env files which are produced by a build. + Other options ============= diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 2ccdee02a3c..a5a2ffdfdf2 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -127,6 +127,15 @@ class Config: val = val ^ hash(key) & hash(value) return val +class Environment: + """Holds information about environment variables for a board.""" + def __init__(self, target): + self.target = target + self.environment = {} + + def Add(self, key, value): + self.environment[key] = value + class Builder: """Class for building U-Boot for a particular commit. @@ -199,13 +208,17 @@ class Builder: value is itself a dictionary: key: config name value: config value + environment: Dictionary keyed by environment variable, Each + value is the value of environment variable. """ - def __init__(self, rc, err_lines, sizes, func_sizes, config): + def __init__(self, rc, err_lines, sizes, func_sizes, config, + environment): self.rc = rc self.err_lines = err_lines self.sizes = sizes self.func_sizes = func_sizes self.config = config + self.environment = environment def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, gnu_make='make', checkout=True, show_unknown=True, step=1, @@ -310,7 +323,8 @@ class Builder: def SetDisplayOptions(self, show_errors=False, show_sizes=False, show_detail=False, show_bloat=False, - list_error_boards=False, show_config=False): + list_error_boards=False, show_config=False, + show_environment=False): """Setup display options for the builder. show_errors: True to show summarised error/warning info @@ -319,6 +333,7 @@ class Builder: show_bloat: Show detail for each function list_error_boards: Show the boards which caused each error/warning show_config: Show config deltas + show_environment: Show environment deltas """ self._show_errors = show_errors self._show_sizes = show_sizes @@ -326,6 +341,7 @@ class Builder: self._show_bloat = show_bloat self._list_error_boards = list_error_boards self._show_config = show_config + self._show_environment = show_environment def _AddTimestamp(self): """Add a new timestamp to the list and record the build period. @@ -609,8 +625,33 @@ class Builder: config[key] = value return config + def _ProcessEnvironment(self, fname): + """Read in a uboot.env file + + This function reads in environment variables from a file. + + Args: + fname: Filename to read + + Returns: + Dictionary: + key: environment variable (e.g. bootlimit) + value: value of environment variable (e.g. 1) + """ + environment = {} + if os.path.exists(fname): + with open(fname) as fd: + for line in fd.read().split('\0'): + try: + key, value = line.split('=', 1) + environment[key] = value + except ValueError: + # ignore lines we can't parse + pass + return environment + def GetBuildOutcome(self, commit_upto, target, read_func_sizes, - read_config): + read_config, read_environment): """Work out the outcome of a build. Args: @@ -618,6 +659,7 @@ class Builder: target: Target board to check read_func_sizes: True to read function size information read_config: True to read .config and autoconf.h files + read_environment: True to read uboot.env files Returns: Outcome object @@ -627,6 +669,7 @@ class Builder: sizes = {} func_sizes = {} config = {} + environment = {} if os.path.exists(done_file): with open(done_file, 'r') as fd: return_code = int(fd.readline()) @@ -676,12 +719,18 @@ class Builder: fname = os.path.join(output_dir, name) config[name] = self._ProcessConfig(fname) - return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) + if read_environment: + output_dir = self.GetBuildDir(commit_upto, target) + fname = os.path.join(output_dir, 'uboot.env') + environment = self._ProcessEnvironment(fname) + + return Builder.Outcome(rc, err_lines, sizes, func_sizes, config, + environment) - return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) + return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {}) def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, - read_config): + read_config, read_environment): """Calculate a summary of the results of building a commit. Args: @@ -689,6 +738,7 @@ class Builder: commit_upto: Commit number to summarize (0..self.count-1) read_func_sizes: True to read function size information read_config: True to read .config and autoconf.h files + read_environment: True to read uboot.env files Returns: Tuple: @@ -705,6 +755,9 @@ class Builder: value is itself a dictionary: key: config name value: config value + Dictionary keyed by board.target. Each value is a dictionary: + key: environment variable + value: value of environment variable """ def AddLine(lines_summary, lines_boards, line, board): line = line.rstrip() @@ -720,10 +773,12 @@ class Builder: warn_lines_summary = [] warn_lines_boards = {} config = {} + environment = {} for board in boards_selected.itervalues(): outcome = self.GetBuildOutcome(commit_upto, board.target, - read_func_sizes, read_config) + read_func_sizes, read_config, + read_environment) board_dict[board.target] = outcome last_func = None last_was_warning = False @@ -756,8 +811,14 @@ class Builder: tconfig.Add(fname, key, value) config[board.target] = tconfig + tenvironment = Environment(board.target) + if outcome.environment: + for key, value in outcome.environment.iteritems(): + tenvironment.Add(key, value) + environment[board.target] = tenvironment + return (board_dict, err_lines_summary, err_lines_boards, - warn_lines_summary, warn_lines_boards, config) + warn_lines_summary, warn_lines_boards, config, environment) def AddOutcome(self, board_dict, arch_list, changes, char, color): """Add an output to our list of outcomes for each architecture @@ -810,12 +871,14 @@ class Builder: """ self._base_board_dict = {} for board in board_selected: - self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) + self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}, + {}) self._base_err_lines = [] self._base_warn_lines = [] self._base_err_line_boards = {} self._base_warn_line_boards = {} self._base_config = None + self._base_environment = None def PrintFuncSizeDetail(self, fname, old, new): grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 @@ -1010,8 +1073,8 @@ class Builder: def PrintResultSummary(self, board_selected, board_dict, err_lines, err_line_boards, warn_lines, warn_line_boards, - config, show_sizes, show_detail, show_bloat, - show_config): + config, environment, show_sizes, show_detail, + show_bloat, show_config, show_environment): """Compare results with the base results and display delta. Only boards mentioned in board_selected will be considered. This @@ -1036,10 +1099,13 @@ class Builder: value is itself a dictionary: key: config name value: config value + environment: Dictionary keyed by environment variable, Each + value is the value of environment variable. show_sizes: Show image size deltas show_detail: Show detail for each board show_bloat: Show detail for each function show_config: Show config changes + show_environment: Show environment changes """ def _BoardList(line, line_boards): """Helper function to get a line of boards containing a line @@ -1188,6 +1254,36 @@ class Builder: self.PrintSizeSummary(board_selected, board_dict, show_detail, show_bloat) + if show_environment and self._base_environment: + lines = [] + + for target in board_dict: + if target not in board_selected: + continue + + tbase = self._base_environment[target] + tenvironment = environment[target] + environment_plus = {} + environment_minus = {} + environment_change = {} + base = tbase.environment + for key, value in tenvironment.environment.iteritems(): + if key not in base: + environment_plus[key] = value + for key, value in base.iteritems(): + if key not in tenvironment.environment: + environment_minus[key] = value + for key, value in base.iteritems(): + new_value = tenvironment.environment.get(key) + if new_value and value != new_value: + desc = '%s -> %s' % (value, new_value) + environment_change[key] = desc + + _AddConfig(lines, target, environment_plus, environment_minus, + environment_change) + + _OutputConfigInfo(lines) + if show_config and self._base_config: summary = {} arch_config_plus = {} @@ -1294,6 +1390,7 @@ class Builder: self._base_err_line_boards = err_line_boards self._base_warn_line_boards = warn_line_boards self._base_config = config + self._base_environment = environment # Get a list of boards that did not get built, if needed not_built = [] @@ -1306,10 +1403,11 @@ class Builder: def ProduceResultSummary(self, commit_upto, commits, board_selected): (board_dict, err_lines, err_line_boards, warn_lines, - warn_line_boards, config) = self.GetResultSummary( + warn_line_boards, config, environment) = self.GetResultSummary( board_selected, commit_upto, read_func_sizes=self._show_bloat, - read_config=self._show_config) + read_config=self._show_config, + read_environment=self._show_environment) if commits: msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) @@ -1317,8 +1415,8 @@ class Builder: self.PrintResultSummary(board_selected, board_dict, err_lines if self._show_errors else [], err_line_boards, warn_lines if self._show_errors else [], warn_line_boards, - config, self._show_sizes, self._show_detail, - self._show_bloat, self._show_config) + config, environment, self._show_sizes, self._show_detail, + self._show_bloat, self._show_config, self._show_environment) def ShowSummary(self, commits, board_selected): """Show a build summary for U-Boot for a given board list. diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 0efe80d9457..c84ba6acf11 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -351,6 +351,16 @@ class BuilderThread(threading.Thread): lines.append(size_result.stdout.splitlines()[1] + ' ' + rodata_size) + # Extract the environment from U-Boot and dump it out + cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary', + '-j', '.rodata.default_environment', + 'env/built-in.o', 'uboot.env'] + command.RunPipe([cmd], capture=True, + capture_stderr=True, cwd=result.out_dir, + raise_on_error=False, env=env) + ubootenv = os.path.join(result.out_dir, 'uboot.env') + self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env']) + # Write out the image sizes file. This is similar to the output # of binutil's 'size' utility, but it omits the header line and # adds an additional hex value at the end of each line for the diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index b8ddd4795c6..e493b1ac4a0 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -92,6 +92,8 @@ def ParseArgs(): default=None, help='Number of builder threads to use') parser.add_option('-u', '--show_unknown', action='store_true', default=False, help='Show boards with unknown build result') + parser.add_option('-U', '--show-environment', action='store_true', + default=False, help='Show environment changes in summary') parser.add_option('-v', '--verbose', action='store_true', default=False, help='Show build results while the build progresses') parser.add_option('-V', '--verbose-build', action='store_true', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 4ac4386db62..bc0819784f8 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -319,7 +319,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, builder.SetDisplayOptions(options.show_errors, options.show_sizes, options.show_detail, options.show_bloat, options.list_error_boards, - options.show_config) + options.show_config, + options.show_environment) if options.summary: builder.ShowSummary(commits, board_selected) else: diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 9206fb299d4..363db9d8ce2 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -327,6 +327,9 @@ class TestFunctional(unittest.TestCase): def _HandleCommandObjdump(self, args): return command.CommandResult(return_code=0) + def _HandleCommandObjcopy(self, args): + return command.CommandResult(return_code=0) + def _HandleCommandSize(self, args): return command.CommandResult(return_code=0) @@ -359,6 +362,8 @@ class TestFunctional(unittest.TestCase): return self._HandleCommandNm(args) elif cmd.endswith('objdump'): return self._HandleCommandObjdump(args) + elif cmd.endswith('objcopy'): + return self._HandleCommandObjcopy(args) elif cmd.endswith( 'size'): return self._HandleCommandSize(args) diff --git a/tools/patman/test.py b/tools/patman/test.py index 343efc99bd8..c7ba4e6bb47 100644 --- a/tools/patman/test.py +++ b/tools/patman/test.py @@ -148,7 +148,7 @@ index 0000000..2234c87 --- /dev/null +++ b/common/bootstage.c @@ -0,0 +1,37 @@ -+/* SPDX-License-Identifier: GPL-2.0+ */ ++// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2011, Google Inc. All rights reserved. + * |