Romlib makefile refactoring and script rewriting

The features of the previously existing gentbl, genvar and genwrappers
scripts were reimplemented in the romlib_generator.py Python script.
This resulted in more readable and maintainable code and the script
introduces additional features that help dependency handling in
makefiles. The assembly templates were separated from the script logic
and were collected in the 'templates' directory.

The targets and their dependencies were reorganized in the makefile and
the dependency handling of included index files is possible now.
Incremental build is available in case of modifying the index files.

Signed-off-by: Imre Kis <imre.kis@arm.com>
Change-Id: I79f65fab9dc5c70d1f6fc8f57b2a3009bf842dc5

13 files changed, 370 insertions, 179 deletions

diff --git a/lib/romlib/Makefile b/lib/romlib/Makefile
index bc05d0fa..60c14580 100644
--- a/lib/romlib/Makefile
+++ b/lib/romlib/Makefile
@@ -5,9 +5,11 @@
 #
 
 AS          = $(CROSS_COMPILE)as
+AR          = $(CROSS_COMPILE)ar
 LD          = $(CROSS_COMPILE)ld
 OC          = $(CROSS_COMPILE)objcopy
 CPP         = $(CROSS_COMPILE)cpp
+ROMLIB_GEN  = ./romlib_generator.py
 BUILD_DIR   = ../../$(BUILD_PLAT)/romlib
 LIB_DIR     = ../../$(BUILD_PLAT)/lib
 WRAPPER_DIR = ../../$(BUILD_PLAT)/libwrapper
@@ -17,6 +19,11 @@ PPFLAGS     = $(INC) $(DEFINES) -P -D__ASSEMBLY__ -D__LINKER__ -MD -MP -MT $(BUI
 OBJS        = $(BUILD_DIR)/jmptbl.o $(BUILD_DIR)/init.o
 MAPFILE     = ../../$(BUILD_PLAT)/romlib/romlib.map
 
+ifneq ($(PLAT_DIR),)
+  WRAPPER_SOURCES   = $(shell $(ROMLIB_GEN) genwrappers -b $(WRAPPER_DIR) --list ../../$(PLAT_DIR)/jmptbl.i)
+  WRAPPER_OBJS      = $(WRAPPER_SOURCES:.s=.o)
+endif
+
 V ?= 0
 ifeq ($(V),0)
   Q := @
@@ -61,19 +68,31 @@ $(BUILD_DIR)/romlib.bin: $(BUILD_DIR)/romlib.elf
 
 $(WRAPPER_DIR)/jmpvar.s: $(BUILD_DIR)/romlib.elf
 	@echo "  VAR     $@"
-	$(Q)./genvar.sh -o $@ $(BUILD_DIR)/romlib.elf
+	$(Q)$(ROMLIB_GEN) genvar --output $@ $<
 
-$(LIB_DIR)/libwrappers.a: $(BUILD_DIR)/jmptbl.i $(WRAPPER_DIR)/jmpvar.o
+$(LIB_DIR)/libwrappers.a: $(WRAPPER_DIR)/jmpvar.o $(WRAPPER_OBJS)
 	@echo "  AR      $@"
-	$(Q)./genwrappers.sh -b $(WRAPPER_DIR) -o $@ --bti=$(ENABLE_BTI) --asflags=$(ASFLAGS) $(BUILD_DIR)/jmptbl.i
+	$(Q)$(AR) -rc $@ $(WRAPPER_DIR)/jmpvar.o $(WRAPPER_OBJS)
+
+$(BUILD_DIR)/jmptbl.i: ../../$(PLAT_DIR)/jmptbl.i
+	@echo "  PRE     $@"
+	$(Q)$(ROMLIB_GEN) pre --output $@ --deps $(BUILD_DIR)/jmptbl.d $<
+
+$(BUILD_DIR)/wrappers.stamp: $(BUILD_DIR)/jmptbl.i
+	@echo "  WRP     $<"
+	$(Q)$(ROMLIB_GEN) genwrappers --bti=$(ENABLE_BTI) -b $(WRAPPER_DIR) $<
+	@touch $@
+
+$(WRAPPER_SOURCES): $(BUILD_DIR)/wrappers.stamp
 
-$(BUILD_DIR)/jmptbl.i: $(BUILD_DIR)/jmptbl.s
+$(WRAPPER_OBJS): $(WRAPPER_SOURCES) $(BUILD_DIR)/wrappers.stamp
 
-$(BUILD_DIR)/jmptbl.s: ../../$(PLAT_DIR)/jmptbl.i
+$(BUILD_DIR)/jmptbl.s: $(BUILD_DIR)/jmptbl.i
 	@echo "  TBL     $@"
-	$(Q)./gentbl.sh -o $@ -b $(BUILD_DIR) --bti=$(ENABLE_BTI) ../../$(PLAT_DIR)/jmptbl.i
+	$(Q)$(ROMLIB_GEN) gentbl --output $@ --bti=$(ENABLE_BTI) $<
 
 clean:
 	@rm -f $(BUILD_DIR)/*
 
 -include $(BUILD_DIR)/romlib.d
+-include $(BUILD_DIR)/jmptbl.d
diff --git a/lib/romlib/gentbl.sh b/lib/romlib/gentbl.sh
deleted file mode 100755
index bfb1ec3cf..00000000
--- a/lib/romlib/gentbl.sh
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/bin/sh
-# Copyright (c) 2018-2019, ARM Limited and Contributors. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-set -e
-
-output=jmptbl.s
-build=.
-
-for i
-do
-	case $i in
-	-o)
-		output=$2
-		shift 2
-		;;
-	-b)
-		build=$2
-		shift 2
-		;;
-	--bti=*)
-		enable_bti=$(echo $1 | sed 's/--bti=\(.*\)/\1/')
-		shift 1
-		;;
-	--)
-		shift
-		break
-		;;
-	-*)
-		echo usage: gentbl.sh [-o output] [-b dir] file ... >&2
-		exit 1
-		;;
-	esac
-done
-
-tmp=`mktemp`
-trap "rm -f $$.tmp" EXIT INT QUIT
-rm -f $output
-
-# Pre-process include files
-awk '!/^$/ && !/[:blank:]*#.*/{
-if (NF == 2 && $1 == "include") {
-	while ((getline line < $2) > 0)
-		if (line !~ /^$/ && line !~ /[:blank:]*#.*/)
-			print line
-		close($2)
-} else
-	print
-}' "$@" |
-awk -v OFS="\t" '
-BEGIN{print "#index\tlib\tfunction\t[patch]"}
-{print NR-1, $0}' | tee $build/jmptbl.i |
-awk -v OFS="\n" -v BTI=$enable_bti '
-BEGIN {print "\t.text",
-             "\t.globl\tjmptbl",
-             "jmptbl:"}
-      {sub(/[:blank:]*#.*/,"")}
-!/^$/ {
-	if (BTI == 1)
-		print "\tbti\tj"
-	if ($3 == "reserved")
-		print "\t.word\t0x0"
-	else
-		print "\tb\t" $3}' > $$.tmp &&
-mv $$.tmp $output
diff --git a/lib/romlib/genvar.sh b/lib/romlib/genvar.sh
deleted file mode 100755
index a3e2cdf6..00000000
--- a/lib/romlib/genvar.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-# Copyright (c) 2018, ARM Limited and Contributors. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-set -e
-
-output=jmpvar.s
-for i
-do
-	case $i in
-	-o)
-		output=$2
-		shift 2
-		;;
-	--)
-		shift
-		break
-		;;
-	-*)
-		echo usage: genvar.sh [-o output] file... >&2
-		;;
-	esac
-done
-
-tmp=`mktemp`
-trap "rm -f $tmp" EXIT INT QUIT
-
-nm -a "$@" |
-awk -v OFS="\n" '
-$3 == ".text" {print "\t.data",
-                     "\t.globl\tjmptbl",
-                     "\t.align\t4",
-                     "jmptbl:\t.quad\t0x" $1}' > $tmp
-
-mv $tmp $output
diff --git a/lib/romlib/genwrappers.sh b/lib/romlib/genwrappers.sh
deleted file mode 100755
index e092548e..00000000
--- a/lib/romlib/genwrappers.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/sh
-# Copyright (c) 2018-2019, ARM Limited and Contributors. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-set -e
-
-build=.
-out=output.a
-
-for i
-do
-	case $i in
-	-o)
-		out=$2
-		shift 2
-		;;
-	-b)
-		build=$2
-		shift 2
-		;;
-	--bti=*)
-		enable_bti=$(echo $1 | sed 's/--bti=\(.*\)/\1/')
-		shift 1
-		;;
-	--asflags=*)
-		asflags=$(echo $1 | sed 's/--asflags=\(.*\)/\1/')
-		shift 1
-		;;
-	--)
-		shift
-		break
-		;;
-	-*)
-		echo usage: genwrappers.sh [-o output] [-b dir] file ... >&2
-		exit 1
-		;;
-	esac
-done
-
-awk -v BTI=$enable_bti '
-{sub(/[:blank:]*#.*/,"")}
-!/^$/ && $NF != "patch" && $NF != "reserved" {
-		if (BTI == 1)
-			print $1*8, $2, $3
-		else
-			print $1*4, $2, $3}' "$@" |
-while read idx lib sym
-do
-	file=$build/${lib}_$sym
-
-	cat <<EOF > $file.s
-	.globl	$sym
-$sym:
-EOF
-if [ $enable_bti = 1 ]
-then
-	echo "\tbti\tjc" >> $file.s
-fi
-	cat <<EOF >> $file.s
-	ldr	x17, =jmptbl
-	mov	x16, #$idx
-	ldr	x17, [x17]
-	add	x16, x16, x17
-	br	x16
-EOF
-
-	${CROSS_COMPILE}as ${asflags} -o $file.o $file.s
-done
-
-${CROSS_COMPILE}ar -rc $out $build/*.o
diff --git a/lib/romlib/romlib_generator.py b/lib/romlib/romlib_generator.py
new file mode 100755
index 00000000..0682dd49
--- /dev/null
+++ b/lib/romlib/romlib_generator.py
@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""
+This module contains a set of classes and a runner that can generate code for the romlib module
+based on the templates in the 'templates' directory.
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import string
+import sys
+
+class IndexFileParser:
+    """
+    Parses the contents of the index file into the items and dependencies variables. It
+    also resolves included files in the index files recursively with circular inclusion detection.
+    """
+
+    def __init__(self):
+        self.items = []
+        self.dependencies = {}
+        self.include_chain = []
+
+    def add_dependency(self, parent, dependency):
+        """ Adds a dependency into the dependencies variable. """
+        if parent in self.dependencies:
+            self.dependencies[parent].append(dependency)
+        else:
+            self.dependencies[parent] = [dependency]
+
+    def get_dependencies(self, parent):
+        """ Gets all the recursive dependencies of a parent file. """
+        parent = os.path.normpath(parent)
+        if parent in self.dependencies:
+            direct_deps = self.dependencies[parent]
+            deps = direct_deps
+            for direct_dep in direct_deps:
+                deps += self.get_dependencies(direct_dep)
+            return deps
+
+        return []
+
+    def parse(self, file_name):
+        """ Opens and parses index file. """
+        file_name = os.path.normpath(file_name)
+
+        if file_name not in self.include_chain:
+            self.include_chain.append(file_name)
+            self.dependencies[file_name] = []
+        else:
+            raise Exception("Circular dependency detected: " + file_name)
+
+        with open(file_name, "r") as index_file:
+            for line in index_file.readlines():
+                line_elements = line.split()
+
+                if line.startswith("#") or not line_elements:
+                    # Comment or empty line
+                    continue
+
+                if line_elements[0] == "reserved":
+                    # Reserved slot in the jump table
+                    self.items.append({"type": "reserved"})
+                elif line_elements[0] == "include" and len(line_elements) > 1:
+                    # Include other index file
+                    included_file = os.path.normpath(line_elements[1])
+                    self.add_dependency(file_name, included_file)
+                    self.parse(included_file)
+                elif len(line_elements) > 1:
+                    # Library function
+                    library_name = line_elements[0]
+                    function_name = line_elements[1]
+                    patch = bool(len(line_elements) > 2 and line_elements[2] == "patch")
+
+                    self.items.append({"type": "function", "library_name": library_name,
+                                       "function_name": function_name, "patch": patch})
+                else:
+                    raise Exception("Invalid line: '" + line + "'")
+
+        self.include_chain.pop()
+
+class RomlibApplication:
+    """ Base class of romlib applications. """
+    TEMPLATE_DIR = os.path.dirname(os.path.realpath(__file__)) + "/templates/"
+
+    def __init__(self, prog):
+        self.args = argparse.ArgumentParser(prog=prog, description=self.__doc__)
+        self.config = None
+
+    def parse_arguments(self, argv):
+        """ Parses the arguments that should come from the command line arguments. """
+        self.config = self.args.parse_args(argv)
+
+    def build_template(self, name, mapping=None, remove_comment=False):
+        """
+        Loads a template and builds it with the defined mapping. Template paths are always relative
+        to this script.
+        """
+
+        with open(self.TEMPLATE_DIR + name, "r") as template_file:
+            if remove_comment:
+                # Removing copyright comment to make the generated code more readable when the
+                # template is inserted multiple times into the output.
+                template_lines = template_file.readlines()
+                end_of_comment_line = 0
+                for index, line in enumerate(template_lines):
+                    if line.find("*/") != -1:
+                        end_of_comment_line = index
+                        break
+                template_data = "".join(template_lines[end_of_comment_line + 1:])
+            else:
+                template_data = template_file.read()
+
+            template = string.Template(template_data)
+            return template.substitute(mapping)
+
+class IndexPreprocessor(RomlibApplication):
+    """ Removes empty and comment lines from the index file and resolves includes. """
+
+    def __init__(self, prog):
+        RomlibApplication.__init__(self, prog)
+
+        self.args.add_argument("-o", "--output", help="Output file", metavar="output",
+                               default="jmpvar.s")
+        self.args.add_argument("--deps", help="Dependency file")
+        self.args.add_argument("file", help="Input file")
+
+    def main(self):
+        """
+        After parsing the input index file it generates a clean output with all includes resolved.
+        Using --deps option it also outputs the dependencies in makefile format like gcc's with -M.
+        """
+
+        index_file_parser = IndexFileParser()
+        index_file_parser.parse(self.config.file)
+
+        with open(self.config.output, "w") as output_file:
+            for item in index_file_parser.items:
+                if item["type"] == "function":
+                    patch = "\tpatch" if item["patch"] else ""
+                    output_file.write(
+                        item["library_name"] + "\t" + item["function_name"] + patch + "\n")
+                else:
+                    output_file.write("reserved\n")
+
+        if self.config.deps:
+            with open(self.config.deps, "w") as deps_file:
+                deps = [self.config.file] + index_file_parser.get_dependencies(self.config.file)
+                deps_file.write(self.config.output + ": " + " \\\n".join(deps) + "\n")
+
+class TableGenerator(RomlibApplication):
+    """ Generates the jump table by parsing the index file. """
+
+    def __init__(self, prog):
+        RomlibApplication.__init__(self, prog)
+
+        self.args.add_argument("-o", "--output", help="Output file", metavar="output",
+                               default="jmpvar.s")
+        self.args.add_argument("--bti", help="Branch Target Identification", type=int)
+        self.args.add_argument("file", help="Input file")
+
+    def main(self):
+        """
+        Inserts the jmptbl definition and the jump entries into the output file. Also can insert
+        BTI related code before entries if --bti option set. It can output a dependency file of the
+        included index files. This can be directly included in makefiles.
+        """
+
+        index_file_parser = IndexFileParser()
+        index_file_parser.parse(self.config.file)
+
+        with open(self.config.output, "w") as output_file:
+            output_file.write(self.build_template("jmptbl_header.S"))
+            bti = "_bti" if self.config.bti == 1 else ""
+
+            for item in index_file_parser.items:
+                template_name = "jmptbl_entry_" + item["type"] + bti + ".S"
+                output_file.write(self.build_template(template_name, item, True))
+
+class WrapperGenerator(RomlibApplication):
+    """
+    Generates a wrapper function for each entry in the index file except for the ones that contain
+    the keyword patch. The generated wrapper file is called <lib>_<fn_name>.s.
+    """
+
+    def __init__(self, prog):
+        RomlibApplication.__init__(self, prog)
+
+        self.args.add_argument("-b", help="Build directory", default=".", metavar="build")
+        self.args.add_argument("--bti", help="Branch Target Identification", type=int)
+        self.args.add_argument("--list", help="Only list assembly files", action="store_true")
+        self.args.add_argument("file", help="Input file")
+
+    def main(self):
+        """
+        Iterates through the items in the parsed index file and builds the template for each entry.
+        """
+
+        index_file_parser = IndexFileParser()
+        index_file_parser.parse(self.config.file)
+
+        bti = "_bti" if self.config.bti == 1 else ""
+        function_offset = 0
+        files = []
+
+        for item_index in range(0, len(index_file_parser.items)):
+            item = index_file_parser.items[item_index]
+
+            if item["type"] == "reserved" or item["patch"]:
+                continue
+
+            asm = self.config.b + "/" + item["function_name"] + ".s"
+            if self.config.list:
+                # Only listing files
+                files.append(asm)
+            else:
+                with open(asm, "w") as asm_file:
+                    # The jump instruction is 4 bytes but BTI requires and extra instruction so
+                    # this makes it 8 bytes per entry.
+                    function_offset = item_index * (8 if self.config.bti else 4)
+
+                    item["function_offset"] = function_offset
+                    asm_file.write(self.build_template("wrapper" + bti + ".S", item))
+
+        if self.config.list:
+            print(" ".join(files))
+
+class VariableGenerator(RomlibApplication):
+    """ Generates the jump table global variable with the absolute address in ROM. """
+
+    def __init__(self, prog):
+        RomlibApplication.__init__(self, prog)
+
+        self.args.add_argument("-o", "--output", help="Output file", metavar="output",
+                               default="jmpvar.s")
+        self.args.add_argument("file", help="Input file")
+
+    def main(self):
+        """
+        Runs nm -a command on the input file and inserts the address of the .text section into the
+        template as the ROM address of the jmp_table.
+        """
+        symbols = subprocess.check_output(["nm", "-a", self.config.file])
+
+        matching_symbol = re.search("([0-9A-Fa-f]+) . \\.text", str(symbols))
+        if not matching_symbol:
+            raise Exception("No '.text' section was found in %s" % self.config.file)
+
+        mapping = {"jmptbl_address": matching_symbol.group(1)}
+
+        with open(self.config.output, "w") as output_file:
+            output_file.write(self.build_template("jmptbl_glob_var.S", mapping))
+
+if __name__ == "__main__":
+    APPS = {"genvar": VariableGenerator, "pre": IndexPreprocessor,
+            "gentbl": TableGenerator, "genwrappers": WrapperGenerator}
+
+    if len(sys.argv) < 2 or sys.argv[1] not in APPS:
+        print("usage: romlib_generator.py [%s] [args]" % "|".join(APPS.keys()), file=sys.stderr)
+        sys.exit(1)
+
+    APP = APPS[sys.argv[1]]("romlib_generator.py " + sys.argv[1])
+    APP.parse_arguments(sys.argv[2:])
+    try:
+        APP.main()
+        sys.exit(0)
+    except FileNotFoundError as file_not_found_error:
+        print(file_not_found_error, file=sys.stderr)
+    except subprocess.CalledProcessError as called_process_error:
+        print(called_process_error.output, file=sys.stderr)
+
+    sys.exit(1)
diff --git a/lib/romlib/templates/jmptbl_entry_function.S b/lib/romlib/templates/jmptbl_entry_function.S
new file mode 100644
index 00000000..a0f84561
--- /dev/null
+++ b/lib/romlib/templates/jmptbl_entry_function.S
@@ -0,0 +1,6 @@
+/*
+ * Copyright (c) 2019, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+	b	${function_name}
diff --git a/lib/romlib/templates/jmptbl_entry_function_bti.S b/lib/romlib/templates/jmptbl_entry_function_bti.S
new file mode 100644
index 00000000..d96ee94d
--- /dev/null
+++ b/lib/romlib/templates/jmptbl_entry_function_bti.S
@@ -0,0 +1,7 @@
+/*
+ * Copyright (c) 2019, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+	bti	j
+	b	${function_name}
diff --git a/lib/romlib/templates/jmptbl_entry_reserved.S b/lib/romlib/templates/jmptbl_entry_reserved.S
new file mode 100644
index 00000000..a9b5f181
--- /dev/null
+++ b/lib/romlib/templates/jmptbl_entry_reserved.S
@@ -0,0 +1,6 @@
+/*
+ * Copyright (c) 2019, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+	b	.
diff --git a/lib/romlib/templates/jmptbl_entry_reserved_bti.S b/lib/romlib/templates/jmptbl_entry_reserved_bti.S
new file mode 100644
index 00000000..a9f03759
--- /dev/null
+++ b/lib/romlib/templates/jmptbl_entry_reserved_bti.S
@@ -0,0 +1,7 @@
+/*
+ * Copyright (c) 2019, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+	bti	j
+	b	.
diff --git a/lib/romlib/templates/jmptbl_glob_var.S b/lib/romlib/templates/jmptbl_glob_var.S
new file mode 100644
index 00000000..d3065125
--- /dev/null
+++ b/lib/romlib/templates/jmptbl_glob_var.S
@@ -0,0 +1,9 @@
+/*
+ * Copyright (c) 2019, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+	.data
+	.globl	jmptbl
+	.align	4
+jmptbl:	.quad	0x${jmptbl_address}
diff --git a/lib/romlib/templates/jmptbl_header.S b/lib/romlib/templates/jmptbl_header.S
new file mode 100644
index 00000000..72b8ce54
--- /dev/null
+++ b/lib/romlib/templates/jmptbl_header.S
@@ -0,0 +1,8 @@
+/*
+ * Copyright (c) 2019, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+	.text
+	.globl	jmptbl
+jmptbl:
diff --git a/lib/romlib/templates/wrapper.S b/lib/romlib/templates/wrapper.S
new file mode 100644
index 00000000..734a68a2
--- /dev/null
+++ b/lib/romlib/templates/wrapper.S
@@ -0,0 +1,12 @@
+/*
+ * Copyright (c) 2019, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+	.globl	${function_name}
+${function_name}:
+	ldr	x17, =jmptbl
+	mov	x16, #${function_offset}
+	ldr	x17, [x17]
+	add	x16, x16, x17
+	br	x16
diff --git a/lib/romlib/templates/wrapper_bti.S b/lib/romlib/templates/wrapper_bti.S
new file mode 100644
index 00000000..ba9b11ce
--- /dev/null
+++ b/lib/romlib/templates/wrapper_bti.S
@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2019, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+	.globl	${function_name}
+${function_name}:
+	bti	jc
+	ldr	x17, =jmptbl
+	mov	x16, #${function_offset}
+	ldr	x17, [x17]
+	add	x16, x16, x17
+	br	x16