Move include and source files to logical locations

Move almost all system include files to a logical sub-directory
under ./include. The only remaining system include directories
not under ./include are specific to the platform. Move the
corresponding source files to match the include directory
structure.

Also remove pm.h as it is no longer used.

Change-Id: Ie5ea6368ec5fad459f3e8a802ad129135527f0b3

7 files changed, 1905 insertions, 0 deletions

diff --git a/lib/aarch64/cache_helpers.S b/lib/aarch64/cache_helpers.S
new file mode 100644
index 00000000..2696d901
--- /dev/null
+++ b/lib/aarch64/cache_helpers.S
@@ -0,0 +1,231 @@
+/*
+ * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of ARM nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <arch_helpers.h>
+#include <asm_macros.S>
+
+	.globl	dcisw
+	.globl	dccisw
+	.globl	dccsw
+	.globl	dccvac
+	.globl	dcivac
+	.globl	dccivac
+	.globl	dccvau
+	.globl	dczva
+	.globl	flush_dcache_range
+	.globl	inv_dcache_range
+	.globl	dcsw_op_louis
+	.globl	dcsw_op_all
+
+func dcisw
+	dc	isw, x0
+	dsb	sy
+	isb
+	ret
+
+
+func dccisw
+	dc	cisw, x0
+	dsb	sy
+	isb
+	ret
+
+
+func dccsw
+	dc	csw, x0
+	dsb	sy
+	isb
+	ret
+
+
+func dccvac
+	dc	cvac, x0
+	dsb	sy
+	isb
+	ret
+
+
+func dcivac
+	dc	ivac, x0
+	dsb	sy
+	isb
+	ret
+
+
+func dccivac
+	dc	civac, x0
+	dsb	sy
+	isb
+	ret
+
+
+func dccvau
+	dc	cvau, x0
+	dsb	sy
+	isb
+	ret
+
+
+func dczva
+	dc	zva, x0
+	dsb	sy
+	isb
+	ret
+
+
+	/* ------------------------------------------
+	 * Clean+Invalidate from base address till
+	 * size. 'x0' = addr, 'x1' = size
+	 * ------------------------------------------
+	 */
+func flush_dcache_range
+	dcache_line_size x2, x3
+	add	x1, x0, x1
+	sub	x3, x2, #1
+	bic	x0, x0, x3
+flush_loop:
+	dc	civac, x0
+	add	x0, x0, x2
+	cmp	x0, x1
+	b.lo    flush_loop
+	dsb	sy
+	ret
+
+
+	/* ------------------------------------------
+	 * Invalidate from base address till
+	 * size. 'x0' = addr, 'x1' = size
+	 * ------------------------------------------
+	 */
+func inv_dcache_range
+	dcache_line_size x2, x3
+	add	x1, x0, x1
+	sub	x3, x2, #1
+	bic	x0, x0, x3
+inv_loop:
+	dc	ivac, x0
+	add	x0, x0, x2
+	cmp	x0, x1
+	b.lo    inv_loop
+	dsb	sy
+	ret
+
+
+	/* ------------------------------------------
+	 * Data cache operations by set/way to the
+	 * level specified
+	 * ------------------------------------------
+	 * ----------------------------------
+	 * Call this func with the clidr in
+	 * x0, starting cache level in x10,
+	 * last cache level in x3 & cm op in
+	 * x14
+	 * ----------------------------------
+	 */
+func dcsw_op
+all_start_at_level:
+	add	x2, x10, x10, lsr #1            // work out 3x current cache level
+	lsr	x1, x0, x2                      // extract cache type bits from clidr
+	and	x1, x1, #7                      // mask of the bits for current cache only
+	cmp	x1, #2                          // see what cache we have at this level
+	b.lt	skip                            // skip if no cache, or just i-cache
+	msr	csselr_el1, x10                 // select current cache level in csselr
+	isb                                     // isb to sych the new cssr&csidr
+	mrs	x1, ccsidr_el1                  // read the new ccsidr
+	and	x2, x1, #7                      // extract the length of the cache lines
+	add	x2, x2, #4                      // add 4 (line length offset)
+	mov	x4, #0x3ff
+	and	x4, x4, x1, lsr #3              // find maximum number on the way size
+	clz	w5, w4                          // find bit position of way size increment
+	mov	x7, #0x7fff
+	and	x7, x7, x1, lsr #13             // extract max number of the index size
+loop2:
+	mov	x9, x4                          // create working copy of max way size
+loop3:
+	lsl	x6, x9, x5
+	orr	x11, x10, x6                    // factor way and cache number into x11
+	lsl	x6, x7, x2
+	orr	x11, x11, x6                    // factor index number into x11
+	mov	x12, x0
+	mov	x13, x30 // lr
+	mov	x0, x11
+	blr	x14
+	mov	x0, x12
+	mov	x30, x13 // lr
+	subs	x9, x9, #1                      // decrement the way
+	b.ge    loop3
+	subs	x7, x7, #1                      // decrement the index
+	b.ge    loop2
+skip:
+	add	x10, x10, #2                    // increment cache number
+	cmp	x3, x10
+	b.gt    all_start_at_level
+finished:
+	mov	x10, #0                         // swith back to cache level 0
+	msr	csselr_el1, x10                 // select current cache level in csselr
+	dsb	sy
+	isb
+	ret
+
+
+func do_dcsw_op
+	cbz	x3, exit
+	cmp	x0, #DCISW
+	b.eq	dc_isw
+	cmp	x0, #DCCISW
+	b.eq	dc_cisw
+	cmp	x0, #DCCSW
+	b.eq	dc_csw
+dc_isw:
+	mov	x0, x9
+	adr	x14, dcisw
+	b	dcsw_op
+dc_cisw:
+	mov	x0, x9
+	adr	x14, dccisw
+	b	dcsw_op
+dc_csw:
+	mov	x0, x9
+	adr	x14, dccsw
+	b	dcsw_op
+exit:
+	ret
+
+
+func dcsw_op_louis
+	dsb	sy
+	setup_dcsw_op_args x10, x3, x9, #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
+	b	do_dcsw_op
+
+
+func dcsw_op_all
+	dsb	sy
+	setup_dcsw_op_args x10, x3, x9, #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
+	b	do_dcsw_op
diff --git a/lib/aarch64/cpu_helpers.S b/lib/aarch64/cpu_helpers.S
new file mode 100644
index 00000000..573d0b8b
--- /dev/null
+++ b/lib/aarch64/cpu_helpers.S
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of ARM nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <arch.h>
+#include <asm_macros.S>
+
+	.weak	cpu_reset_handler
+
+
+func cpu_reset_handler
+	mov	x19, x30 // lr
+
+	/* ---------------------------------------------
+	 * As a bare minimal enable the SMP bit.
+	 * ---------------------------------------------
+	 */
+	bl	read_midr
+	lsr	x0, x0, #MIDR_PN_SHIFT
+	and	x0, x0, #MIDR_PN_MASK
+	cmp	x0, #MIDR_PN_A57
+	b.eq	smp_setup_begin
+	cmp	x0, #MIDR_PN_A53
+	b.ne	smp_setup_end
+smp_setup_begin:
+	bl	read_cpuectlr
+	orr	x0, x0, #CPUECTLR_SMP_BIT
+	bl	write_cpuectlr
+smp_setup_end:
+	ret	x19
diff --git a/lib/aarch64/misc_helpers.S b/lib/aarch64/misc_helpers.S
new file mode 100644
index 00000000..e3b4ab58
--- /dev/null
+++ b/lib/aarch64/misc_helpers.S
@@ -0,0 +1,341 @@
+/*
+ * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of ARM nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <arch_helpers.h>
+#include <runtime_svc.h>
+#include <asm_macros.S>
+
+	.globl	enable_irq
+	.globl	disable_irq
+
+	.globl	enable_fiq
+	.globl	disable_fiq
+
+	.globl	enable_serror
+	.globl	disable_serror
+
+	.globl	enable_debug_exceptions
+	.globl	disable_debug_exceptions
+
+	.globl	read_daif
+	.globl	write_daif
+
+	.globl	read_spsr
+	.globl	read_spsr_el1
+	.globl	read_spsr_el2
+	.globl	read_spsr_el3
+
+	.globl	write_spsr
+	.globl	write_spsr_el1
+	.globl	write_spsr_el2
+	.globl	write_spsr_el3
+
+	.globl	read_elr
+	.globl	read_elr_el1
+	.globl	read_elr_el2
+	.globl	read_elr_el3
+
+	.globl	write_elr
+	.globl	write_elr_el1
+	.globl	write_elr_el2
+	.globl	write_elr_el3
+
+	.globl	get_afflvl_shift
+	.globl	mpidr_mask_lower_afflvls
+	.globl	dsb
+	.globl	isb
+	.globl	sev
+	.globl	wfe
+	.globl	wfi
+	.globl	eret
+	.globl	smc
+
+	.globl	zeromem16
+	.globl	memcpy16
+
+
+func get_afflvl_shift
+	cmp	x0, #3
+	cinc	x0, x0, eq
+	mov	x1, #MPIDR_AFFLVL_SHIFT
+	lsl	x0, x0, x1
+	ret
+
+func mpidr_mask_lower_afflvls
+	cmp	x1, #3
+	cinc	x1, x1, eq
+	mov	x2, #MPIDR_AFFLVL_SHIFT
+	lsl	x2, x1, x2
+	lsr	x0, x0, x2
+	lsl	x0, x0, x2
+	ret
+
+	/* -----------------------------------------------------
+	 * Asynchronous exception manipulation accessors
+	 * -----------------------------------------------------
+	 */
+func enable_irq
+	msr	daifclr, #DAIF_IRQ_BIT
+	ret
+
+
+func enable_fiq
+	msr	daifclr, #DAIF_FIQ_BIT
+	ret
+
+
+func enable_serror
+	msr	daifclr, #DAIF_ABT_BIT
+	ret
+
+
+func enable_debug_exceptions
+	msr	daifclr, #DAIF_DBG_BIT
+	ret
+
+
+func disable_irq
+	msr	daifset, #DAIF_IRQ_BIT
+	ret
+
+
+func disable_fiq
+	msr	daifset, #DAIF_FIQ_BIT
+	ret
+
+
+func disable_serror
+	msr	daifset, #DAIF_ABT_BIT
+	ret
+
+
+func disable_debug_exceptions
+	msr	daifset, #DAIF_DBG_BIT
+	ret
+
+
+func read_daif
+	mrs	x0, daif
+	ret
+
+
+func write_daif
+	msr	daif, x0
+	ret
+
+
+func read_spsr
+	mrs	x0, CurrentEl
+	cmp	x0, #(MODE_EL1 << MODE_EL_SHIFT)
+	b.eq	read_spsr_el1
+	cmp	x0, #(MODE_EL2 << MODE_EL_SHIFT)
+	b.eq	read_spsr_el2
+	cmp	x0, #(MODE_EL3 << MODE_EL_SHIFT)
+	b.eq	read_spsr_el3
+
+
+func read_spsr_el1
+	mrs	x0, spsr_el1
+	ret
+
+
+func read_spsr_el2
+	mrs	x0, spsr_el2
+	ret
+
+
+func read_spsr_el3
+	mrs	x0, spsr_el3
+	ret
+
+
+func write_spsr
+	mrs	x1, CurrentEl
+	cmp	x1, #(MODE_EL1 << MODE_EL_SHIFT)
+	b.eq	write_spsr_el1
+	cmp	x1, #(MODE_EL2 << MODE_EL_SHIFT)
+	b.eq	write_spsr_el2
+	cmp	x1, #(MODE_EL3 << MODE_EL_SHIFT)
+	b.eq	write_spsr_el3
+
+
+func write_spsr_el1
+	msr	spsr_el1, x0
+	isb
+	ret
+
+
+func write_spsr_el2
+	msr	spsr_el2, x0
+	isb
+	ret
+
+
+func write_spsr_el3
+	msr	spsr_el3, x0
+	isb
+	ret
+
+
+func read_elr
+	mrs	x0, CurrentEl
+	cmp	x0, #(MODE_EL1 << MODE_EL_SHIFT)
+	b.eq	read_elr_el1
+	cmp	x0, #(MODE_EL2 << MODE_EL_SHIFT)
+	b.eq	read_elr_el2
+	cmp	x0, #(MODE_EL3 << MODE_EL_SHIFT)
+	b.eq	read_elr_el3
+
+
+func read_elr_el1
+	mrs	x0, elr_el1
+	ret
+
+
+func read_elr_el2
+	mrs	x0, elr_el2
+	ret
+
+
+func read_elr_el3
+	mrs	x0, elr_el3
+	ret
+
+
+func write_elr
+	mrs	x1, CurrentEl
+	cmp	x1, #(MODE_EL1 << MODE_EL_SHIFT)
+	b.eq	write_elr_el1
+	cmp	x1, #(MODE_EL2 << MODE_EL_SHIFT)
+	b.eq	write_elr_el2
+	cmp	x1, #(MODE_EL3 << MODE_EL_SHIFT)
+	b.eq	write_elr_el3
+
+
+func write_elr_el1
+	msr	elr_el1, x0
+	isb
+	ret
+
+
+func write_elr_el2
+	msr	elr_el2, x0
+	isb
+	ret
+
+
+func write_elr_el3
+	msr	elr_el3, x0
+	isb
+	ret
+
+
+func dsb
+	dsb	sy
+	ret
+
+
+func isb
+	isb
+	ret
+
+
+func sev
+	sev
+	ret
+
+
+func wfe
+	wfe
+	ret
+
+
+func wfi
+	wfi
+	ret
+
+
+func eret
+	eret
+
+
+func smc
+	smc	#0
+
+/* -----------------------------------------------------------------------
+ * void zeromem16(void *mem, unsigned int length);
+ *
+ * Initialise a memory region to 0.
+ * The memory address must be 16-byte aligned.
+ * -----------------------------------------------------------------------
+ */
+func zeromem16
+	add	x2, x0, x1
+/* zero 16 bytes at a time */
+z_loop16:
+	sub	x3, x2, x0
+	cmp	x3, #16
+	b.lt	z_loop1
+	stp	xzr, xzr, [x0], #16
+	b	z_loop16
+/* zero byte per byte */
+z_loop1:
+	cmp	x0, x2
+	b.eq	z_end
+	strb	wzr, [x0], #1
+	b	z_loop1
+z_end:	ret
+
+
+/* --------------------------------------------------------------------------
+ * void memcpy16(void *dest, const void *src, unsigned int length)
+ *
+ * Copy length bytes from memory area src to memory area dest.
+ * The memory areas should not overlap.
+ * Destination and source addresses must be 16-byte aligned.
+ * --------------------------------------------------------------------------
+ */
+func memcpy16
+/* copy 16 bytes at a time */
+m_loop16:
+	cmp	x2, #16
+	b.lt	m_loop1
+	ldp	x3, x4, [x1], #16
+	stp	x3, x4, [x0], #16
+	sub	x2, x2, #16
+	b	m_loop16
+/* copy byte per byte */
+m_loop1:
+	cbz	x2, m_end
+	ldrb	w3, [x1], #1
+	strb	w3, [x0], #1
+	subs	x2, x2, #1
+	b.ne	m_loop1
+m_end:	ret
diff --git a/lib/aarch64/sysreg_helpers.S b/lib/aarch64/sysreg_helpers.S
new file mode 100644
index 00000000..8e816f03
--- /dev/null
+++ b/lib/aarch64/sysreg_helpers.S
@@ -0,0 +1,833 @@
+/*
+ * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of ARM nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <arch_helpers.h>
+#include <asm_macros.S>
+
+	.globl	read_vbar_el1
+	.globl	read_vbar_el2
+	.globl	read_vbar_el3
+	.globl	write_vbar_el1
+	.globl	write_vbar_el2
+	.globl	write_vbar_el3
+
+	.globl	read_sctlr_el1
+	.globl	read_sctlr_el2
+	.globl	read_sctlr_el3
+	.globl	write_sctlr_el1
+	.globl	write_sctlr_el2
+	.globl	write_sctlr_el3
+
+	.globl	read_actlr_el1
+	.globl	read_actlr_el2
+	.globl	read_actlr_el3
+	.globl	write_actlr_el1
+	.globl	write_actlr_el2
+	.globl	write_actlr_el3
+
+	.globl	read_esr_el1
+	.globl	read_esr_el2
+	.globl	read_esr_el3
+	.globl	write_esr_el1
+	.globl	write_esr_el2
+	.globl	write_esr_el3
+
+	.globl	read_afsr0_el1
+	.globl	read_afsr0_el2
+	.globl	read_afsr0_el3
+	.globl	write_afsr0_el1
+	.globl	write_afsr0_el2
+	.globl	write_afsr0_el3
+
+	.globl	read_afsr1_el1
+	.globl	read_afsr1_el2
+	.globl	read_afsr1_el3
+	.globl	write_afsr1_el1
+	.globl	write_afsr1_el2
+	.globl	write_afsr1_el3
+
+	.globl	read_far_el1
+	.globl	read_far_el2
+	.globl	read_far_el3
+	.globl	write_far_el1
+	.globl	write_far_el2
+	.globl	write_far_el3
+
+	.globl	read_mair_el1
+	.globl	read_mair_el2
+	.globl	read_mair_el3
+	.globl	write_mair_el1
+	.globl	write_mair_el2
+	.globl	write_mair_el3
+
+	.globl	read_amair_el1
+	.globl	read_amair_el2
+	.globl	read_amair_el3
+	.globl	write_amair_el1
+	.globl	write_amair_el2
+	.globl	write_amair_el3
+
+	.globl	read_rvbar_el1
+	.globl	read_rvbar_el2
+	.globl	read_rvbar_el3
+
+	.globl	read_rmr_el1
+	.globl	read_rmr_el2
+	.globl	read_rmr_el3
+	.globl	write_rmr_el1
+	.globl	write_rmr_el2
+	.globl	write_rmr_el3
+
+	.globl	read_tcr_el1
+	.globl	read_tcr_el2
+	.globl	read_tcr_el3
+	.globl	write_tcr_el1
+	.globl	write_tcr_el2
+	.globl	write_tcr_el3
+
+	.globl	read_cptr_el2
+	.globl	read_cptr_el3
+	.globl	write_cptr_el2
+	.globl	write_cptr_el3
+
+	.globl	read_ttbr0_el1
+	.globl	read_ttbr0_el2
+	.globl	read_ttbr0_el3
+	.globl	write_ttbr0_el1
+	.globl	write_ttbr0_el2
+	.globl	write_ttbr0_el3
+
+	.globl	read_ttbr1_el1
+	.globl	read_ttbr1_el2
+	.globl	write_ttbr1
+	.globl	write_ttbr1_el1
+	.globl	write_ttbr1_el2
+
+	.globl	read_cpacr
+	.globl	write_cpacr
+
+	.globl	read_cntfrq
+	.globl	write_cntfrq
+
+	.globl	read_cpuectlr
+	.globl	write_cpuectlr
+
+	.globl	read_cnthctl_el2
+	.globl	write_cnthctl_el2
+
+	.globl	read_cntfrq_el0
+	.globl	write_cntfrq_el0
+
+	.globl	read_scr
+	.globl	write_scr
+
+	.globl	read_hcr
+	.globl	write_hcr
+
+	.globl	read_midr
+	.globl	read_mpidr
+
+	.globl	read_current_el
+	.globl	read_id_pfr1_el1
+	.globl	read_id_aa64pfr0_el1
+
+#if SUPPORT_VFP
+	.globl	enable_vfp
+	.globl	read_fpexc
+	.globl	write_fpexc
+#endif
+
+
+func read_current_el
+	mrs	x0, CurrentEl
+	ret
+
+
+func read_id_pfr1_el1
+	mrs	x0, id_pfr1_el1
+	ret
+
+
+func read_id_aa64pfr0_el1
+	mrs	x0, id_aa64pfr0_el1
+	ret
+
+
+	/* -----------------------------------------------------
+	 * VBAR accessors
+	 * -----------------------------------------------------
+	 */
+func read_vbar_el1
+	mrs	x0, vbar_el1
+	ret
+
+
+func read_vbar_el2
+	mrs	x0, vbar_el2
+	ret
+
+
+func read_vbar_el3
+	mrs	x0, vbar_el3
+	ret
+
+
+func write_vbar_el1
+	msr	vbar_el1, x0
+	isb
+	ret
+
+
+func write_vbar_el2
+	msr	vbar_el2, x0
+	isb
+	ret
+
+
+func write_vbar_el3
+	msr	vbar_el3, x0
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * AFSR0 accessors
+	 * -----------------------------------------------------
+	 */
+func read_afsr0_el1
+	mrs	x0, afsr0_el1
+	ret
+
+
+func read_afsr0_el2
+	mrs	x0, afsr0_el2
+	ret
+
+
+func read_afsr0_el3
+	mrs	x0, afsr0_el3
+	ret
+
+
+func write_afsr0_el1
+	msr	afsr0_el1, x0
+	isb
+	ret
+
+
+func write_afsr0_el2
+	msr	afsr0_el2, x0
+	isb
+	ret
+
+
+func write_afsr0_el3
+	msr	afsr0_el3, x0
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * FAR accessors
+	 * -----------------------------------------------------
+	 */
+func read_far_el1
+	mrs	x0, far_el1
+	ret
+
+
+func read_far_el2
+	mrs	x0, far_el2
+	ret
+
+
+func read_far_el3
+	mrs	x0, far_el3
+	ret
+
+
+func write_far_el1
+	msr	far_el1, x0
+	isb
+	ret
+
+
+func write_far_el2
+	msr	far_el2, x0
+	isb
+	ret
+
+
+func write_far_el3
+	msr	far_el3, x0
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * MAIR accessors
+	 * -----------------------------------------------------
+	 */
+func read_mair_el1
+	mrs	x0, mair_el1
+	ret
+
+
+func read_mair_el2
+	mrs	x0, mair_el2
+	ret
+
+
+func read_mair_el3
+	mrs	x0, mair_el3
+	ret
+
+
+func write_mair_el1
+	msr	mair_el1, x0
+	isb
+	ret
+
+
+func write_mair_el2
+	msr	mair_el2, x0
+	isb
+	ret
+
+
+func write_mair_el3
+	msr	mair_el3, x0
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * AMAIR accessors
+	 * -----------------------------------------------------
+	 */
+func read_amair_el1
+	mrs	x0, amair_el1
+	ret
+
+
+func read_amair_el2
+	mrs	x0, amair_el2
+	ret
+
+
+func read_amair_el3
+	mrs	x0, amair_el3
+	ret
+
+
+func write_amair_el1
+	msr	amair_el1, x0
+	isb
+	ret
+
+
+func write_amair_el2
+	msr	amair_el2, x0
+	isb
+	ret
+
+
+func write_amair_el3
+	msr	amair_el3, x0
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * RVBAR accessors
+	 * -----------------------------------------------------
+	 */
+func read_rvbar_el1
+	mrs	x0, rvbar_el1
+	ret
+
+
+func read_rvbar_el2
+	mrs	x0, rvbar_el2
+	ret
+
+
+func read_rvbar_el3
+	mrs	x0, rvbar_el3
+	ret
+
+
+	/* -----------------------------------------------------
+	 * RMR accessors
+	 * -----------------------------------------------------
+	 */
+func read_rmr_el1
+	mrs	x0, rmr_el1
+	ret
+
+
+func read_rmr_el2
+	mrs	x0, rmr_el2
+	ret
+
+
+func read_rmr_el3
+	mrs	x0, rmr_el3
+	ret
+
+
+func write_rmr_el1
+	msr	rmr_el1, x0
+	isb
+	ret
+
+
+func write_rmr_el2
+	msr	rmr_el2, x0
+	isb
+	ret
+
+
+func write_rmr_el3
+	msr	rmr_el3, x0
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * AFSR1 accessors
+	 * -----------------------------------------------------
+	 */
+func read_afsr1_el1
+	mrs	x0, afsr1_el1
+	ret
+
+
+func read_afsr1_el2
+	mrs	x0, afsr1_el2
+	ret
+
+
+func read_afsr1_el3
+	mrs	x0, afsr1_el3
+	ret
+
+
+func write_afsr1_el1
+	msr	afsr1_el1, x0
+	isb
+	ret
+
+
+func write_afsr1_el2
+	msr	afsr1_el2, x0
+	isb
+	ret
+
+
+func write_afsr1_el3
+	msr	afsr1_el3, x0
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * SCTLR accessors
+	 * -----------------------------------------------------
+	 */
+func read_sctlr_el1
+	mrs	x0, sctlr_el1
+	ret
+
+
+func read_sctlr_el2
+	mrs	x0, sctlr_el2
+	ret
+
+
+func read_sctlr_el3
+	mrs	x0, sctlr_el3
+	ret
+
+
+func write_sctlr_el1
+	msr	sctlr_el1, x0
+	dsb	sy
+	isb
+	ret
+
+
+func write_sctlr_el2
+	msr	sctlr_el2, x0
+	dsb	sy
+	isb
+	ret
+
+
+func write_sctlr_el3
+	msr	sctlr_el3, x0
+	dsb	sy
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * ACTLR accessors
+	 * -----------------------------------------------------
+	 */
+func read_actlr_el1
+	mrs	x0, actlr_el1
+	ret
+
+
+func read_actlr_el2
+	mrs	x0, actlr_el2
+	ret
+
+
+func read_actlr_el3
+	mrs	x0, actlr_el3
+	ret
+
+
+func write_actlr_el1
+	msr	actlr_el1, x0
+	dsb	sy
+	isb
+	ret
+
+
+func write_actlr_el2
+	msr	actlr_el2, x0
+	dsb	sy
+	isb
+	ret
+
+
+func write_actlr_el3
+	msr	actlr_el3, x0
+	dsb	sy
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * ESR accessors
+	 * -----------------------------------------------------
+	 */
+func read_esr_el1
+	mrs	x0, esr_el1
+	ret
+
+
+func read_esr_el2
+	mrs	x0, esr_el2
+	ret
+
+
+func read_esr_el3
+	mrs	x0, esr_el3
+	ret
+
+
+func write_esr_el1
+	msr	esr_el1, x0
+	dsb	sy
+	isb
+	ret
+
+
+func write_esr_el2
+	msr	esr_el2, x0
+	dsb	sy
+	isb
+	ret
+
+
+func write_esr_el3
+	msr	esr_el3, x0
+	dsb	sy
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * TCR accessors
+	 * -----------------------------------------------------
+	 */
+func read_tcr_el1
+	mrs	x0, tcr_el1
+	ret
+
+
+func read_tcr_el2
+	mrs	x0, tcr_el2
+	ret
+
+
+func read_tcr_el3
+	mrs	x0, tcr_el3
+	ret
+
+
+func write_tcr_el1
+	msr	tcr_el1, x0
+	dsb	sy
+	isb
+	ret
+
+
+func write_tcr_el2
+	msr	tcr_el2, x0
+	dsb	sy
+	isb
+	ret
+
+
+func write_tcr_el3
+	msr	tcr_el3, x0
+	dsb	sy
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * CPTR accessors
+	 * -----------------------------------------------------
+	 */
+func read_cptr_el1
+	b	read_cptr_el1
+	ret
+
+
+func read_cptr_el2
+	mrs	x0, cptr_el2
+	ret
+
+
+func read_cptr_el3
+	mrs	x0, cptr_el3
+	ret
+
+
+func write_cptr_el1
+	b	write_cptr_el1
+
+
+func write_cptr_el2
+	msr	cptr_el2, x0
+	dsb	sy
+	isb
+	ret
+
+
+func write_cptr_el3
+	msr	cptr_el3, x0
+	dsb	sy
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * TTBR0 accessors
+	 * -----------------------------------------------------
+	 */
+func read_ttbr0_el1
+	mrs	x0, ttbr0_el1
+	ret
+
+
+func read_ttbr0_el2
+	mrs	x0, ttbr0_el2
+	ret
+
+
+func read_ttbr0_el3
+	mrs	x0, ttbr0_el3
+	ret
+
+
+func write_ttbr0_el1
+	msr	ttbr0_el1, x0
+	isb
+	ret
+
+
+func write_ttbr0_el2
+	msr	ttbr0_el2, x0
+	isb
+	ret
+
+
+func write_ttbr0_el3
+	msr	ttbr0_el3, x0
+	isb
+	ret
+
+
+	/* -----------------------------------------------------
+	 * TTBR1 accessors
+	 * -----------------------------------------------------
+	 */
+func read_ttbr1_el1
+	mrs	x0, ttbr1_el1
+	ret
+
+
+func read_ttbr1_el2
+	b	read_ttbr1_el2
+
+
+func read_ttbr1_el3
+	b	read_ttbr1_el3
+
+
+func write_ttbr1_el1
+	msr	ttbr1_el1, x0
+	isb
+	ret
+
+
+func write_ttbr1_el2
+	b	write_ttbr1_el2
+
+
+func write_ttbr1_el3
+	b	write_ttbr1_el3
+
+
+func read_hcr
+	mrs	x0, hcr_el2
+	ret
+
+
+func write_hcr
+	msr	hcr_el2, x0
+	dsb	sy
+	isb
+	ret
+
+
+func read_cpacr
+	mrs	x0, cpacr_el1
+	ret
+
+
+func write_cpacr
+	msr	cpacr_el1, x0
+	ret
+
+
+func read_cntfrq_el0
+	mrs	x0, cntfrq_el0
+	ret
+
+
+func write_cntfrq_el0
+	msr	cntfrq_el0, x0
+	ret
+
+
+func read_cpuectlr
+	mrs	x0, CPUECTLR_EL1
+	ret
+
+
+func write_cpuectlr
+	msr	CPUECTLR_EL1, x0
+	dsb	sy
+	isb
+	ret
+
+
+func read_cnthctl_el2
+	mrs	x0, cnthctl_el2
+	ret
+
+
+func write_cnthctl_el2
+	msr	cnthctl_el2, x0
+	ret
+
+
+func read_cntfrq
+	mrs	x0, cntfrq_el0
+	ret
+
+
+func write_cntfrq
+	msr	cntfrq_el0, x0
+	ret
+
+
+func write_scr
+	msr	scr_el3, x0
+	dsb	sy
+	isb
+	ret
+
+
+func read_scr
+	mrs	x0, scr_el3
+	ret
+
+
+func read_midr
+	mrs	x0, midr_el1
+	ret
+
+
+func read_mpidr
+	mrs	x0, mpidr_el1
+	ret
+
+
+#if SUPPORT_VFP
+func enable_vfp
+	mrs	x0, cpacr_el1
+	orr	x0, x0, #CPACR_VFP_BITS
+	msr	cpacr_el1, x0
+	mrs	x0, cptr_el3
+	mov	x1, #AARCH64_CPTR_TFP
+	bic	x0, x0, x1
+	msr	cptr_el3, x0
+	ret
+
+
+func read_fpexc
+	b	read_fpexc
+	ret
+
+
+func write_fpexc
+	b	write_fpexc
+	ret
+
+#endif
diff --git a/lib/aarch64/tlb_helpers.S b/lib/aarch64/tlb_helpers.S
new file mode 100644
index 00000000..42449747
--- /dev/null
+++ b/lib/aarch64/tlb_helpers.S
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of ARM nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <arch_helpers.h>
+#include <asm_macros.S>
+
+	.globl	tlbialle1
+	.globl	tlbialle1is
+	.globl	tlbialle2
+	.globl	tlbialle2is
+	.globl	tlbialle3
+	.globl	tlbialle3is
+	.globl	tlbivmalle1
+
+
+func tlbialle1
+	tlbi	alle1
+	dsb	sy
+	isb
+	ret
+
+
+func tlbialle1is
+	tlbi	alle1is
+	dsb	sy
+	isb
+	ret
+
+
+func tlbialle2
+	tlbi	alle2
+	dsb	sy
+	isb
+	ret
+
+
+func tlbialle2is
+	tlbi	alle2is
+	dsb	sy
+	isb
+	ret
+
+
+func tlbialle3
+	tlbi	alle3
+	dsb	sy
+	isb
+	ret
+
+
+func tlbialle3is
+	tlbi	alle3is
+	dsb	sy
+	isb
+	ret
+
+func tlbivmalle1
+	tlbi	vmalle1
+	dsb	sy
+	isb
+	ret
diff --git a/lib/aarch64/xlat_helpers.c b/lib/aarch64/xlat_helpers.c
new file mode 100644
index 00000000..87d24ec9
--- /dev/null
+++ b/lib/aarch64/xlat_helpers.c
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of ARM nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <string.h>
+#include <assert.h>
+#include <arch.h>
+
+/*******************************************************************************
+ * Helper to create a level 1/2 table descriptor which points to a level 2/3
+ * table.
+ ******************************************************************************/
+unsigned long create_table_desc(unsigned long *next_table_ptr)
+{
+	unsigned long desc = (unsigned long) next_table_ptr;
+
+	/* Clear the last 12 bits */
+	desc >>= FOUR_KB_SHIFT;
+	desc <<= FOUR_KB_SHIFT;
+
+	desc |= TABLE_DESC;
+
+	return desc;
+}
+
+/*******************************************************************************
+ * Helper to create a level 1/2/3 block descriptor which maps the va to addr
+ ******************************************************************************/
+unsigned long create_block_desc(unsigned long desc,
+				unsigned long addr,
+				unsigned int level)
+{
+	switch (level) {
+	case LEVEL1:
+		desc |= (addr << FIRST_LEVEL_DESC_N) | BLOCK_DESC;
+		break;
+	case LEVEL2:
+		desc |= (addr << SECOND_LEVEL_DESC_N) | BLOCK_DESC;
+		break;
+	case LEVEL3:
+		desc |= (addr << THIRD_LEVEL_DESC_N) | TABLE_DESC;
+		break;
+	default:
+		assert(0);
+	}
+
+	return desc;
+}
+
+/*******************************************************************************
+ * Helper to create a level 1/2/3 block descriptor which maps the va to output_
+ * addr with Device nGnRE attributes.
+ ******************************************************************************/
+unsigned long create_device_block(unsigned long output_addr,
+				  unsigned int level,
+				  unsigned int ns)
+{
+	unsigned long upper_attrs, lower_attrs, desc;
+
+	lower_attrs = LOWER_ATTRS(ACCESS_FLAG | OSH | AP_RW);
+	lower_attrs |= LOWER_ATTRS(ns | ATTR_DEVICE_INDEX);
+	upper_attrs = UPPER_ATTRS(XN);
+	desc = upper_attrs | lower_attrs;
+
+	return create_block_desc(desc, output_addr, level);
+}
+
+/*******************************************************************************
+ * Helper to create a level 1/2/3 block descriptor which maps the va to output_
+ * addr with inner-shareable normal wbwa read-only memory attributes.
+ ******************************************************************************/
+unsigned long create_romem_block(unsigned long output_addr,
+				 unsigned int level,
+				 unsigned int ns)
+{
+	unsigned long upper_attrs, lower_attrs, desc;
+
+	lower_attrs = LOWER_ATTRS(ACCESS_FLAG | ISH | AP_RO);
+	lower_attrs |= LOWER_ATTRS(ns | ATTR_IWBWA_OWBWA_NTR_INDEX);
+	upper_attrs = UPPER_ATTRS(0ull);
+	desc = upper_attrs | lower_attrs;
+
+	return create_block_desc(desc, output_addr, level);
+}
+
+/*******************************************************************************
+ * Helper to create a level 1/2/3 block descriptor which maps the va to output_
+ * addr with inner-shareable normal wbwa read-write memory attributes.
+ ******************************************************************************/
+unsigned long create_rwmem_block(unsigned long output_addr,
+				 unsigned int level,
+				 unsigned int ns)
+{
+	unsigned long upper_attrs, lower_attrs, desc;
+
+	lower_attrs = LOWER_ATTRS(ACCESS_FLAG | ISH | AP_RW);
+	lower_attrs |= LOWER_ATTRS(ns | ATTR_IWBWA_OWBWA_NTR_INDEX);
+	upper_attrs = UPPER_ATTRS(XN);
+	desc = upper_attrs | lower_attrs;
+
+	return create_block_desc(desc, output_addr, level);
+}
diff --git a/lib/aarch64/xlat_tables.c b/lib/aarch64/xlat_tables.c
new file mode 100644
index 00000000..33a8b6da
--- /dev/null
+++ b/lib/aarch64/xlat_tables.c
@@ -0,0 +1,228 @@
+/*
+ * Copyright (c) 2014, ARM Limited and Contributors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of ARM nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <platform.h>
+#include <string.h>
+#include <xlat_tables.h>
+
+
+#ifndef DEBUG_XLAT_TABLE
+#define DEBUG_XLAT_TABLE 0
+#endif
+
+#if DEBUG_XLAT_TABLE
+#define debug_print(...) printf(__VA_ARGS__)
+#else
+#define debug_print(...) ((void)0)
+#endif
+
+
+#define UNSET_DESC	~0ul
+
+#define NUM_L1_ENTRIES (ADDR_SPACE_SIZE >> L1_XLAT_ADDRESS_SHIFT)
+
+uint64_t l1_xlation_table[NUM_L1_ENTRIES]
+__aligned(NUM_L1_ENTRIES * sizeof(uint64_t));
+
+static uint64_t xlat_tables[MAX_XLAT_TABLES][XLAT_TABLE_ENTRIES]
+__aligned(XLAT_TABLE_SIZE) __attribute__((section("xlat_table")));
+
+static unsigned next_xlat;
+
+/*
+ * Array of all memory regions stored in order of ascending base address.
+ * The list is terminated by the first entry with size == 0.
+ */
+static mmap_region mmap[MAX_MMAP_REGIONS + 1];
+
+
+static void print_mmap(void)
+{
+#if DEBUG_XLAT_TABLE
+	debug_print("mmap:\n");
+	mmap_region *mm = mmap;
+	while (mm->size) {
+		debug_print(" %010lx %10lx %x\n", mm->base, mm->size, mm->attr);
+		++mm;
+	};
+	debug_print("\n");
+#endif
+}
+
+void mmap_add_region(unsigned long base, unsigned long size, unsigned attr)
+{
+	mmap_region *mm = mmap;
+	mmap_region *mm_last = mm + sizeof(mmap) / sizeof(mmap[0]) - 1;
+
+	assert(IS_PAGE_ALIGNED(base));
+	assert(IS_PAGE_ALIGNED(size));
+
+	if (!size)
+		return;
+
+	/* Find correct place in mmap to insert new region */
+	while (mm->base < base && mm->size)
+		++mm;
+
+	/* Make room for new region by moving other regions up by one place */
+	memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm);
+
+	/* Check we haven't lost the empty sentinal from the end of the array */
+	assert(mm_last->size == 0);
+
+	mm->base = base;
+	mm->size = size;
+	mm->attr = attr;
+}
+
+void mmap_add(const mmap_region *mm)
+{
+	while (mm->size) {
+		mmap_add_region(mm->base, mm->size, mm->attr);
+		++mm;
+	}
+}
+
+static unsigned long mmap_desc(unsigned attr, unsigned long addr,
+					unsigned level)
+{
+	unsigned long desc = addr;
+
+	desc |= level == 3 ? TABLE_DESC : BLOCK_DESC;
+
+	desc |= attr & MT_NS ? LOWER_ATTRS(NS) : 0;
+
+	desc |= attr & MT_RW ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO);
+
+	desc |= LOWER_ATTRS(ACCESS_FLAG);
+
+	if (attr & MT_MEMORY) {
+		desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH);
+		if (attr & MT_RW)
+			desc |= UPPER_ATTRS(XN);
+	} else {
+		desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH);
+		desc |= UPPER_ATTRS(XN);
+	}
+
+	debug_print(attr & MT_MEMORY ? "MEM" : "DEV");
+	debug_print(attr & MT_RW ? "-RW" : "-RO");
+	debug_print(attr & MT_NS ? "-NS" : "-S");
+
+	return desc;
+}
+
+static int mmap_region_attr(mmap_region *mm, unsigned long base,
+					unsigned long size)
+{
+	int attr = mm->attr;
+
+	for (;;) {
+		++mm;
+
+		if (!mm->size)
+			return attr; /* Reached end of list */
+
+		if (mm->base >= base + size)
+			return attr; /* Next region is after area so end */
+
+		if (mm->base + mm->size <= base)
+			continue; /* Next region has already been overtaken */
+
+		if ((mm->attr & attr) == attr)
+			continue; /* Region doesn't override attribs so skip */
+
+		attr &= mm->attr;
+
+		if (mm->base > base || mm->base + mm->size < base + size)
+			return -1; /* Region doesn't fully cover our area */
+	}
+}
+
+static mmap_region *init_xlation_table(mmap_region *mm, unsigned long base,
+					unsigned long *table, unsigned level)
+{
+	unsigned level_size_shift = L1_XLAT_ADDRESS_SHIFT - (level - 1) *
+						XLAT_TABLE_ENTRIES_SHIFT;
+	unsigned level_size = 1 << level_size_shift;
+	unsigned level_index_mask = XLAT_TABLE_ENTRIES_MASK << level_size_shift;
+
+	assert(level <= 3);
+
+	debug_print("New xlat table:\n");
+
+	do  {
+		unsigned long desc = UNSET_DESC;
+
+		if (mm->base + mm->size <= base) {
+			/* Area now after the region so skip it */
+			++mm;
+			continue;
+		}
+
+		debug_print("      %010lx %8lx " + 6 - 2 * level, base, level_size);
+
+		if (mm->base >= base + level_size) {
+			/* Next region is after area so nothing to map yet */
+			desc = INVALID_DESC;
+		} else if (mm->base <= base &&
+				mm->base + mm->size >= base + level_size) {
+			/* Next region covers all of area */
+			int attr = mmap_region_attr(mm, base, level_size);
+			if (attr >= 0)
+				desc = mmap_desc(attr, base, level);
+		}
+		/* else Next region only partially covers area, so need */
+
+		if (desc == UNSET_DESC) {
+			/* Area not covered by a region so need finer table */
+			unsigned long *new_table = xlat_tables[next_xlat++];
+			assert(next_xlat <= MAX_XLAT_TABLES);
+			desc = TABLE_DESC | (unsigned long)new_table;
+
+			/* Recurse to fill in new table */
+			mm = init_xlation_table(mm, base, new_table, level+1);
+		}
+
+		debug_print("\n");
+
+		*table++ = desc;
+		base += level_size;
+	} while (mm->size && (base & level_index_mask));
+
+	return mm;
+}
+
+void init_xlat_tables(void)
+{
+	print_mmap();
+	init_xlation_table(mmap, 0, l1_xlation_table, 1);
+}