Merge pull request #983 from dp-arm/dp/aarch32-errata

aarch32: Implement errata workarounds for Cortex A53 and A57

3 files changed, 496 insertions, 2 deletions

diff --git a/lib/cpus/aarch32/cortex_a53.S b/lib/cpus/aarch32/cortex_a53.S
index 3d5f833a..bc2c7628 100644
--- a/lib/cpus/aarch32/cortex_a53.S
+++ b/lib/cpus/aarch32/cortex_a53.S
@@ -10,6 +10,11 @@
 #include <cpu_macros.S>
 #include <debug.h>
 
+#if A53_DISABLE_NON_TEMPORAL_HINT
+#undef ERRATA_A53_836870
+#define ERRATA_A53_836870	1
+#endif
+
 	/* ---------------------------------------------
 	 * Disable intra-cluster coherency
 	 * ---------------------------------------------
@@ -23,11 +28,133 @@ func cortex_a53_disable_smp
 	bx	lr
 endfunc cortex_a53_disable_smp
 
+	/* --------------------------------------------------
+	 * Errata Workaround for Cortex A53 Errata #826319.
+	 * This applies only to revision <= r0p2 of Cortex A53.
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * --------------------------------------------------
+	 */
+func errata_a53_826319_wa
+	/*
+	 * Compare r0 against revision r0p2
+	 */
+	mov	r2, lr
+	bl	check_errata_826319
+	mov	lr, r2
+	cmp	r0, #ERRATA_NOT_APPLIES
+	beq	1f
+	ldcopr	r0, CORTEX_A53_L2ACTLR
+	bic	r0, #CORTEX_A53_L2ACTLR_ENABLE_UNIQUECLEAN
+	orr	r0, #CORTEX_A53_L2ACTLR_DISABLE_CLEAN_PUSH
+	stcopr	r0, CORTEX_A53_L2ACTLR
+1:
+	bx	lr
+endfunc errata_a53_826319_wa
+
+func check_errata_826319
+	mov	r1, #0x02
+	b	cpu_rev_var_ls
+endfunc check_errata_826319
+
+	/* ---------------------------------------------------------------------
+	 * Disable the cache non-temporal hint.
+	 *
+	 * This ignores the Transient allocation hint in the MAIR and treats
+	 * allocations the same as non-transient allocation types. As a result,
+	 * the LDNP and STNP instructions in AArch64 behave the same as the
+	 * equivalent LDP and STP instructions.
+	 *
+	 * This is relevant only for revisions <= r0p3 of Cortex-A53.
+	 * From r0p4 and onwards, the bit to disable the hint is enabled by
+	 * default at reset.
+	 *
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * ---------------------------------------------------------------------
+	 */
+func a53_disable_non_temporal_hint
+	/*
+	 * Compare r0 against revision r0p3
+	 */
+	mov		r2, lr
+	bl		check_errata_disable_non_temporal_hint
+	mov		lr, r2
+	cmp		r0, #ERRATA_NOT_APPLIES
+	beq		1f
+	ldcopr16	r0, r1, CORTEX_A53_ACTLR
+	orr64_imm	r0, r1, CORTEX_A53_ACTLR_DTAH
+	stcopr16	r0, r1, CORTEX_A53_ACTLR
+1:
+	bx		lr
+endfunc a53_disable_non_temporal_hint
+
+func check_errata_disable_non_temporal_hint
+	mov	r1, #0x03
+	b	cpu_rev_var_ls
+endfunc check_errata_disable_non_temporal_hint
+
+	/* --------------------------------------------------
+	 * Errata Workaround for Cortex A53 Errata #855873.
+	 *
+	 * This applies only to revisions >= r0p3 of Cortex A53.
+	 * Earlier revisions of the core are affected as well, but don't
+	 * have the chicken bit in the CPUACTLR register. It is expected that
+	 * the rich OS takes care of that, especially as the workaround is
+	 * shared with other erratas in those revisions of the CPU.
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * --------------------------------------------------
+	 */
+func errata_a53_855873_wa
+	/*
+	 * Compare r0 against revision r0p3 and higher
+	 */
+	mov		r2, lr
+	bl		check_errata_855873
+	mov		lr, r2
+	cmp		r0, #ERRATA_NOT_APPLIES
+	beq		1f
+	ldcopr16	r0, r1, CORTEX_A53_ACTLR
+	orr64_imm	r0, r1, CORTEX_A53_ACTLR_ENDCCASCI
+	stcopr16	r0, r1, CORTEX_A53_ACTLR
+1:
+	bx		lr
+endfunc errata_a53_855873_wa
+
+func check_errata_855873
+	mov	r1, #0x03
+	b	cpu_rev_var_hs
+endfunc check_errata_855873
+
 	/* -------------------------------------------------
 	 * The CPU Ops reset function for Cortex-A53.
+	 * Shall clobber: r0-r6
 	 * -------------------------------------------------
 	 */
 func cortex_a53_reset_func
+	mov	r5, lr
+	bl	cpu_get_rev_var
+	mov	r4, r0
+
+#if ERRATA_A53_826319
+	mov	r0, r4
+	bl	errata_a53_826319_wa
+#endif
+
+#if ERRATA_A53_836870
+	mov	r0, r4
+	bl	a53_disable_non_temporal_hint
+#endif
+
+#if ERRATA_A53_855873
+	mov	r0, r4
+	bl	errata_a53_855873_wa
+#endif
+
 	/* ---------------------------------------------
 	 * Enable the SMP bit.
 	 * ---------------------------------------------
@@ -36,7 +163,7 @@ func cortex_a53_reset_func
 	orr64_imm	r0, r1, CORTEX_A53_ECTLR_SMP_BIT
 	stcopr16	r0, r1,	CORTEX_A53_ECTLR
 	isb
-	bx	lr
+	bx	r5
 endfunc cortex_a53_reset_func
 
 	/* ----------------------------------------------------
@@ -111,6 +238,29 @@ func cortex_a53_cluster_pwr_dwn
 	b	cortex_a53_disable_smp
 endfunc cortex_a53_cluster_pwr_dwn
 
+#if REPORT_ERRATA
+/*
+ * Errata printing function for Cortex A53. Must follow AAPCS.
+ */
+func cortex_a53_errata_report
+	push	{r12, lr}
+
+	bl	cpu_get_rev_var
+	mov	r4, r0
+
+	/*
+	 * Report all errata. The revision-variant information is passed to
+	 * checking functions of each errata.
+	 */
+	report_errata ERRATA_A53_826319, cortex_a53, 826319
+	report_errata ERRATA_A53_836870, cortex_a53, disable_non_temporal_hint
+	report_errata ERRATA_A53_855873, cortex_a53, 855873
+
+	pop	{r12, lr}
+	bx	lr
+endfunc cortex_a53_errata_report
+#endif
+
 declare_cpu_ops cortex_a53, CORTEX_A53_MIDR, \
 	cortex_a53_reset_func, \
 	cortex_a53_core_pwr_dwn, \
diff --git a/lib/cpus/aarch32/cortex_a57.S b/lib/cpus/aarch32/cortex_a57.S
index ed478463..a791e4ee 100644
--- a/lib/cpus/aarch32/cortex_a57.S
+++ b/lib/cpus/aarch32/cortex_a57.S
@@ -50,11 +50,312 @@ func cortex_a57_disable_ext_debug
 	bx	lr
 endfunc cortex_a57_disable_ext_debug
 
+	/* --------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #806969.
+	 * This applies only to revision r0p0 of Cortex A57.
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * --------------------------------------------------
+	 */
+func errata_a57_806969_wa
+	/*
+	 * Compare r0 against revision r0p0
+	 */
+	mov		r2, lr
+	bl		check_errata_806969
+	mov		lr, r2
+	cmp		r0, #ERRATA_NOT_APPLIES
+	beq		1f
+	ldcopr16	r0, r1, CORTEX_A57_ACTLR
+	orr64_imm	r0, r1, CORTEX_A57_ACTLR_NO_ALLOC_WBWA
+	stcopr16	r0, r1, CORTEX_A57_ACTLR
+1:
+	bx	lr
+endfunc errata_a57_806969_wa
+
+func check_errata_806969
+	mov	r1, #0x00
+	b	cpu_rev_var_ls
+endfunc check_errata_806969
+
+	/* ---------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #813419.
+	 * This applies only to revision r0p0 of Cortex A57.
+	 * ---------------------------------------------------
+	 */
+func check_errata_813419
+	/*
+	 * Even though this is only needed for revision r0p0, it
+	 * is always applied due to limitations of the current
+	 * errata framework.
+	 */
+	mov	r0, #ERRATA_APPLIES
+	bx	lr
+endfunc check_errata_813419
+
+	/* ---------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #813420.
+	 * This applies only to revision r0p0 of Cortex A57.
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * ---------------------------------------------------
+	 */
+func errata_a57_813420_wa
+	/*
+	 * Compare r0 against revision r0p0
+	 */
+	mov		r2, lr
+	bl		check_errata_813420
+	mov		lr, r2
+	cmp		r0, #ERRATA_NOT_APPLIES
+	beq		1f
+	ldcopr16	r0, r1, CORTEX_A57_ACTLR
+	orr64_imm	r0, r1, CORTEX_A57_ACTLR_DCC_AS_DCCI
+	stcopr16	r0, r1, CORTEX_A57_ACTLR
+1:
+	bx		lr
+endfunc errata_a57_813420_wa
+
+func check_errata_813420
+	mov	r1, #0x00
+	b	cpu_rev_var_ls
+endfunc check_errata_813420
+
+	/* --------------------------------------------------------------------
+	 * Disable the over-read from the LDNP instruction.
+	 *
+	 * This applies to all revisions <= r1p2. The performance degradation
+	 * observed with LDNP/STNP has been fixed on r1p3 and onwards.
+	 *
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * ---------------------------------------------------------------------
+	 */
+func a57_disable_ldnp_overread
+	/*
+	 * Compare r0 against revision r1p2
+	 */
+	mov		r2, lr
+	bl		check_errata_disable_ldnp_overread
+	mov		lr, r2
+	cmp		r0, #ERRATA_NOT_APPLIES
+	beq		1f
+	ldcopr16	r0, r1, CORTEX_A57_ACTLR
+	orr64_imm	r0, r1, CORTEX_A57_ACTLR_DIS_OVERREAD
+	stcopr16	r0, r1, CORTEX_A57_ACTLR
+1:
+	bx		lr
+endfunc a57_disable_ldnp_overread
+
+func check_errata_disable_ldnp_overread
+	mov	r1, #0x12
+	b	cpu_rev_var_ls
+endfunc check_errata_disable_ldnp_overread
+
+	/* ---------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #826974.
+	 * This applies only to revision <= r1p1 of Cortex A57.
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * ---------------------------------------------------
+	 */
+func errata_a57_826974_wa
+	/*
+	 * Compare r0 against revision r1p1
+	 */
+	mov		r2, lr
+	bl		check_errata_826974
+	mov		lr, r2
+	cmp		r0, #ERRATA_NOT_APPLIES
+	beq		1f
+	ldcopr16	r0, r1, CORTEX_A57_ACTLR
+	orr64_imm	r0, r1, CORTEX_A57_ACTLR_DIS_LOAD_PASS_DMB
+	stcopr16	r0, r1, CORTEX_A57_ACTLR
+1:
+	bx		lr
+endfunc errata_a57_826974_wa
+
+func check_errata_826974
+	mov	r1, #0x11
+	b	cpu_rev_var_ls
+endfunc check_errata_826974
+
+	/* ---------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #826977.
+	 * This applies only to revision <= r1p1 of Cortex A57.
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * ---------------------------------------------------
+	 */
+func errata_a57_826977_wa
+	/*
+	 * Compare r0 against revision r1p1
+	 */
+	mov		r2, lr
+	bl		check_errata_826977
+	mov		lr, r2
+	cmp		r0, #ERRATA_NOT_APPLIES
+	beq		1f
+	ldcopr16	r0, r1, CORTEX_A57_ACTLR
+	orr64_imm	r0, r1, CORTEX_A57_ACTLR_GRE_NGRE_AS_NGNRE
+	stcopr16	r0, r1, CORTEX_A57_ACTLR
+1:
+	bx		lr
+endfunc errata_a57_826977_wa
+
+func check_errata_826977
+	mov	r1, #0x11
+	b	cpu_rev_var_ls
+endfunc check_errata_826977
+
+	/* ---------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #828024.
+	 * This applies only to revision <= r1p1 of Cortex A57.
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * ---------------------------------------------------
+	 */
+func errata_a57_828024_wa
+	/*
+	 * Compare r0 against revision r1p1
+	 */
+	mov		r2, lr
+	bl		check_errata_828024
+	mov		lr, r2
+	cmp		r0, #ERRATA_NOT_APPLIES
+	beq		1f
+	ldcopr16	r0, r1, CORTEX_A57_ACTLR
+	/*
+	 * Setting the relevant bits in CORTEX_A57_ACTLR has to be done in 2
+	 * instructions here because the resulting bitmask doesn't fit in a
+	 * 16-bit value so it cannot be encoded in a single instruction.
+	 */
+	orr64_imm	r0, r1, CORTEX_A57_ACTLR_NO_ALLOC_WBWA
+	orr64_imm	r0, r1, (CORTEX_A57_ACTLR_DIS_L1_STREAMING | CORTEX_A57_ACTLR_DIS_STREAMING)
+	stcopr16	r0, r1, CORTEX_A57_ACTLR
+1:
+	bx		lr
+endfunc errata_a57_828024_wa
+
+func check_errata_828024
+	mov	r1, #0x11
+	b	cpu_rev_var_ls
+endfunc check_errata_828024
+
+	/* ---------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #829520.
+	 * This applies only to revision <= r1p2 of Cortex A57.
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * ---------------------------------------------------
+	 */
+func errata_a57_829520_wa
+	/*
+	 * Compare r0 against revision r1p2
+	 */
+	mov		r2, lr
+	bl		check_errata_829520
+	mov		lr, r2
+	cmp		r0, #ERRATA_NOT_APPLIES
+	beq		1f
+	ldcopr16	r0, r1, CORTEX_A57_ACTLR
+	orr64_imm	r0, r1, CORTEX_A57_ACTLR_DIS_INDIRECT_PREDICTOR
+	stcopr16	r0, r1, CORTEX_A57_ACTLR
+1:
+	bx		lr
+endfunc errata_a57_829520_wa
+
+func check_errata_829520
+	mov	r1, #0x12
+	b	cpu_rev_var_ls
+endfunc check_errata_829520
+
+	/* ---------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #833471.
+	 * This applies only to revision <= r1p2 of Cortex A57.
+	 * Inputs:
+	 * r0: variant[4:7] and revision[0:3] of current cpu.
+	 * Shall clobber: r0-r3
+	 * ---------------------------------------------------
+	 */
+func errata_a57_833471_wa
+	/*
+	 * Compare r0 against revision r1p2
+	 */
+	mov		r2, lr
+	bl		check_errata_833471
+	mov		lr, r2
+	cmp		r0, #ERRATA_NOT_APPLIES
+	beq		1f
+	ldcopr16	r0, r1, CORTEX_A57_ACTLR
+	orr64_imm	r1, r1, CORTEX_A57_ACTLR_FORCE_FPSCR_FLUSH
+	stcopr16	r0, r1, CORTEX_A57_ACTLR
+1:
+	bx		lr
+endfunc errata_a57_833471_wa
+
+func check_errata_833471
+	mov	r1, #0x12
+	b	cpu_rev_var_ls
+endfunc check_errata_833471
+
 	/* -------------------------------------------------
 	 * The CPU Ops reset function for Cortex-A57.
+	 * Shall clobber: r0-r6
 	 * -------------------------------------------------
 	 */
 func cortex_a57_reset_func
+	mov	r5, lr
+	bl	cpu_get_rev_var
+	mov	r4, r0
+
+#if ERRATA_A57_806969
+	mov	r0, r4
+	bl	errata_a57_806969_wa
+#endif
+
+#if ERRATA_A57_813420
+	mov	r0, r4
+	bl	errata_a57_813420_wa
+#endif
+
+#if A57_DISABLE_NON_TEMPORAL_HINT
+	mov	r0, r4
+	bl	a57_disable_ldnp_overread
+#endif
+
+#if ERRATA_A57_826974
+	mov	r0, r4
+	bl	errata_a57_826974_wa
+#endif
+
+#if ERRATA_A57_826977
+	mov	r0, r4
+	bl	errata_a57_826977_wa
+#endif
+
+#if ERRATA_A57_828024
+	mov	r0, r4
+	bl	errata_a57_828024_wa
+#endif
+
+#if ERRATA_A57_829520
+	mov	r0, r4
+	bl	errata_a57_829520_wa
+#endif
+
+#if ERRATA_A57_833471
+	mov	r0, r4
+	bl	errata_a57_833471_wa
+#endif
+
 	/* ---------------------------------------------
 	 * Enable the SMP bit.
 	 * ---------------------------------------------
@@ -63,7 +364,7 @@ func cortex_a57_reset_func
 	orr64_imm	r0, r1, CORTEX_A57_ECTLR_SMP_BIT
 	stcopr16	r0, r1,	CORTEX_A57_ECTLR
 	isb
-	bx	lr
+	bx	r5
 endfunc cortex_a57_reset_func
 
 	/* ----------------------------------------------------
@@ -162,6 +463,36 @@ func cortex_a57_cluster_pwr_dwn
 	b	cortex_a57_disable_ext_debug
 endfunc cortex_a57_cluster_pwr_dwn
 
+#if REPORT_ERRATA
+/*
+ * Errata printing function for Cortex A57. Must follow AAPCS.
+ */
+func cortex_a57_errata_report
+	push	{r12, lr}
+
+	bl	cpu_get_rev_var
+	mov	r4, r0
+
+	/*
+	 * Report all errata. The revision-variant information is passed to
+	 * checking functions of each errata.
+	 */
+	report_errata ERRATA_A57_806969, cortex_a57, 806969
+	report_errata ERRATA_A57_813419, cortex_a57, 813419
+	report_errata ERRATA_A57_813420, cortex_a57, 813420
+	report_errata A57_DISABLE_NON_TEMPORAL_HINT, cortex_a57, \
+		disable_ldnp_overread
+	report_errata ERRATA_A57_826974, cortex_a57, 826974
+	report_errata ERRATA_A57_826977, cortex_a57, 826977
+	report_errata ERRATA_A57_828024, cortex_a57, 828024
+	report_errata ERRATA_A57_829520, cortex_a57, 829520
+	report_errata ERRATA_A57_833471, cortex_a57, 833471
+
+	pop	{r12, lr}
+	bx	lr
+endfunc cortex_a57_errata_report
+#endif
+
 declare_cpu_ops cortex_a57, CORTEX_A57_MIDR, \
 	cortex_a57_reset_func, \
 	cortex_a57_core_pwr_dwn, \
diff --git a/lib/cpus/aarch32/cpu_helpers.S b/lib/cpus/aarch32/cpu_helpers.S
index dc78f6ee..bfdc1e4f 100644
--- a/lib/cpus/aarch32/cpu_helpers.S
+++ b/lib/cpus/aarch32/cpu_helpers.S
@@ -182,6 +182,19 @@ func cpu_rev_var_ls
 	bx	lr
 endfunc cpu_rev_var_ls
 
+/*
+ * Compare the CPU's revision-variant (r0) with a given value (r1), for errata
+ * application purposes. If the revision-variant is higher than or same as a
+ * given value, indicates that errata applies; otherwise not.
+ */
+	.globl	cpu_rev_var_hs
+func cpu_rev_var_hs
+	cmp	r0, r1
+	movge	r0, #ERRATA_APPLIES
+	movlt	r0, #ERRATA_NOT_APPLIES
+	bx	lr
+endfunc cpu_rev_var_hs
+
 #if REPORT_ERRATA
 /*
  * void print_errata_status(void);