ENGR00334447 [imx6qdl] Fix random failures caused by ddr frequency change procedure

Backport busfreq patch from 3.10.x kernel:
2d1f2b76919e13d8744cc8579316c002f832c675

The ddr frequency change code was responsible for some random kernel crashes.
This was due to the fact that L1 and L2 caches were not correctly flushed/synced
during the frequency change procedure.
This patch attempts to fix the issue by:
1. Ensure that every active core put into wfe flushes and disables its L1.
2. All cores (except the one executing the ddr freq change code) informs the SCU
that it going into a power down state after flushing and disabling its L1. It
also removes itself out the SMP cluster.
3. Variables shared across cores are stored in non-cacheable IRAM space.
4. SCU power status register is used to identify if all cores have reached a quiscent state
before the core running the ddr freq change code proceeds further.

Signed-off-by: Ranjani Vaidyanathan <Ranjani.Vaidyanathan@freescale.com>

3 files changed, 244 insertions, 36 deletions

diff --git a/arch/arm/mach-mx6/mx6_ddr_freq.S b/arch/arm/mach-mx6/mx6_ddr_freq.S
index 716a67b4bd7a..4705207a3fee 100644
--- a/arch/arm/mach-mx6/mx6_ddr_freq.S
+++ b/arch/arm/mach-mx6/mx6_ddr_freq.S
@@ -17,11 +17,18 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/smp_scu.h>
 #include <mach/hardware.h>
 
+#define L2_CACHE_SYNC	0x730
+
 .extern iram_tlb_phys_addr
+.extern imx_scu_base
+
 .globl mx6_ddr3_iram_start
 .globl mx6_ddr3_iram_end
+.globl wfe_ddr3_freq_change_start
+.globl wfe_ddr3_freq_change_end
 
 	.macro   switch_to_528MHz
 
@@ -374,6 +381,36 @@ skip_gpt_workaround4:
 
 	.endm
 
+	.macro	disable_l1_dcache
+
+	/*
+	 * Flush all data from the L1 data cache before disabling
+	 * SCTLR.C bit.
+	 */
+	push	{r0 - r11, lr}
+
+	ldr	r7, =v7_flush_kern_cache_all
+	mov	lr, pc
+	mov	pc, r7
+	pop	{r0 - r11, lr}
+
+	/* disable d-cache */
+	mrc	p15, 0, r6, c1, c0, 0
+	bic	r6, r6, #0x4
+	mcr	p15, 0, r6, c1, c0, 0
+	dsb
+	isb
+
+	push	{r0 - r11, lr}
+
+	ldr	r7, =v7_flush_kern_cache_all
+	mov	lr, pc
+	mov	pc, r7
+	pop	{r0 - r11, lr}
+
+	.endm
+
+
 /*
  *  mx6_ddr_freq_change
  *
@@ -404,8 +441,44 @@ ddr_freq_change:
 	 * 4. Set TTBR0.N=1, implying 0-2G is translated by TTBR0
 	 *     and 2-4G is translated by TTBR1.
 	 */
-	ldr    r6, =iram_tlb_phys_addr
-	ldr    r7, [r6]
+	/* flush the TLB */
+	ldr	r6, =0x0
+	mcr	p15, 0, r6, c8, c3, 0
+
+	ldr	r6, =iram_tlb_phys_addr
+	ldr	r7, [r6]
+
+	/*
+	  * Need to flush and disable L1 before disabling L2, we need data to
+	  * coherent. Flushing L1 pushes everyhting to L2. We sync L2 later, but
+	  * it can still have dirty lines. While exiting, we need to enable L2 first
+	  * and then L1.
+	 . */
+	disable_l1_dcache
+
+#ifdef CONFIG_CACHE_L2X0
+	/*
+	 * Make sure the L2 buffers are drained.
+	 * Sync operation on L2 drains the buffers.
+	 */
+	ldr   r6, =L2_BASE_ADDR
+	add   r6, r6, #PERIPBASE_VIRT
+
+	/* Wait for background operations to complete. */
+wait_for_l2_to_idle:
+	ldr	r1, [r6, #L2_CACHE_SYNC]
+	cmp	r1, #0x0
+	bne	wait_for_l2_to_idle
+
+	mov	r1, #0x0
+	str	r1, [r6, #L2_CACHE_SYNC]
+
+	/* Disable L2. */
+	str	r1, [r6, #0x100]
+
+	dsb
+	isb
+#endif
 
 	/* Disable Branch Prediction, Z bit in SCTLR. */
 	mrc    p15, 0, r6, c1, c0, 0
@@ -433,6 +506,7 @@ ddr_freq_change:
 	ldr    r6, =0x0
 	mcr    p15, 0, r6, c8, c3, 0
 
+#if RANJANI
 	/* Disable L1 data cache. */
 	mrc    p15, 0, r6, c1, c0, 0
 	bic    r6, r6, #0x4
@@ -455,10 +529,11 @@ wait_for_l2_to_idle:
 	str   r6, [r7, #0x730]
 	/* Disable L2. */
 	str   r6, [r7, #0x100]
+#endif
+#endif
 
 	dsb
 	isb
-#endif
 
 	ldr     r6, =CCM_BASE_ADDR
 	add     r6, r6, #PERIPBASE_VIRT
@@ -793,7 +868,7 @@ poll_dvfs_clear_2:
 	cmp    	r9, #0
 	beq 	       update_calibration_only
 
-	ldr     r0, =0xa5390003
+	ldr     r0, =0xa1390003
 	str     r0, [r5, #0x800]
 	ldr     r2, =0x4800
 	str     r0, [r5, r2]
@@ -1056,6 +1131,8 @@ done:
 	add   r1, r1, #PERIPBASE_VIRT
 	ldr   r6, =0x1
 	str   r6, [r1, #0x100]
+	dsb
+	isb
 #endif
 
 	/* Enable L1 data cache. */
@@ -1086,10 +1163,15 @@ done:
 	orr    r6, r6, #0x800
 	mcr    p15, 0, r6, c1, c0, 0
 
+	isb
+
 	/* Flush the Branch Target Address Cache (BTAC) */
 	ldr    r6, =0x0
 	mcr    p15, 0, r6, c7, c1, 6
 
+	isb
+	dsb
+
 	/* Restore registers */
 	ldmfd sp!, {r4 - r12}
 
@@ -1101,3 +1183,75 @@ done:
 	 */
 	.ltorg
 mx6_ddr3_iram_end:
+
+	.align 3
+
+ENTRY(wfe_ddr3_freq_change)
+wfe_ddr3_freq_change_start:
+	push	{r4 - r11, lr}
+
+	mov	r6, r0
+	mov	r7, r1
+
+	dsb
+	isb
+
+	disable_l1_dcache
+
+	isb
+
+	/* Turn off SMP bit. */
+	mrc	p15, 0, r8, c1, c0, 1
+	bic	r8, r8, #0x40
+	mcr	p15, 0, r8, c1, c0, 1
+
+	isb
+
+	/* Inform the SCU we are going to enter WFE. */
+	push	{r0 - r11, lr}
+
+	ldr	r0,=imx_scu_base
+	ldr	r0, [r0]
+	mov	r1, #SCU_PM_DORMANT
+	ldr	r3, =scu_power_mode
+	mov	lr, pc
+	mov	pc, r3
+
+	pop	{r0 - r11, lr}
+
+go_back_wfe:
+	wfe
+
+	ldr	r3, [r7]
+	cmp	r3, #1
+	beq	go_back_wfe
+
+	/* Turn ON SMP bit. */
+	mrc	p15, 0, r8, c1, c0, 1
+	orr	r8, r8, #0x40
+	mcr	p15, 0, r8, c1, c0, 1
+
+	isb
+	/* Enable L1 data cache. */
+	mrc	p15, 0, r8, c1, c0, 0
+	orr	r8, r8, #0x4
+	mcr	p15, 0, r8, c1, c0, 0
+	isb
+
+	/* Inform the SCU we have exited WFE. */
+	push	{r0 - r11, lr}
+
+	ldr	r0,=imx_scu_base
+	ldr	r0, [r0]
+	mov	r1, #SCU_PM_NORMAL
+	ldr	r3, =scu_power_mode
+	mov	lr, pc
+	mov	pc, r3
+
+	pop	{r0 - r11, lr}
+
+	/* Pop all saved registers. */
+	pop	{r4 - r11, lr}
+	mov	pc, lr
+	.ltorg
+wfe_ddr3_freq_change_end:
diff --git a/arch/arm/mach-mx6/mx6_mmdc.c b/arch/arm/mach-mx6/mx6_mmdc.c
index 907e8aaee294..71d6ede4b49a 100644
--- a/arch/arm/mach-mx6/mx6_mmdc.c
+++ b/arch/arm/mach-mx6/mx6_mmdc.c
@@ -22,6 +22,8 @@
 #include <linux/sched.h>
 #include <linux/proc_fs.h>
 #include <linux/clk.h>
+#include <linux/clockchips.h>
+#include <asm/fncpy.h>
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
 #include <linux/iram_alloc.h>
@@ -48,6 +50,7 @@ void __iomem *mmdc_base;
 void __iomem *iomux_base;
 void __iomem *gic_dist_base;
 void __iomem *gic_cpu_base;
+void __iomem *imx_scu_base;
 
 void (*mx6_change_ddr_freq)(u32 freq, void *ddr_settings, bool dll_mode, void* iomux_offsets) = NULL;
 
@@ -55,6 +58,7 @@ void (*mx6sl_lpddr2_change_freq)(u32 freq, int low_bus_freq_mode,
 								void *ddr_settings) = NULL;
 
 void (*mx6sl_ddr3_change_freq)(u32 freq, void *ddr_settings, bool dll_mode, void* iomux_offsets, int low_bus_freq_mode) = NULL;
+void (*wfe_change_ddr_freq)(u32 cpuid, u32 *ddr_freq_change_done);
 
 extern unsigned int ddr_low_rate;
 extern unsigned int ddr_med_rate;
@@ -62,10 +66,12 @@ extern unsigned int ddr_normal_rate;
 extern int low_bus_freq_mode;
 extern int audio_bus_freq_mode;
 extern int mmdc_med_rate;
+extern unsigned long iram_tlb_phys_addr;
 extern void __iomem *ccm_base;
 extern void mx6_ddr_freq_change(u32 freq, void *ddr_settings, bool dll_mode, void *iomux_offsets);
 extern void mx6sl_ddr_iram(int ddr_freq, int low_bus_freq_mode, void *ddr_settings);
 extern void mx6sl_ddr3_freq_change(u32 freq, void *ddr_settings, bool dll_mode, void *iomux_offsets, int low_bus_freq_mode);
+extern void wfe_ddr3_freq_change(u32 cpuid, u32 *ddr_freq_change_done);
 extern unsigned long save_ttbr1(void);
 extern void restore_ttbr1(u32 ttbr1);
 
@@ -75,16 +81,20 @@ extern unsigned long mx6sl_lpddr2_iram_end asm("mx6sl_lpddr2_iram_end");
 extern unsigned long mx6sl_lpddr2_iram_start asm("mx6sl_lpddr2_iram_start");
 extern unsigned long mx6sl_ddr3_iram_end asm("mx6sl_ddr3_iram_end");
 extern unsigned long mx6sl_ddr3_iram_start asm("mx6sl_ddr3_iram_start");
+extern unsigned long wfe_ddr3_freq_change_start asm("wfe_ddr3_freq_change_start");
+extern unsigned long wfe_ddr3_freq_change_end asm("wfe_ddr3_freq_change_end");
 
 
 unsigned long ddr_freq_change_iram_phys_addr;
+u32 *wait_for_ddr_freq_update;
+
 static void *ddr_freq_change_iram_base;
 static int ddr_settings_size;
 static int iomux_settings_size;
 static volatile unsigned int cpus_in_wfe;
-static volatile bool wait_for_ddr_freq_update;
 static int curr_ddr_rate;
 static unsigned int ddr_type;
+static unsigned long wfe_freq_change_iram_base;
 
 #define MIN_DLL_ON_FREQ		333000000
 #define MAX_DLL_OFF_FREQ		125000000
@@ -232,19 +242,23 @@ int can_change_ddr_freq(void)
   */
 irqreturn_t wait_in_wfe_irq(int irq, void *dev_id)
 {
-	u32 me = smp_processor_id();
-
-	*((char *)(&cpus_in_wfe) + (u8)me) = 0xff;
+	u32 me;
 
-	while (wait_for_ddr_freq_update)
-		wfe();
+	me = smp_processor_id();
+#ifdef CONFIG_LOCAL_TIMERS
+		clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER,
+				   &me);
+#endif
+	wfe_change_ddr_freq(0xff << (me * 8), (u32 *)&iram_iomux_settings[0][1]);
 
-	*((char *)(&cpus_in_wfe) + (u8)me) = 0;
+#ifdef CONFIG_LOCAL_TIMERS
+		clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT,
+				   &me);
+#endif
 
 	return IRQ_HANDLED;
 }
 
-extern unsigned long iram_tlb_phys_addr;
 /* Change the DDR frequency. */
 int update_ddr_freq(int ddr_rate)
 {
@@ -335,20 +349,43 @@ int update_ddr_freq(int ddr_rate)
 
 	me = smp_processor_id();
 
-	*((char *)(&cpus_in_wfe) + (u8)me) = 0xff;
-	wait_for_ddr_freq_update = true;
+	/* Make sure all the online cores are active */
+	while (1) {
+		bool not_exited_busfreq = false;
+		for_each_online_cpu(cpu) {
+			u32 reg = __raw_readl(imx_scu_base + 0x08);
+			if (reg & (0x02 << (cpu * 8)))
+				not_exited_busfreq = true;
+		}
+		if (!not_exited_busfreq)
+			break;
+	}
+
+	wmb();
+	*wait_for_ddr_freq_update = 1;
+	dsb();
+
+	online_cpus = readl_relaxed(imx_scu_base + 0x08);
 	for_each_online_cpu(cpu) {
-		*((char *)(&online_cpus) + (u8)cpu) = 0xff;
-		if (!cpu_is_mx6sl()) {
-			if (cpu != me) {
-				/* Set the interrupt to be pending in the GIC. */
-				reg = 1 << (irq_used[cpu] % 32);
-				writel_relaxed(reg, gic_dist_base + GIC_DIST_PENDING_SET + (irq_used[cpu] / 32) * 4);
-			}
+		*((char *)(&online_cpus) + (u8)cpu) = 0x02;
+		if (cpu != me) {
+			/* set the interrupt to be pending in the GIC. */
+			reg = 1 << (irq_used[cpu] % 32);
+			writel_relaxed(reg, gic_dist_base + GIC_DIST_PENDING_SET
+				+ (irq_used[cpu] / 32) * 4);
 		}
 	}
-	while (cpus_in_wfe != online_cpus)
-		udelay(5);
+	/* Wait for the other active CPUs to idle */
+	while (1) {
+		u32 reg = readl_relaxed(imx_scu_base + 0x08);
+		reg |= (0x02 << (me * 8));
+		if (reg == online_cpus)
+			break;
+	}
+
+	/* Ensure iram_tlb_phys_addr is flushed to DDR. */
+	__cpuc_flush_dcache_area(&iram_tlb_phys_addr, sizeof(iram_tlb_phys_addr));
+	outer_clean_range(virt_to_phys(&iram_tlb_phys_addr), virt_to_phys(&iram_tlb_phys_addr + 1));
 
 	/* Save TTBR1 */
 	ttbr1 = save_ttbr1();
@@ -366,15 +403,15 @@ int update_ddr_freq(int ddr_rate)
 	curr_ddr_rate = ddr_rate;
 
 	/* DDR frequency change is done . */
-	wait_for_ddr_freq_update = false;
+	*wait_for_ddr_freq_update = false;
 
 	/* Wake up all the cores. */
 	sev();
 
-	*((char *)(&cpus_in_wfe) + (u8)me) = 0;
-
 	local_irq_enable();
 
+	printk(KERN_DEBUG "Bus freq set to %d done! cpu=%d\n", ddr_rate, me);
+
 	return 0;
 }
 
@@ -382,6 +419,9 @@ int init_mmdc_settings(void)
 {
 	int i, err, cpu;
 	unsigned long ddr_code_size = 0;
+	unsigned long wfe_code_size = 0;
+
+	imx_scu_base = IO_ADDRESS(SCU_BASE_ADDR);
 
 	mmdc_base = ioremap(MMDC_P0_BASE_ADDR, SZ_32K);
 	iomux_base = ioremap(MX6Q_IOMUXC_BASE_ADDR, SZ_16K);
@@ -398,6 +438,7 @@ int init_mmdc_settings(void)
 			ddr_code_size = (&mx6sl_ddr3_iram_end -&mx6sl_ddr3_iram_start) *4;
 		} else {
 			ddr_code_size = (&mx6_ddr3_iram_end -&mx6_ddr3_iram_start) *4;
+			wfe_code_size = (&wfe_ddr3_freq_change_end -&wfe_ddr3_freq_change_start) *4;
 			if (cpu_is_mx6q()) {
 				ddr_settings_size = ARRAY_SIZE(ddr3_dll_mx6q) + ARRAY_SIZE(ddr3_calibration);
 				iomux_settings_size = ARRAY_SIZE(iomux_offsets_mx6q);
@@ -467,7 +508,7 @@ int init_mmdc_settings(void)
 	iram_iomux_settings = ddr_freq_change_iram_base + ddr_code_size;
 	iram_ddr_settings = iram_iomux_settings + (iomux_settings_size * 8) + 8;
 
-	if ((ddr_code_size + (iomux_settings_size + ddr_settings_size) * 8 + 16)
+	if ((ddr_code_size + wfe_code_size + (iomux_settings_size + ddr_settings_size) * 8 + 16)
 		> MX6_IRAM_DDR_FREQ_CODE_SIZE) {
 		printk(KERN_ERR "Not enough memory allocated for DDR Frequency change code.\n");
 		return -EINVAL;
@@ -505,6 +546,18 @@ int init_mmdc_settings(void)
 			}
 		}
 	}
+
+	wfe_freq_change_iram_base = (unsigned long)((u32 *)iram_ddr_settings + (ddr_settings_size * 8) + 8);
+
+	if (wfe_freq_change_iram_base & (FNCPY_ALIGN - 1))
+			wfe_freq_change_iram_base += FNCPY_ALIGN - ((uintptr_t)wfe_freq_change_iram_base % (FNCPY_ALIGN));
+
+	wfe_change_ddr_freq = (void *)fncpy((void *)wfe_freq_change_iram_base,
+		&wfe_ddr3_freq_change, wfe_code_size);
+
+	/* Store the variable used to communicate between cores in a non-cacheable IRAM area */
+	wait_for_ddr_freq_update = (u32 *)&iram_iomux_settings[0][1];
+
 	curr_ddr_rate = ddr_normal_rate;
 
 	if (!cpu_is_mx6sl()) {
diff --git a/arch/arm/mach-mx6/mx6_suspend.S b/arch/arm/mach-mx6/mx6_suspend.S
index 0fa7b5e15b55..fdecbd2bf417 100644
--- a/arch/arm/mach-mx6/mx6_suspend.S
+++ b/arch/arm/mach-mx6/mx6_suspend.S
@@ -1153,15 +1153,15 @@ ddr_io_save_done:
 	 * 4. Set TTBR0.N=1, implying 0-2G is translated by TTBR0
 	 *     and 2-4G is translated by TTBR1.
 	 */
+	/* Flush the BTAC. */
+	ldr r6, =0x0
+	mcr p15, 0, r6, c7, c1, 6
+
 	/* Disable Branch Prediction, Z bit in SCTLR. */
 	mrc    p15, 0, r6, c1, c0, 0
 	bic    r6, r6, #0x800
 	mcr    p15, 0, r6, c1, c0, 0
 
-	/* Flush the BTAC. */
-	ldr r6, =0x0
-	mcr p15, 0, r6, c7, c1, 6
-
 	ldr r6, =iram_tlb_phys_addr
 	ldr r6, [r6]
 	dsb
@@ -1186,7 +1186,7 @@ ddr_io_save_done:
 	bic r6, r6, #0x4
 	mcr p15, 0, r6, c1, c0, 0
 
-    dsb
+	dsb
 
 	/* Disable L2 cache */
 #ifdef CONFIG_CACHE_L2X0
@@ -1540,10 +1540,6 @@ poll_dvfs_clear_1:
 	nop
 	nop
 
-	mrc	p15, 0, r1, c1, c0, 0
-	orr	r1, r1, #(1 << 2)	@ Enable the C bit
-	mcr	p15, 0, r1, c1, c0, 0
-
 	/* Enable L2 cache here */
 #ifdef CONFIG_CACHE_L2X0
 	ldr r2, =L2_BASE_ADDR
@@ -1552,9 +1548,14 @@ poll_dvfs_clear_1:
 	str r4, [r2, #L2X0_CTRL]
 #endif
 
+	mrc	p15, 0, r1, c1, c0, 0
+	orr	r1, r1, #(1 << 2)	@ Enable the C bit
+	mcr	p15, 0, r1, c1, c0, 0
+
 	/* Restore TTBCR */
 	dsb
 	isb
+
 	/* Read TTBCR and set PD0=0, N = 0 */
 	mrc    p15, 0, r6, c2, c0, 2
 	bic    r6, r6, #0x11