From 058a759ffed26797314bc0a52db3c440240fa6a5 Mon Sep 17 00:00:00 2001 From: Bai Ping Date: Fri, 28 Sep 2018 14:55:31 +0800 Subject: imx8m: Add DDR4 DVFS support Add DDR4 DVFS support for i.MX8M. Currently, only tested on i.MX8MM. Signed-off-by: Bai Ping --- plat/imx/common/imx8m/ddr4_dvfs.c | 355 ++++++++++++++++++++++++++++++++++++++ plat/imx/common/imx8m/dram.c | 24 ++- plat/imx/common/include/dram.h | 4 + plat/imx/imx8mm/platform.mk | 3 +- 4 files changed, 377 insertions(+), 9 deletions(-) create mode 100644 plat/imx/common/imx8m/ddr4_dvfs.c diff --git a/plat/imx/common/imx8m/ddr4_dvfs.c b/plat/imx/common/imx8m/ddr4_dvfs.c new file mode 100644 index 00000000..71237f0a --- /dev/null +++ b/plat/imx/common/imx8m/ddr4_dvfs.c @@ -0,0 +1,355 @@ +/* + * Copyright 2018 NXP + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include + +unsigned fsp_init_reg[3][4] = { + { DDRC_INIT3(0), DDRC_INIT4(0), DDRC_INIT6(0), DDRC_INIT7(0) }, + { DDRC_FREQ1_INIT3(0), DDRC_FREQ1_INIT4(0), DDRC_FREQ1_INIT6(0), DDRC_FREQ1_INIT7(0) }, + { DDRC_FREQ2_INIT3(0), DDRC_FREQ2_INIT4(0), DDRC_FREQ2_INIT6(0), DDRC_FREQ2_INIT7(0) }, +}; + +static unsigned int mr_value[7]; + +void ddr4_mr_write(unsigned int mr, unsigned int data, unsigned int mr_type, unsigned int rank) +{ + unsigned int tmp, mr_mirror, data_mirror; + /* + * 1. Poll MRSTAT.mr_wr_busy until it is 0 to make sure that there is + * no outstanding MR transAction. No + */ + do { + tmp = mmio_read_32(DDRC_MRSTAT(0)); + } while (tmp & 0x1); + + /* + * 2. Write the MRCTRL0.mr_type, MRCTRL0.mr_addr, MRCTRL0.mr_rank + * and (for MRWs) MRCTRL1.mr_data to define the MR transaction. + */ + tmp = mmio_read_32(DDRC_DIMMCTL(0)); + if ((tmp & 0x2) && (rank == 0x2)) { + mr_mirror = (mr & 0x4) | ((mr & 0x1) << 1) | ((mr & 0x2) >> 1); /* BA0, BA1 swap */ + data_mirror = (data & 0x1607) | ((data & 0x8) << 1) | ((data & 0x10) >> 1) | + ((data & 0x20) << 1) | ((data & 0x40) >> 1) | ((data & 0x80) << 1) | + ((data & 0x100) >> 1) | ((data & 0x800) << 2) | ((data & 0x2000) >> 2) ; + } else { + mr_mirror = mr; + data_mirror = data; + } + + mmio_write_32(DDRC_MRCTRL0(0), mr_type | (mr_mirror << 12) | (rank << 4) ); + mmio_write_32(DDRC_MRCTRL1(0), data_mirror ); + + /* + * 3. In a separate APB transaction, write the MRCTRL0.mr_wr to 1. This bit is + * self-clearing, and triggers the MR transaction. The uMCTL2 then asserts the + * MRSTAT.mr_wr_busy while it performs the MR transaction to SDRAM, and no further + * accesses can be initiated until it is deasserted. + */ + mmio_setbits_32(DDRC_MRCTRL0(0), (1 << 31)); + do { + tmp = mmio_read_32(DDRC_MRSTAT(0)); + } while (tmp); +} + +void dram_cfg_all_mr(unsigned int num_rank, unsigned int pstate) +{ + /* + * 15. Perform MRS commands as required to re-program timing registers + * in the SDRAM for the new frequency (in particular, CL, CWL and WR + * may need to be changed). + */ + + for (int i = 1; i <= num_rank; i++) { + for (int j = 0; j < 7; j++) + ddr4_mr_write(j, mr_value[j], 0, i); + } +} + +void sw_pstate(unsigned int pstate) +{ + volatile unsigned int tmp; + unsigned int i; + + mmio_write_32(DDRC_SWCTL(0), 0x0000); + /* 12. Change the clock frequency to the desired value. */ + + /* + * 13. Update any registers which may be required to change for the new frequency. This includes + * quasidynamic and dynamic registers. This includes both uMCTL2 registers and PHY registers. + */ + mmio_write_32(DDRC_MSTR2(0), pstate); /* UMCTL2_REGS_FREQ1 */ + mmio_setbits_32(DDRC_MSTR(0), (0x1 << 29)); + + /* + * dvfs.18. Toggle RFSHCTL3.refresh_update_level to allow the new refresh-related + * register values to propagate to the refresh logic. + */ + tmp = mmio_read_32(DDRC_RFSHCTL3(0)); + if ((tmp & 0x2) == 0x2) + mmio_write_32(DDRC_RFSHCTL3(0), tmp & 0xFFFFFFFD); + else + mmio_write_32(DDRC_RFSHCTL3(0), tmp | 0x2); + /* + *dvfs.19. If required, trigger the initialization in the PHY. If using the gen2 + * multiPHY, PLL initialization should be triggered at this point. See the PHY + * databook for details about the frequency change procedure. + */ + /* before write Dynamic reg, sw_done should be 0 */ + mmio_write_32(DDRC_DFIMISC(0), 0x00000000 | (pstate << 8));//pstate1 + mmio_write_32(DDRC_DFIMISC(0), 0x00000020 | (pstate << 8)); + + /* wait DFISTAT.dfi_init_complete to 0 */ + do { + tmp = 0x1 & mmio_read_32(DDRC_DFISTAT(0)); + } while (tmp); + + /* change the clock to the target frequency */ + dram_clock_switch(pstate); + + mmio_write_32(DDRC_DFIMISC(0), 0x00000000 | (pstate << 8)); + /* wait DFISTAT.dfi_init_complete to 1 */ + do { + tmp = 0x1 & mmio_read_32(DDRC_DFISTAT(0)); + } while (!tmp); + + /* + * When changing frequencies the controller may violate the JEDEC + * requirement that no more than 16 refreshes should be issued within + * 2*tREFI. These extra refreshes are not expected to cause a problem + * in the SDRAM. This issue can be avoided by waiting for at least 2*tREFI + * before exiting self-refresh in step 19. + */ + for (i = 20; i > 0; i--) + printf("waiting for 2*tREF (2*7.8us)\n"); + + /* 14. Exit the self-refresh state by setting PWRCTL.selfref_sw = 0. */ + mmio_clrbits_32(DDRC_PWRCTL(0), (1 << 5)); + do { + tmp = 0x3f & (mmio_read_32((DDRC_STAT(0)))); + } while (tmp == 0x23); +} + +void ddr4_dll_change(unsigned int num_rank, unsigned int pstate, unsigned int cur_pstate) +{ + volatile unsigned int tmp; + enum DLL_STATE { NO_CHANGE=0, ON2OFF=1, OFF2ON=2} dll_sw; /* 0-no change, 1-on2off, 2-off2on.; */ + + if (pstate != 0 && cur_pstate == 0) + dll_sw = ON2OFF; + else if (pstate == 0 && cur_pstate != 0) + dll_sw = OFF2ON; + else + dll_sw = NO_CHANGE; + + /* the the following software programming sequence to switch from DLL-on to DLL-off, or reverse: */ + mmio_write_32(DDRC_SWCTL(0), 0x0000); + + mmio_write_32(DDRC_PCTRL_0(0), 0x00000000); + + /* 1. Set the DBG1.dis_hif = 1. This prevents further reads/writes being received on the HIF. */ + mmio_setbits_32(DDRC_DBG1(0), (0x1 << 1)); + + /* 2. Set ZQCTL0.dis_auto_zq=1, to disable automatic generation of ZQCS/MPC(ZQ calibration) */ + mmio_setbits_32(DDRC_FREQ1_ZQCTL0(0), (1 << 31)); + mmio_setbits_32(DDRC_FREQ2_ZQCTL0(0), (1 << 31)); + mmio_setbits_32(DDRC_ZQCTL0(0), (1 << 31)); + + /* 3. Set RFSHCTL3.dis_auto_refresh=1, to disable automatic refreshes */ + /* mmio_setbits_32(DDRC_RFSHCTL3(0), 0x1); */ + /* 4. Ensure all commands have been flushed from the uMCTL2 by polling */ + do { + tmp = 0x36000000 & mmio_read_32(DDRC_DBGCAM(0)); + } while (tmp != 0x36000000); + + /* mmio_write_32(DDRC_PCTRL_0(0), 0x00000000); */ + + /* 5. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to disable RTT_NOM: */ + /* a. DDR3: Write 0 to MR1[9], MR1[6] and MR1[2] */ + /* b. DDR4: Write 0 to MR1[10:8] */ + for (int i = 1; i <= num_rank; i++) { + if (mr_value[1] & 0x700) + ddr4_mr_write(1, mr_value[1] & 0xF8FF, 0, i); + } + + /* 6. For DDR4 only: Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to write 0 to */ + /* MR5[8:6] to disable RTT_PARK */ + for (int i = i; i < num_rank; i++) { + if (mr_value[5] & 0x1C0) + ddr4_mr_write(5, mr_value[5] & 0xFE3F, 0, i); + } + + if(dll_sw == ON2OFF) { + /* 7. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to write 0 to MR2[11:9], to */ + /* disable RTT_WR (and therefore disable dynamic ODT). This applies for both DDR3 and DDR4. */ + for (int i = 1; i <= num_rank; i++) { + if (mr_value[2] & 0xE00) + ddr4_mr_write(2, mr_value[2] & 0xF1FF, 0, i); + } + + /* 8. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to disable the DLL. The */ + /* timing of this MRS is automatically handled by the uMCTL2. */ + /* a. DDR3: Write 1 to MR1[0] */ + /* b. DDR4: Write 0 to MR1[0] */ + for (int i = 1; i <= num_rank; i++) { + ddr4_mr_write(1, mr_value[1] & 0xFFFE, 0, i); + } + } + + /* 9. Put the SDRAM into self-refresh mode by setting PWRCTL.selfref_sw = 1, and polling */ + /* STAT.operating_mode to ensure the DDRC has entered self-refresh. */ + mmio_setbits_32(DDRC_PWRCTL(0), (1 << 5)); + /* + * 10. Wait until STAT.operating_mode[1:0]==11 indicating that the DWC_ddr_umctl2 + * core is in selfrefresh mode. Ensure transition to self-refresh was due to software + * by checking that + */ + /* STAT.selfref_type[1:0]=2`b10. */ + do { + tmp = 0x3f & (mmio_read_32((DDRC_STAT(0)))); + } while (tmp != 0x23); + + /* FIXME Set RFSHCTL3.dis_auto_refresh=1, to disable automatic refreshes ??? */ + mmio_setbits_32(DDRC_RFSHCTL3(0), 0x1); + + /* 11. Set the MSTR.dll_off_mode = 1 or 0. */ + if (dll_sw == ON2OFF) + mmio_setbits_32(DDRC_MSTR(0), (1 << 15)); + + if(dll_sw == OFF2ON) + mmio_clrbits_32(DDRC_MSTR(0), (1 << 15)); + + sw_pstate(pstate); + + /* DRAM dll enable */ + if (dll_sw == OFF2ON) { + for (int i = 1; i <= num_rank; i++) { + ddr4_mr_write(1, mr_value[1] | 0x1, 0, i); + } + + for (int i = 1; i <= num_rank; i++) { + ddr4_mr_write(0, mr_value[0] | 0x100, 0, i); + } + } + + dram_cfg_all_mr(num_rank, pstate); + + /* 16. Re-enable automatic generation of ZQCS/MPC(ZQ calibration) commands */ + mmio_clrbits_32(DDRC_FREQ1_ZQCTL0(0), (1 << 31)); + mmio_clrbits_32(DDRC_FREQ2_ZQCTL0(0), (1 << 31)); + mmio_clrbits_32(DDRC_ZQCTL0(0), (1 << 31)); + + /* 17. Re-enable automatic refreshes (RFSHCTL3.dis_auto_refresh = 0) if they have been previously disabled. */ + mmio_clrbits_32(DDRC_RFSHCTL3(0), 0x1); + /* 18. Restore ZQCTL0.dis_srx_zqcl */ + /* 19. Write DBG1.dis_hif = 0 to re-enable reads and writes. */ + mmio_clrbits_32(DDRC_DBG1(0), (0x1 << 1)); + + mmio_write_32(DDRC_PCTRL_0(0), 0x00000001); + /* 27. Write 1 to SBRCTL.scrub_en. Enable SBR if desired, only required if SBR instantiated. */ + + /* set SWCTL.sw_done to enable quasi-dynamic register programming outside reset. */ + mmio_write_32(DDRC_SWCTL(0), 0x0001); + + /* wait SWSTAT.sw_done_ack to 1 */ + do { + tmp = 0x1 & mmio_read_32(DDRC_SWSTAT(0)); + } while (!tmp); +} + +void ddr4_dll_no_change(unsigned int num_rank, unsigned int pstate) +{ + volatile unsigned int tmp; + /* 1. set SWCTL.sw_done to disable quasi-dynamic register programming outside reset. */ + mmio_write_32(DDRC_SWCTL(0), 0x0000); + + /* 2. Write 0 to PCTRL_n.port_en. This blocks AXI port(s) from taking any transaction (blocks traffic on + AXI ports). */ + mmio_write_32(DDRC_PCTRL_0(0), 0x00000000); + + /* 3. Poll PSTAT.rd_port_busy_n=0 and PSTAT.wr_port_busy_n=0. Wait until all AXI ports are idle (the + uMCTL2 core has to be idle). */ + do { + tmp = mmio_read_32(DDRC_PSTAT(0)); + } while (tmp & 0x10001); + + /* 4. Write 0 to SBRCTL.scrub_en. Disable SBR, required only if SBR instantiated. */ + /* 5. Poll SBRSTAT.scrub_busy=0. Indicates that there are no outstanding SBR read commands */ + /* 6. Set DERATEEN.derate_enable = 0, if DERATEEN.derate_eanble = 1 and the read latency (RL) value */ + //needs to change after the frequency change (LPDDR2/3/4 only). + /* 7. Set DBG1.dis_hif=1 so that no new commands will be accepted by the uMCTL2. */ + mmio_setbits_32(DDRC_DBG1(0), (0x1 << 1)); + /* 8. Poll DBGCAM.dbg_wr_q_empty and DBGCAM.dbg_rd_q_empty to ensure that write and read data buffers are empty. */ + do { + tmp = 0x06000000 & mmio_read_32(DDRC_DBGCAM(0)); + } while (tmp != 0x06000000); + /* 9. For DDR4, update MR6 with the new tDLLK value via the Mode Register Write signals */ + + /* 10. Set DFILPCFG0.dfi_lp_en_sr = 0, if DFILPCFG0.dfi_lp_en_sr = 1, and wait until DFISTAT.dfi_lp_ack */ + /* 11. If DFI PHY Master interface is active in uMCTL2 (DFIPHYMSTR.phymstr_en == 1'b1) then disable it */ + /* 12. Wait until STAT.operating_mode[1:0]!=11 indicating that the DWC_ddr_umctl2 controller is not in self-refresh mode. */ + tmp = 0x3 & (mmio_read_32((DDRC_STAT(0)))); + if (tmp == 0x3) { + printf("C: Error DRAM should not in Self Refresh\n"); + } + /* 13. Assert PWRCTL.selfref_sw for the DWC_ddr_umctl2 core to enter the self-refresh mode. */ + mmio_setbits_32(DDRC_PWRCTL(0), (1 << 5)); + /* 14. Wait until STAT.operating_mode[1:0]==11 indicating that the DWC_ddr_umctl2 core is in selfrefresh mode. */ + do { + tmp = 0x3f & (mmio_read_32((DDRC_STAT(0)))); + } while (tmp != 0x23); + + sw_pstate(pstate); + dram_cfg_all_mr(num_rank, pstate); + + /* 23. Enable HIF commands by setting DBG1.dis_hif=0. */ + mmio_clrbits_32(DDRC_DBG1(0), (0x1 << 1)); + /* 24. Reset DERATEEN.derate_enable = 1 if DERATEEN.derate_enable has been set to 0 in step 6. */ + /* 25. If DFI PHY Master interface was active in uMCTL2 (DFIPHYMSTR.phymstr_en == 1'b1) before the */ + /* step 11 then enable it back by programming DFIPHYMSTR.phymstr_en = 1'b1. */ + /* 26. Write 1 to PCTRL_n.port_en. AXI port(s) are no longer blocked from taking transactions (Re-enable traffic on AXI ports). */ + /* Re-enable traffic on the AXI ports */ + mmio_write_32(DDRC_PCTRL_0(0), 0x00000001); + /* 27. Write 1 to SBRCTL.scrub_en. Enable SBR if desired, only required if SBR instantiated. */ + + /* set SWCTL.sw_done to enable quasi-dynamic register programming outside reset. */ + mmio_write_32(DDRC_SWCTL(0), 0x0001); + + /* wait SWSTAT.sw_done_ack to 1 */ + do { + tmp = 0x1 & mmio_read_32(DDRC_SWSTAT(0)); + } while (!tmp); +} + +void get_mr_values(unsigned int pstate) +{ + uint32_t init_val; + + for (int i = 0; i < 3; i++) { + init_val = mmio_read_32(fsp_init_reg[pstate][i]); + mr_value[2*i] = init_val >> 16; + mr_value[2*i + 1] = init_val & 0xFFFF; + } + + mr_value[6] = mmio_read_32(fsp_init_reg[pstate][3]) & 0xFFFF; +} + +void ddr4_swffc(struct dram_info *dram_info, unsigned int pstate) +{ + unsigned int cur_pstate = dram_info->current_fsp; + + get_mr_values(pstate); + + if ((pstate == 0 && cur_pstate !=0) || (pstate != 0 && cur_pstate == 0)) + ddr4_dll_change(dram_info->num_rank, pstate, cur_pstate); + else + ddr4_dll_no_change(dram_info->num_rank, pstate); + + dram_info->current_fsp = pstate; +} diff --git a/plat/imx/common/imx8m/dram.c b/plat/imx/common/imx8m/dram.c index e00ca96d..df34c4b0 100644 --- a/plat/imx/common/imx8m/dram.c +++ b/plat/imx/common/imx8m/dram.c @@ -67,13 +67,17 @@ void dram_phy_init(void) } } + void dram_info_init(unsigned long dram_timing_base) { - uint32_t current_fsp, ddr_type; + uint32_t current_fsp, ddr_type, ddrc_mstr; /* get the dram type */ - ddr_type = mmio_read_32(DDRC_MSTR(0)) & DDR_TYPE_MASK; + ddrc_mstr = mmio_read_32(DDRC_MSTR(0)); + ddr_type = ddrc_mstr & DDR_TYPE_MASK; + dram_info.dram_type = ddr_type; + dram_info.num_rank = (ddrc_mstr >> 24) & DDRC_ACTIVE_RANK_MASK; /* init the boot_fsp & current_fsp */ current_fsp = mmio_read_32(DDRC_DFIMISC(0)); @@ -93,6 +97,10 @@ void dram_info_init(unsigned long dram_timing_base) dcsw_op_all(DCCSW); lpddr4_swffc(dev_fsp, 0x0); dev_fsp = (~dev_fsp) & 0x1; + } else if (ddr_type == DDRC_DDR4 && current_fsp != 0x0) { + /* flush the L1/L2 cache */ + dcsw_op_all(DCCSW); + ddr4_swffc(&dram_info, 0x0); } } @@ -122,10 +130,6 @@ int dram_dvfs_handler(uint32_t smc_fid, unsigned int target_freq = x1; uint32_t online_cores = x2; - /* TODO add ddr4 dvfs support later */ - if (dram_info.dram_type != DDRC_LPDDR4) - return 0; - if (target_freq == 0xf) { /* set the WFE done status */ spin_lock(&dfs_lock); @@ -159,8 +163,12 @@ int dram_dvfs_handler(uint32_t smc_fid, /* flush the L1/L2 cache */ dcsw_op_all(DCCSW); - lpddr4_swffc(dev_fsp, target_freq); - dev_fsp = (~dev_fsp) & 0x1; + if (dram_info.dram_type == DDRC_LPDDR4) { + lpddr4_swffc(dev_fsp, target_freq); + dev_fsp = (~dev_fsp) & 0x1; + } else if (dram_info.dram_type == DDRC_DDR4) { + ddr4_swffc(&dram_info, target_freq); + } wait_ddrc_hwffc_done = false; wfe_done = 0; diff --git a/plat/imx/common/include/dram.h b/plat/imx/common/include/dram.h index fa203da2..f3d72dd8 100644 --- a/plat/imx/common/include/dram.h +++ b/plat/imx/common/include/dram.h @@ -30,6 +30,7 @@ #define DDRC_LPDDR4 BIT(5) #define DDRC_DDR4 BIT(4) #define DDR_TYPE_MASK 0x3f +#define DDRC_ACTIVE_RANK_MASK 0x3 #define DDRPHY_REG(x) (0x3c000000 + 4*x) @@ -59,6 +60,7 @@ struct dram_timing_info { struct dram_info { int dram_type; + unsigned int num_rank; int current_fsp; int boot_fsp; struct dram_timing_info *timing_info; @@ -78,6 +80,8 @@ void ddr4_enter_retention(void); void ddr4_exit_retention(void); /* lpddr4 swffc for dvfs */ void lpddr4_swffc(unsigned int dev_fsp, unsigned int tgt_freq); +/* ddr4 swffw for dvfs */ +void ddr4_swffc(struct dram_info *dram_info, unsigned int target_fsp); /* dram retention */ void dram_enter_retention(void); diff --git a/plat/imx/imx8mm/platform.mk b/plat/imx/imx8mm/platform.mk index ac9db97c..30052bcc 100644 --- a/plat/imx/imx8mm/platform.mk +++ b/plat/imx/imx8mm/platform.mk @@ -14,7 +14,8 @@ PLAT_DRAM_SOURCES := plat/imx/common/imx8m/dram.c \ plat/imx/common/imx8m/lpddr4_retention.c \ plat/imx/common/imx8m/ddr4_retention.c \ plat/imx/common/imx8m/lpddr4_helper.c \ - plat/imx/common/imx8m/lpddr4_dvfs.c + plat/imx/common/imx8m/lpddr4_dvfs.c \ + plat/imx/common/imx8m/ddr4_dvfs.c BL31_SOURCES += plat/imx/common/imx8_helpers.S \ plat/imx/common/mxcuart_console.S \ -- cgit v1.2.3