diff options
Diffstat (limited to 'drivers/dma/imx-sdma.c')
-rw-r--r-- | drivers/dma/imx-sdma.c | 911 |
1 files changed, 705 insertions, 206 deletions
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index cacc725ca545..188684fc7654 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -23,10 +23,12 @@ #include <linux/semaphore.h> #include <linux/spinlock.h> #include <linux/device.h> +#include <linux/genalloc.h> #include <linux/dma-mapping.h> #include <linux/firmware.h> #include <linux/slab.h> #include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include <linux/dmaengine.h> #include <linux/of.h> #include <linux/of_address.h> @@ -73,6 +75,9 @@ #define SDMA_CHNENBL0_IMX35 0x200 #define SDMA_CHNENBL0_IMX31 0x080 #define SDMA_CHNPRI_0 0x100 +#define SDMA_DONE0_CONFIG 0x1000 +#define SDMA_DONE0_CONFIG_DONE_SEL 0x7 +#define SDMA_DONE0_CONFIG_DONE_DIS 0x6 /* * Buffer descriptor status values. @@ -167,6 +172,8 @@ #define SDMA_WATERMARK_LEVEL_SPDIF BIT(10) #define SDMA_WATERMARK_LEVEL_SP BIT(11) #define SDMA_WATERMARK_LEVEL_DP BIT(12) +#define SDMA_WATERMARK_LEVEL_SD BIT(13) +#define SDMA_WATERMARK_LEVEL_DD BIT(14) #define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16) #define SDMA_WATERMARK_LEVEL_LWE BIT(28) #define SDMA_WATERMARK_LEVEL_HWE BIT(29) @@ -174,6 +181,7 @@ #define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) #define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \ @@ -234,11 +242,19 @@ struct sdma_script_start_addrs { s32 sai_2_mcu_addr; s32 uart_2_mcu_addr; s32 uartsh_2_mcu_addr; + s32 i2c_2_mcu_addr; + s32 mcu_2_i2c_addr; /* End of v3 array */ s32 mcu_2_zqspi_addr; /* End of v4 array */ }; +#define SDMA_WATERMARK_LEVEL_FIFOS_OFF 12 +#define SDMA_WATERMARK_LEVEL_FIFO_OFF_OFF 16 +#define SDMA_WATERMARK_LEVEL_SW_DONE BIT(23) +#define SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF 24 +#define SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO_OFF 28 + /* * Mode/Count of data node descriptors - IPCv2 */ @@ -422,8 +438,9 @@ struct sdma_channel { struct sdma_desc *desc; struct sdma_engine *sdma; unsigned int channel; - enum dma_transfer_direction direction; + enum dma_transfer_direction direction; struct dma_slave_config slave_config; + struct sdma_audio_config *audio_config; enum sdma_peripheral_type peripheral_type; unsigned int event_id0; unsigned int event_id1; @@ -441,6 +458,7 @@ struct sdma_channel { struct work_struct terminate_worker; struct list_head terminated; bool is_ram_script; + int prio; }; #define IMX_DMA_SG_LOOP BIT(0) @@ -450,6 +468,15 @@ struct sdma_channel { #define MXC_SDMA_MIN_PRIORITY 1 #define MXC_SDMA_MAX_PRIORITY 7 +/* + * 0x78(SDMA_XTRIG_CONF2+4)~0x100(SDMA_CHNPRI_O) registers are reserved and + * can't be accessed. Skip these register touch in suspend/resume. Also below + * two macros are only used on i.mx6sx. + */ +#define MXC_SDMA_RESERVED_REG (SDMA_CHNPRI_0 - SDMA_XTRIG_CONF2 - 4) +#define MXC_SDMA_SAVED_REG_NUM (((SDMA_CHNENBL0_IMX35 + 4 * 48) - \ + MXC_SDMA_RESERVED_REG) / 4) + #define SDMA_FIRMWARE_MAGIC 0x414d4453 /** @@ -488,15 +515,22 @@ struct sdma_driver_data { * https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf */ bool ecspi_fixed; + bool has_done0; + bool pm_runtime; }; struct sdma_engine { struct device *dev; struct sdma_channel channel[MAX_DMA_CHANNELS]; struct sdma_channel_control *channel_control; + u32 save_regs[MXC_SDMA_SAVED_REG_NUM]; + u32 save_done0_regs[2]; + const char *fw_name; void __iomem *regs; struct sdma_context_data *context; dma_addr_t context_phys; + dma_addr_t ccb_phys; + bool is_on; struct dma_device dma_device; struct clk *clk_ipg; struct clk *clk_ahb; @@ -512,6 +546,10 @@ struct sdma_engine { /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/ bool clk_ratio; bool fw_loaded; + struct gen_pool *iram_pool; + u32 fw_fail; + u8 *fw_data; + unsigned short ram_code_start; }; static int sdma_config_write(struct dma_chan *chan, @@ -608,6 +646,12 @@ static struct sdma_driver_data sdma_imx6q = { .script_addrs = &sdma_script_imx6q, }; +static struct sdma_driver_data sdma_imx6sx = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx6q, +}; + static struct sdma_driver_data sdma_imx6ul = { .chnenbl0 = SDMA_CHNENBL0_IMX35, .num_events = 48, @@ -640,6 +684,16 @@ static struct sdma_driver_data sdma_imx8mq = { .check_ratio = 1, }; +static struct sdma_driver_data sdma_imx8mp = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx7d, + .check_ratio = 1, + .ecspi_fixed = true, + .has_done0 = true, + .pm_runtime = true, +}; + static const struct of_device_id sdma_dt_ids[] = { { .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, }, { .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, }, @@ -648,8 +702,10 @@ static const struct of_device_id sdma_dt_ids[] = { { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, }, { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, }, { .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, }, + { .compatible = "fsl,imx6sx-sdma", .data = &sdma_imx6sx, }, { .compatible = "fsl,imx6ul-sdma", .data = &sdma_imx6ul, }, { .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, }, + { .compatible = "fsl,imx8mp-sdma", .data = &sdma_imx8mp, }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, sdma_dt_ids); @@ -659,6 +715,30 @@ MODULE_DEVICE_TABLE(of, sdma_dt_ids); #define SDMA_H_CONFIG_ACR BIT(4) /* indicates if AHB freq /core freq = 2 or 1 */ #define SDMA_H_CONFIG_CSM (3) /* indicates which context switch mode is selected*/ +static void sdma_pm_clk_enable(struct sdma_engine *sdma, bool direct, bool enable) +{ + if (enable) { + if (sdma->drvdata->pm_runtime) + pm_runtime_get_sync(sdma->dev); + else { + clk_enable(sdma->clk_ipg); + clk_enable(sdma->clk_ahb); + } + } else { + if (sdma->drvdata->pm_runtime) { + if (direct) { + pm_runtime_put_sync_suspend(sdma->dev); + } else { + pm_runtime_mark_last_busy(sdma->dev); + pm_runtime_put_autosuspend(sdma->dev); + } + } else { + clk_disable(sdma->clk_ipg); + clk_disable(sdma->clk_ahb); + } + } +} + static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event) { u32 chnenbl0 = sdma->drvdata->chnenbl0; @@ -717,7 +797,7 @@ static int sdma_run_channel0(struct sdma_engine *sdma) sdma_enable_channel(sdma, 0); ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP, - reg, !(reg & 1), 1, 500); + reg, !(reg & 1), 1, 5000); if (ret) dev_err(sdma->dev, "Timeout waiting for CH0 ready\n"); @@ -731,36 +811,80 @@ static int sdma_run_channel0(struct sdma_engine *sdma) return ret; } -static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, - u32 address) +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 47 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 48 + +static void sdma_add_scripts(struct sdma_engine *sdma, + const struct sdma_script_start_addrs *addr) +{ + s32 *addr_arr = (u32 *)addr; + s32 *saddr_arr = (u32 *)sdma->script_addrs; + int i; + + /* use the default firmware in ROM if missing external firmware */ + if (!sdma->script_number) + sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; + + if (sdma->script_number > sizeof(struct sdma_script_start_addrs) + / sizeof(s32)) { + dev_err(sdma->dev, + "SDMA script number %d not match with firmware.\n", + sdma->script_number); + return; + } + + for (i = 0; i < sdma->script_number; i++) + if (addr_arr[i] > 0) + saddr_arr[i] = addr_arr[i]; +} + +static int sdma_load_script(struct sdma_engine *sdma) { struct sdma_buffer_descriptor *bd0 = sdma->bd0; + const struct sdma_script_start_addrs *addr; + struct sdma_firmware_header *header; + unsigned short *ram_code; void *buf_virt; dma_addr_t buf_phys; int ret; unsigned long flags; - buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL); - if (!buf_virt) { + header = (struct sdma_firmware_header *)sdma->fw_data; + + addr = (void *)header + header->script_addrs_start; + ram_code = (void *)header + header->ram_code_start; + sdma->ram_code_start = header->ram_code_start; + + buf_virt = dma_alloc_coherent(sdma->dev, header->ram_code_size, + &buf_phys, GFP_KERNEL); + if (!buf_virt) return -ENOMEM; - } spin_lock_irqsave(&sdma->channel_0_lock, flags); bd0->mode.command = C0_SETPM; bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; - bd0->mode.count = size / 2; + bd0->mode.count = header->ram_code_size / 2; bd0->buffer_addr = buf_phys; - bd0->ext_buffer_addr = address; + bd0->ext_buffer_addr = addr->ram_code_start_addr; - memcpy(buf_virt, buf, size); + memcpy(buf_virt, ram_code, header->ram_code_size); ret = sdma_run_channel0(sdma); spin_unlock_irqrestore(&sdma->channel_0_lock, flags); - dma_free_coherent(sdma->dev, size, buf_virt, buf_phys); + dma_free_coherent(sdma->dev, header->ram_code_size, buf_virt, buf_phys); + + sdma_add_scripts(sdma, addr); + + sdma->fw_loaded = true; + dev_info_once(sdma->dev, "loaded firmware %d.%d\n", + header->version_major, + header->version_minor); return ret; } @@ -774,6 +898,38 @@ static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event) val = readl_relaxed(sdma->regs + chnenbl); __set_bit(channel, &val); writel_relaxed(val, sdma->regs + chnenbl); + + /* Set SDMA_DONEx_CONFIG is sw_done enabled */ + if (sdmac->audio_config && sdmac->audio_config->sw_done_sel & BIT(31)) { + u32 sw_done_sel = sdmac->audio_config->sw_done_sel & 0xff; + u32 offset = SDMA_DONE0_CONFIG + sw_done_sel / 4; + u32 done_sel = SDMA_DONE0_CONFIG_DONE_SEL + + ((sw_done_sel % 4) << 3); + u32 sw_done_dis = SDMA_DONE0_CONFIG_DONE_DIS + + ((sw_done_sel % 4) << 3); + + val = readl_relaxed(sdma->regs + offset); + __set_bit(done_sel, &val); + __clear_bit(sw_done_dis, &val); + writel_relaxed(val, sdma->regs + offset); + } + +} + +static int sdma_set_channel_priority(struct sdma_channel *sdmac, + unsigned int priority) +{ + struct sdma_engine *sdma = sdmac->sdma; + int channel = sdmac->channel; + + if (priority < MXC_SDMA_MIN_PRIORITY + || priority > MXC_SDMA_MAX_PRIORITY) { + return -EINVAL; + } + + writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel); + + return 0; } static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event) @@ -818,6 +974,7 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) struct sdma_buffer_descriptor *bd; int error = 0; enum dma_status old_status = sdmac->status; + int count = 0; /* * loop mode. Iterate over descriptors, re-setup them and @@ -828,6 +985,17 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) bd = &desc->bd[desc->buf_tail]; + /* + * re-enable HSTART_HE if all bds consumed at the last time, + * that happens in high loading case which sdma_handle_channel_ + * loop can't be handled in time while all bds run out in sdma + * side, then sdma script clear HE and cause channel stop. + */ + if (count == desc->num_bd) { + dev_warn(sdmac->sdma->dev, "All bds consumed,restart now.\n"); + sdma_enable_channel(sdmac->sdma, sdmac->channel); + } + if (bd->mode.status & BD_DONE) break; @@ -840,13 +1008,20 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) /* * We use bd->mode.count to calculate the residue, since contains * the number of bytes present in the current buffer descriptor. + * Note: in IMX_DMATYPE_MULTI_SAI case, bd->mode.count used as + * remaining bytes instead so that one register could be saved. + * so chn_real_count = desc->period_len - bd->mode.count. */ + if (sdmac->peripheral_type == IMX_DMATYPE_MULTI_SAI) + desc->chn_real_count = desc->period_len - bd->mode.count; + else + desc->chn_real_count = bd->mode.count; - desc->chn_real_count = bd->mode.count; bd->mode.status |= BD_DONE; bd->mode.count = desc->period_len; desc->buf_ptail = desc->buf_tail; desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd; + count++; /* * The callback is called from the interrupt context in order @@ -893,6 +1068,7 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) struct sdma_engine *sdma = dev_id; unsigned long stat; + sdma_pm_clk_enable(sdma, false, true); stat = readl_relaxed(sdma->regs + SDMA_H_INTR); writel_relaxed(stat, sdma->regs + SDMA_H_INTR); /* channel 0 is special and not handled here, see run_channel0() */ @@ -907,7 +1083,10 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) desc = sdmac->desc; if (desc) { if (sdmac->flags & IMX_DMA_SG_LOOP) { - sdma_update_channel_loop(sdmac); + if (sdmac->peripheral_type != IMX_DMATYPE_HDMI) + sdma_update_channel_loop(sdmac); + else + vchan_cyclic_callback(&desc->vd); } else { mxc_sdma_handle_channel_normal(sdmac); vchan_cookie_complete(&desc->vd); @@ -919,6 +1098,8 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) __clear_bit(channel, &stat); } + sdma_pm_clk_enable(sdma, false, false); + return IRQ_HANDLED; } @@ -1023,6 +1204,19 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_IPU_MEMORY: emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr; break; + case IMX_DMATYPE_HDMI: + emi_2_per = sdma->script_addrs->hdmi_dma_addr; + sdmac->is_ram_script = true; + break; + case IMX_DMATYPE_MULTI_SAI: + per_2_emi = sdma->script_addrs->sai_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_sai_addr; + sdmac->is_ram_script = true; + break; + case IMX_DMATYPE_I2C: + per_2_emi = sdma->script_addrs->i2c_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_i2c_addr; + sdmac->is_ram_script = true; default: break; } @@ -1031,6 +1225,13 @@ static void sdma_get_pc(struct sdma_channel *sdmac, sdmac->pc_to_device = emi_2_per; sdmac->device_to_device = per_2_per; sdmac->pc_to_pc = emi_2_emi; + + if (sdma->ram_code_start && + ((sdmac->pc_from_device >= sdma->ram_code_start) || + (sdmac->pc_to_device >= sdma->ram_code_start) || + (sdmac->device_to_device >= sdma->ram_code_start || + (sdmac->pc_to_pc >= sdma->ram_code_start)))) + sdmac->is_ram_script = true; } static int sdma_load_context(struct sdma_channel *sdmac) @@ -1070,11 +1271,16 @@ static int sdma_load_context(struct sdma_channel *sdmac) /* Send by context the event mask,base address for peripheral * and watermark level */ - context->gReg[0] = sdmac->event_mask[1]; - context->gReg[1] = sdmac->event_mask[0]; - context->gReg[2] = sdmac->per_addr; - context->gReg[6] = sdmac->shp_addr; - context->gReg[7] = sdmac->watermark_level; + if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { + context->gReg[4] = sdmac->per_addr; + context->gReg[6] = sdmac->shp_addr; + } else { + context->gReg[0] = sdmac->event_mask[1]; + context->gReg[1] = sdmac->event_mask[0]; + context->gReg[2] = sdmac->per_addr; + context->gReg[6] = sdmac->shp_addr; + context->gReg[7] = sdmac->watermark_level; + } bd0->mode.command = C0_SETDM; bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; @@ -1088,6 +1294,31 @@ static int sdma_load_context(struct sdma_channel *sdmac) return ret; } +static int sdma_save_restore_context(struct sdma_engine *sdma, bool save) +{ + struct sdma_context_data *context = sdma->context; + struct sdma_buffer_descriptor *bd0 = sdma->bd0; + unsigned long flags; + int ret; + + spin_lock_irqsave(&sdma->channel_0_lock, flags); + + if (save) + bd0->mode.command = C0_GETDM; + else + bd0->mode.command = C0_SETDM; + + bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; + bd0->mode.count = MAX_DMA_CHANNELS * sizeof(*context) / 4; + bd0->buffer_addr = sdma->context_phys; + bd0->ext_buffer_addr = 2048; + ret = sdma_run_channel0(sdma); + + spin_unlock_irqrestore(&sdma->channel_0_lock, flags); + + return ret; +} + static struct sdma_channel *to_sdma_chan(struct dma_chan *chan) { return container_of(chan, struct sdma_channel, vc.chan); @@ -1124,6 +1355,7 @@ static int sdma_terminate_all(struct dma_chan *chan) struct sdma_channel *sdmac = to_sdma_chan(chan); unsigned long flags; + sdma_pm_clk_enable(sdmac->sdma, false, true); spin_lock_irqsave(&sdmac->vc.lock, flags); sdma_disable_channel(chan); @@ -1143,6 +1375,8 @@ static int sdma_terminate_all(struct dma_chan *chan) spin_unlock_irqrestore(&sdmac->vc.lock, flags); + sdma_pm_clk_enable(sdmac->sdma, false, false); + return 0; } @@ -1193,6 +1427,49 @@ static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP; sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT; + + if (sdmac->audio_config->src_fifo_num > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SD; + if (sdmac->audio_config->dst_fifo_num > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD; +} + +static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac) +{ + u8 fifo_num = sdmac->audio_config->src_fifo_num | + sdmac->audio_config->dst_fifo_num; + u8 fifo_offset = sdmac->audio_config->src_fifo_off | + sdmac->audio_config->dst_fifo_off; + u8 words_per_fifo = sdmac->audio_config->words_per_fifo; + + sdmac->watermark_level &= ~(0xFF << SDMA_WATERMARK_LEVEL_FIFOS_OFF | + SDMA_WATERMARK_LEVEL_SW_DONE | + 0xf << SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF | + 0xf << SDMA_WATERMARK_LEVEL_FIFO_OFF_OFF | + 0xf << SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO_OFF); + + if (sdmac->audio_config->sw_done_sel & BIT(31)) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SW_DONE | + (sdmac->audio_config->sw_done_sel & 0xff) << + SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF; + + /* For fifo_num + * bit 12-15 is the fifo number; + * bit 16-19 is the fifo offset, + * bit 28-31 is the channels per fifo. + * so here only need to shift left fifo_num 12 bit for watermake_level + */ + if (fifo_num) + sdmac->watermark_level |= fifo_num << + SDMA_WATERMARK_LEVEL_FIFOS_OFF; + + if (fifo_offset) + sdmac->watermark_level |= fifo_offset << + SDMA_WATERMARK_LEVEL_FIFO_OFF_OFF; + + if (words_per_fifo) + sdmac->watermark_level |= (words_per_fifo - 1) << + SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO_OFF; } static int sdma_config_channel(struct dma_chan *chan) @@ -1218,7 +1495,11 @@ static int sdma_config_channel(struct dma_chan *chan) break; } - sdma_get_pc(sdmac, sdmac->peripheral_type); + sdma_set_channel_priority(sdmac, sdmac->prio); + + sdma_event_enable(sdmac, sdmac->event_id0); + if (sdmac->event_id1) + sdma_event_enable(sdmac, sdmac->event_id1); if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) && (sdmac->peripheral_type != IMX_DMATYPE_DSP)) { @@ -1227,8 +1508,12 @@ static int sdma_config_channel(struct dma_chan *chan) if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP || sdmac->peripheral_type == IMX_DMATYPE_ASRC) sdma_set_watermarklevel_for_p2p(sdmac); - } else + } else { + if (sdmac->peripheral_type == IMX_DMATYPE_MULTI_SAI) + sdma_set_watermarklevel_for_sais(sdmac); + __set_bit(sdmac->event_id0, sdmac->event_mask); + } /* Address */ sdmac->shp_addr = sdmac->per_address; @@ -1240,28 +1525,18 @@ static int sdma_config_channel(struct dma_chan *chan) return 0; } -static int sdma_set_channel_priority(struct sdma_channel *sdmac, - unsigned int priority) -{ - struct sdma_engine *sdma = sdmac->sdma; - int channel = sdmac->channel; - - if (priority < MXC_SDMA_MIN_PRIORITY - || priority > MXC_SDMA_MAX_PRIORITY) { - return -EINVAL; - } - - writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel); - - return 0; -} - static int sdma_request_channel0(struct sdma_engine *sdma) { int ret = -EBUSY; - sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys, - GFP_NOWAIT); + if (sdma->iram_pool) + sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys); + else + sdma->bd0 = dma_alloc_coherent(sdma->dev, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys, GFP_NOWAIT); if (!sdma->bd0) { ret = -ENOMEM; goto out; @@ -1281,10 +1556,15 @@ out: static int sdma_alloc_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; int ret = 0; - desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size, - &desc->bd_phys, GFP_NOWAIT); + if (sdma->iram_pool) + desc->bd = gen_pool_dma_alloc(sdma->iram_pool, bd_size, + &desc->bd_phys); + else + desc->bd = dma_alloc_coherent(sdma->dev, bd_size, + &desc->bd_phys, GFP_NOWAIT); if (!desc->bd) { ret = -ENOMEM; goto out; @@ -1296,9 +1576,14 @@ out: static void sdma_free_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; - dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, - desc->bd_phys); + if (sdma->iram_pool) + gen_pool_free(sdma->iram_pool, (unsigned long)desc->bd, + bd_size); + else + dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, + desc->bd_phys); } static void sdma_desc_free(struct virt_dma_desc *vd) @@ -1309,12 +1594,122 @@ static void sdma_desc_free(struct virt_dma_desc *vd) kfree(desc); } +static int sdma_runtime_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sdma_engine *sdma = platform_get_drvdata(pdev); + + if (!sdma->is_on) + return 0; + + sdma->fw_loaded = false; + sdma->is_on = false; + + clk_disable(sdma->clk_ipg); + clk_disable(sdma->clk_ahb); + + /* free channel0 bd */ + if (sdma->iram_pool) + gen_pool_free(sdma->iram_pool, (unsigned long)sdma->bd0, + sizeof(struct sdma_buffer_descriptor)); + else + dma_free_coherent(sdma->dev, + sizeof(struct sdma_buffer_descriptor), + sdma->bd0, sdma->bd0_phys); + + return 0; +} + +static int sdma_runtime_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sdma_engine *sdma = platform_get_drvdata(pdev); + int i, ret = 0; + + ret = clk_enable(sdma->clk_ipg); + if (ret) + return ret; + ret = clk_enable(sdma->clk_ahb); + if (ret) + goto disable_clk_ipg; + + /* Do nothing at HW level if audiomix which shared with audio driver + * not off indeed. + */ + if (readl_relaxed(sdma->regs + SDMA_H_C0PTR)) { + if (sdma->iram_pool) + sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys); + else + sdma->bd0 = dma_alloc_coherent(sdma->dev, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys, GFP_NOWAIT); + if (!sdma->bd0) + ret = -ENOMEM; + + sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys; + sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys; + + sdma->is_on = true; + sdma->fw_loaded = true; + + return ret; + } + + /* Be sure SDMA has not started yet */ + writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); + + /* disable all channels */ + for (i = 0; i < sdma->drvdata->num_events; i++) + writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i)); + + /* All channels have priority 0 */ + for (i = 0; i < MAX_DMA_CHANNELS; i++) + writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4); + + ret = sdma_request_channel0(sdma); + if (ret) + return ret; + + sdma_config_ownership(&sdma->channel[0], false, true, false); + + /* Set Command Channel (Channel Zero) */ + writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR); + + /* Set bits of CONFIG register but with static context switching */ + if (sdma->clk_ratio) + writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG); + else + writel_relaxed(0, sdma->regs + SDMA_H_CONFIG); + + writel_relaxed(sdma->ccb_phys, sdma->regs + SDMA_H_C0PTR); + + /* Initializes channel's priorities */ + sdma_set_channel_priority(&sdma->channel[0], 7); + + if (!sdma->fw_data) + dev_dbg(sdma->dev, "firmware not ready.\n"); + else if (sdma_load_script(sdma)) + dev_warn(sdma->dev, "failed to load script.\n"); + + sdma->is_on = true; + + return 0; + +disable_clk_ipg: + clk_disable(sdma->clk_ipg); + dev_err(sdma->dev, "initialisation failed with %d\n", ret); + + return ret; +} + static int sdma_alloc_chan_resources(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); struct imx_dma_data *data = chan->private; struct imx_dma_data mem_data; - int prio, ret; + int prio; /* * MEMCPY may never setup chan->private by filter function such as @@ -1352,32 +1747,30 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) sdmac->peripheral_type = data->peripheral_type; sdmac->event_id0 = data->dma_request; sdmac->event_id1 = data->dma_request2; - - ret = clk_enable(sdmac->sdma->clk_ipg); - if (ret) - return ret; - ret = clk_enable(sdmac->sdma->clk_ahb); - if (ret) - goto disable_clk_ipg; - - ret = sdma_set_channel_priority(sdmac, prio); - if (ret) - goto disable_clk_ahb; + sdmac->prio = prio; return 0; - -disable_clk_ahb: - clk_disable(sdmac->sdma->clk_ahb); -disable_clk_ipg: - clk_disable(sdmac->sdma->clk_ipg); - return ret; } static void sdma_free_chan_resources(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); - struct sdma_engine *sdma = sdmac->sdma; + /* + * Per fw_data is null which means firmware not loaded and sdma + * not initialized, directly return. This happens in below case: + * + * -- driver request dma chan in probe phase, after that driver + * fall into -EPROBE_DEFER and free channel again without + * anything else about dma, so just return directly, otherwise + * kernel could hang since dma hardware not ready if drvdata-> + * pm_runtime is false. + * + */ + if (unlikely(!sdmac->sdma->fw_data)) + return; + + sdma_pm_clk_enable(sdmac->sdma, false, true); sdma_terminate_all(chan); sdma_channel_synchronize(chan); @@ -1391,8 +1784,9 @@ static void sdma_free_chan_resources(struct dma_chan *chan) sdma_set_channel_priority(sdmac, 0); - clk_disable(sdma->clk_ipg); - clk_disable(sdma->clk_ahb); + kfree(sdmac->audio_config); + sdmac->audio_config = NULL; + sdma_pm_clk_enable(sdmac->sdma, false, false); } static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, @@ -1400,11 +1794,6 @@ static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, { struct sdma_desc *desc; - if (!sdmac->sdma->fw_loaded && sdmac->is_ram_script) { - dev_warn_once(sdmac->sdma->dev, "sdma firmware not ready!\n"); - goto err_out; - } - desc = kzalloc((sizeof(*desc)), GFP_NOWAIT); if (!desc) goto err_out; @@ -1420,12 +1809,14 @@ static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, desc->sdmac = sdmac; desc->num_bd = bds; - if (sdma_alloc_bd(desc)) + if (bds && sdma_alloc_bd(desc)) goto err_desc_out; /* No slave_config called in MEMCPY case, so do here */ - if (direction == DMA_MEM_TO_MEM) + if (direction == DMA_MEM_TO_MEM) { sdma_config_ownership(sdmac, false, true, false); + sdma_set_channel_priority(sdmac, sdmac->prio); + } if (sdma_load_context(sdmac)) goto err_desc_out; @@ -1456,10 +1847,13 @@ static struct dma_async_tx_descriptor *sdma_prep_memcpy( dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n", &dma_src, &dma_dst, len, channel); + sdma_pm_clk_enable(sdmac->sdma, false, true); desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM, len / SDMA_BD_MAX_CNT + 1); - if (!desc) + if (!desc) { + sdma_pm_clk_enable(sdmac->sdma, true, false); return NULL; + } do { count = min_t(size_t, len, SDMA_BD_MAX_CNT); @@ -1491,6 +1885,8 @@ static struct dma_async_tx_descriptor *sdma_prep_memcpy( bd->mode.status = param; } while (len); + sdma_pm_clk_enable(sdmac->sdma, false, false); + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); } @@ -1505,8 +1901,12 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( int channel = sdmac->channel; struct scatterlist *sg; struct sdma_desc *desc; + int ret; - sdma_config_write(chan, &sdmac->slave_config, direction); + sdma_pm_clk_enable(sdmac->sdma, false, true); + ret = sdma_config_write(chan, &sdmac->slave_config, direction); + if (ret) + goto err_out; desc = sdma_transfer_init(sdmac, direction, sg_len); if (!desc) @@ -1541,6 +1941,9 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( if (count & 3 || sg->dma_address & 3) goto err_bd_out; break; + case DMA_SLAVE_BUSWIDTH_3_BYTES: + bd->mode.command = 3; + break; case DMA_SLAVE_BUSWIDTH_2_BYTES: bd->mode.command = 2; if (count & 1 || sg->dma_address & 1) @@ -1569,12 +1972,16 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( bd->mode.status = param; } + sdma_pm_clk_enable(sdmac->sdma, false, false); + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); err_bd_out: sdma_free_bd(desc); kfree(desc); err_out: sdmac->status = DMA_ERROR; + sdma_pm_clk_enable(sdmac->sdma, true, false); + return NULL; } @@ -1585,14 +1992,22 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( { struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; - int num_periods = buf_len / period_len; + int num_periods = 0; int channel = sdmac->channel; int i = 0, buf = 0; struct sdma_desc *desc; + int ret; dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel); - sdma_config_write(chan, &sdmac->slave_config, direction); + sdma_pm_clk_enable(sdmac->sdma, false, true); + + if (sdmac->peripheral_type != IMX_DMATYPE_HDMI) + num_periods = buf_len / period_len; + + ret = sdma_config_write(chan, &sdmac->slave_config, direction); + if (ret) + goto err_out; desc = sdma_transfer_init(sdmac, direction, num_periods); if (!desc) @@ -1608,6 +2023,9 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( goto err_bd_out; } + if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); + while (buf < buf_len) { struct sdma_buffer_descriptor *bd = &desc->bd[i]; int param; @@ -1640,12 +2058,16 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( i++; } + sdma_pm_clk_enable(sdmac->sdma, false, false); + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); err_bd_out: sdma_free_bd(desc); kfree(desc); err_out: sdmac->status = DMA_ERROR; + sdma_pm_clk_enable(sdmac->sdma, true, false); + return NULL; } @@ -1655,6 +2077,14 @@ static int sdma_config_write(struct dma_chan *chan, { struct sdma_channel *sdmac = to_sdma_chan(chan); + sdmac->watermark_level = 0; + sdma_get_pc(sdmac, sdmac->peripheral_type); + + if (!sdmac->sdma->fw_loaded && sdmac->is_ram_script) { + dev_warn_once(sdmac->sdma->dev, "sdma firmware not ready!\n"); + return -EPERM; + } + if (direction == DMA_DEV_TO_MEM) { sdmac->per_address = dmaengine_cfg->src_addr; sdmac->watermark_level = dmaengine_cfg->src_maxburst * @@ -1668,6 +2098,10 @@ static int sdma_config_write(struct dma_chan *chan, sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) & SDMA_WATERMARK_LEVEL_HWML; sdmac->word_size = dmaengine_cfg->dst_addr_width; + } else if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { + sdmac->per_address = dmaengine_cfg->dst_addr; + sdmac->per_address2 = dmaengine_cfg->src_addr; + sdmac->watermark_level = 0; } else { sdmac->per_address = dmaengine_cfg->dst_addr; sdmac->watermark_level = dmaengine_cfg->dst_maxburst * @@ -1682,19 +2116,31 @@ static int sdma_config(struct dma_chan *chan, struct dma_slave_config *dmaengine_cfg) { struct sdma_channel *sdmac = to_sdma_chan(chan); + void *tmp; memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg)); + /* Allocate special sdma_audio_config if it's used */ + if (dmaengine_cfg->peripheral_config) { + tmp = krealloc(sdmac->audio_config, + dmaengine_cfg->peripheral_size, GFP_NOWAIT); + if (!tmp) + return -ENOMEM; + + sdmac->audio_config = (struct sdma_audio_config *)tmp; + + memcpy(tmp, dmaengine_cfg->peripheral_config, + dmaengine_cfg->peripheral_size); + } + /* Set ENBLn earlier to make sure dma request triggered after that */ if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) return -EINVAL; - sdma_event_enable(sdmac, sdmac->event_id0); - if (sdmac->event_id1) { - if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) - return -EINVAL; - sdma_event_enable(sdmac, sdmac->event_id1); - } + + if (sdmac->event_id1 && + sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) + return -EINVAL; return 0; } @@ -1745,68 +2191,38 @@ static void sdma_issue_pending(struct dma_chan *chan) struct sdma_channel *sdmac = to_sdma_chan(chan); unsigned long flags; + sdma_pm_clk_enable(sdmac->sdma, false, true); spin_lock_irqsave(&sdmac->vc.lock, flags); if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc) sdma_start_desc(sdmac); spin_unlock_irqrestore(&sdmac->vc.lock, flags); -} - -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 45 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 46 - -static void sdma_add_scripts(struct sdma_engine *sdma, - const struct sdma_script_start_addrs *addr) -{ - s32 *addr_arr = (u32 *)addr; - s32 *saddr_arr = (u32 *)sdma->script_addrs; - int i; - - /* use the default firmware in ROM if missing external firmware */ - if (!sdma->script_number) - sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; - - if (sdma->script_number > sizeof(struct sdma_script_start_addrs) - / sizeof(s32)) { - dev_err(sdma->dev, - "SDMA script number %d not match with firmware.\n", - sdma->script_number); - return; - } - - for (i = 0; i < sdma->script_number; i++) - if (addr_arr[i] > 0) - saddr_arr[i] = addr_arr[i]; - - /* - * get uart_2_mcu_addr/uartsh_2_mcu_addr rom script specially because - * they are now replaced by uart_2_mcu_ram_addr/uartsh_2_mcu_ram_addr - * to be compatible with legacy freescale/nxp sdma firmware, and they - * are located in the bottom part of sdma_script_start_addrs which are - * beyond the SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1. - */ - if (addr->uart_2_mcu_addr) - sdma->script_addrs->uart_2_mcu_addr = addr->uart_2_mcu_addr; - if (addr->uartsh_2_mcu_addr) - sdma->script_addrs->uartsh_2_mcu_addr = addr->uartsh_2_mcu_addr; + sdma_pm_clk_enable(sdmac->sdma, false, false); } static void sdma_load_firmware(const struct firmware *fw, void *context) { struct sdma_engine *sdma = context; const struct sdma_firmware_header *header; - const struct sdma_script_start_addrs *addr; - unsigned short *ram_code; if (!fw) { - dev_info(sdma->dev, "external firmware not found, using ROM firmware\n"); - /* In this case we just use the ROM firmware. */ + /* Load firmware once more time if timeout */ + if (sdma->fw_fail) + dev_info(sdma->dev, "external firmware not found, using ROM firmware\n"); + else { + /*add a bit delay to wait for firmware priv released */ + msleep(20); + request_firmware_nowait(THIS_MODULE, + FW_ACTION_UEVENT, sdma->fw_name, + sdma->dev, GFP_KERNEL, sdma, + sdma_load_firmware); + sdma->fw_fail++; + } + return; } - if (fw->size < sizeof(*header)) + if (fw->size < sizeof(*header) || sdma->fw_loaded) goto err_firmware; header = (struct sdma_firmware_header *)fw->data; @@ -1833,25 +2249,18 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) goto err_firmware; } - addr = (void *)header + header->script_addrs_start; - ram_code = (void *)header + header->ram_code_start; + dev_info(sdma->dev, "firmware found.\n"); - clk_enable(sdma->clk_ipg); - clk_enable(sdma->clk_ahb); - /* download the RAM image for SDMA */ - sdma_load_script(sdma, ram_code, - header->ram_code_size, - addr->ram_code_start_addr); - clk_disable(sdma->clk_ipg); - clk_disable(sdma->clk_ahb); - - sdma_add_scripts(sdma, addr); + if (!sdma->fw_data) { + sdma->fw_data = kmalloc(fw->size, GFP_KERNEL); + if (!sdma->fw_data) + goto err_firmware; - sdma->fw_loaded = true; + memcpy(sdma->fw_data, fw->data, fw->size); - dev_info(sdma->dev, "loaded firmware %d.%d\n", - header->version_major, - header->version_minor); + if (!sdma->drvdata->pm_runtime) + sdma_runtime_resume(sdma->dev); + } err_firmware: release_firmware(fw); @@ -1935,79 +2344,34 @@ static int sdma_get_firmware(struct sdma_engine *sdma, return ret; } -static int sdma_init(struct sdma_engine *sdma) +static int sdma_init_sw(struct sdma_engine *sdma) { - int i, ret; - dma_addr_t ccb_phys; - - ret = clk_enable(sdma->clk_ipg); - if (ret) - return ret; - ret = clk_enable(sdma->clk_ahb); - if (ret) - goto disable_clk_ipg; + int ret, ccbsize; if (sdma->drvdata->check_ratio && (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg))) sdma->clk_ratio = 1; - /* Be sure SDMA has not started yet */ - writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); - - sdma->channel_control = dma_alloc_coherent(sdma->dev, - MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) + - sizeof(struct sdma_context_data), - &ccb_phys, GFP_KERNEL); + ccbsize = MAX_DMA_CHANNELS * (sizeof(struct sdma_channel_control) + + sizeof(struct sdma_context_data)); + if (sdma->iram_pool) + sdma->channel_control = gen_pool_dma_alloc(sdma->iram_pool, + ccbsize, &sdma->ccb_phys); + else + sdma->channel_control = dma_alloc_coherent(sdma->dev, ccbsize, + &sdma->ccb_phys, GFP_KERNEL); if (!sdma->channel_control) { ret = -ENOMEM; - goto err_dma_alloc; + return ret; } sdma->context = (void *)sdma->channel_control + MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control); - sdma->context_phys = ccb_phys + + sdma->context_phys = sdma->ccb_phys + MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control); - /* disable all channels */ - for (i = 0; i < sdma->drvdata->num_events; i++) - writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i)); - - /* All channels have priority 0 */ - for (i = 0; i < MAX_DMA_CHANNELS; i++) - writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4); - - ret = sdma_request_channel0(sdma); - if (ret) - goto err_dma_alloc; - - sdma_config_ownership(&sdma->channel[0], false, true, false); - - /* Set Command Channel (Channel Zero) */ - writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR); - - /* Set bits of CONFIG register but with static context switching */ - if (sdma->clk_ratio) - writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG); - else - writel_relaxed(0, sdma->regs + SDMA_H_CONFIG); - - writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR); - - /* Initializes channel's priorities */ - sdma_set_channel_priority(&sdma->channel[0], 7); - - clk_disable(sdma->clk_ipg); - clk_disable(sdma->clk_ahb); - return 0; - -err_dma_alloc: - clk_disable(sdma->clk_ahb); -disable_clk_ipg: - clk_disable(sdma->clk_ipg); - dev_err(sdma->dev, "initialisation failed with %d\n", ret); - return ret; } static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param) @@ -2034,9 +2398,11 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, if (dma_spec->args_count != 3) return NULL; + memset(&data, 0, sizeof(data)); + data.dma_request = dma_spec->args[0]; data.peripheral_type = dma_spec->args[1]; - data.priority = dma_spec->args[2]; + data.priority = dma_spec->args[2] & 0xff; /* * init dma_request2 to zero, which is not used by the dts. * For P2P, dma_request2 is init from dma_request_channel(), @@ -2144,7 +2510,13 @@ static int sdma_probe(struct platform_device *pdev) vchan_init(&sdmac->vc, &sdma->dma_device); } - ret = sdma_init(sdma); + if (np) { + sdma->iram_pool = of_gen_pool_get(np, "iram", 0); + if (sdma->iram_pool) + dev_info(&pdev->dev, "alloc bd from iram.\n"); + } + + ret = sdma_init_sw(sdma); if (ret) goto err_init; @@ -2205,12 +2577,22 @@ static int sdma_probe(struct platform_device *pdev) */ ret = of_property_read_string(np, "fsl,sdma-ram-script-name", &fw_name); - if (ret) { + if (ret) dev_warn(&pdev->dev, "failed to get firmware name\n"); - } else { - ret = sdma_get_firmware(sdma, fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); + else + sdma->fw_name = fw_name; + + ret = sdma_get_firmware(sdma, sdma->fw_name); + if (ret) + dev_warn(sdma->dev, "failed to get firmware.\n"); + + /* enable autosuspend for pm_runtime */ + if (sdma->drvdata->pm_runtime) { + pm_runtime_set_autosuspend_delay(&pdev->dev, 8000); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_mark_last_busy(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_enable(&pdev->dev); } return 0; @@ -2234,6 +2616,7 @@ static int sdma_remove(struct platform_device *pdev) devm_free_irq(&pdev->dev, sdma->irq, sdma); dma_async_device_unregister(&sdma->dma_device); kfree(sdma->script_addrs); + kfree(sdma->fw_data); clk_unprepare(sdma->clk_ahb); clk_unprepare(sdma->clk_ipg); /* Kill the tasklet */ @@ -2244,14 +2627,130 @@ static int sdma_remove(struct platform_device *pdev) sdma_free_chan_resources(&sdmac->vc.chan); } + if (sdma->drvdata->pm_runtime) { + pm_runtime_disable(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); + } else { + sdma_runtime_suspend(&pdev->dev); + } + platform_set_drvdata(pdev, NULL); return 0; } +#ifdef CONFIG_PM_SLEEP +static int sdma_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sdma_engine *sdma = platform_get_drvdata(pdev); + int i, ret = 0; + + /* Do nothing if not i.MX6SX/6UL or i.MX7D, i.MX8MP */ + if (sdma->drvdata != &sdma_imx6sx && sdma->drvdata != &sdma_imx7d + && sdma->drvdata != &sdma_imx6ul && sdma->drvdata != &sdma_imx8mp) + return 0; + + if (!sdma->is_on) + return 0; + + ret = sdma_save_restore_context(sdma, true); + if (ret) { + dev_err(sdma->dev, "save context error!\n"); + return ret; + } + + /* save regs */ + for (i = 0; i < MXC_SDMA_SAVED_REG_NUM; i++) { + /* + * 0x78(SDMA_XTRIG_CONF2+4)~0x100(SDMA_CHNPRI_O) registers are + * reserved and can't be touched. Skip these regs. + */ + if (i > SDMA_XTRIG_CONF2 / 4) + sdma->save_regs[i] = readl_relaxed(sdma->regs + + MXC_SDMA_RESERVED_REG + + 4 * i); + else + sdma->save_regs[i] = readl_relaxed(sdma->regs + 4 * i); + } + + if (sdma->drvdata->has_done0) { + for (i = 0; i < 2; i++) + sdma->save_done0_regs[i] = + readl_relaxed(sdma->regs + SDMA_DONE0_CONFIG + 4 * i); + } + + return 0; +} + +static int sdma_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sdma_engine *sdma = platform_get_drvdata(pdev); + int i, ret; + + /* Do nothing if not i.MX6SX/6UL or i.MX7D, i.MX8MP */ + if (sdma->drvdata != &sdma_imx6sx && sdma->drvdata != &sdma_imx7d + && sdma->drvdata != &sdma_imx6ul && sdma->drvdata != &sdma_imx8mp) + return 0; + + if (!sdma->is_on) + return 0; + + /* Do nothing if mega/fast mix not turned off */ + if (readl_relaxed(sdma->regs + SDMA_H_C0PTR)) + return 0; + + /* Firmware was lost, mark as "not ready" */ + sdma->fw_loaded = false; + + /* restore regs and load firmware */ + for (i = 0; i < MXC_SDMA_SAVED_REG_NUM; i++) { + /* + * 0x78(SDMA_XTRIG_CONF2+4)~0x100(SDMA_CHNPRI_O) registers are + * reserved and can't be touched. Skip these regs. + */ + if (i > SDMA_XTRIG_CONF2 / 4) + writel_relaxed(sdma->save_regs[i], sdma->regs + + MXC_SDMA_RESERVED_REG + 4 * i); + /* set static context switch mode before channel0 running */ + else if (i == SDMA_H_CONFIG / 4) + writel_relaxed(sdma->save_regs[i] & ~SDMA_H_CONFIG_CSM, + sdma->regs + SDMA_H_CONFIG); + else + writel_relaxed(sdma->save_regs[i], sdma->regs + 4 * i); + } + + /* restore SDMA_DONEx_CONFIG */ + if (sdma->drvdata->has_done0) { + for (i = 0; i < 2; i++) + writel_relaxed(sdma->save_done0_regs[i], + sdma->regs + SDMA_DONE0_CONFIG + 4 * i); + } + + /* prepare priority for channel0 to start */ + sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_MAX_PRIORITY); + + if (sdma_load_script(sdma)) + dev_warn(sdma->dev, "failed to load firmware.\n"); + + ret = sdma_save_restore_context(sdma, false); + if (ret) + dev_err(sdma->dev, "restore context error!\n"); + + return ret; +} +#endif + +static const struct dev_pm_ops sdma_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(sdma_suspend, sdma_resume) + SET_RUNTIME_PM_OPS(sdma_runtime_suspend, sdma_runtime_resume, NULL) +}; + static struct platform_driver sdma_driver = { .driver = { .name = "imx-sdma", .of_match_table = sdma_dt_ids, + .pm = &sdma_pm_ops, }, .remove = sdma_remove, .probe = sdma_probe, |