// SPDX-License-Identifier: GPL-2.0 /* * IPU remoteproc driver for various SoCs * * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ * Angela Stegmaier * Venkateswara Rao Mandela * Keerthy */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define IPU1_LOAD_ADDR (0xa17ff000) #define MAX_REMOTECORE_BIN_SIZE (8 * 0x100000) enum ipu_num { IPU1 = 0, IPU2, RPROC_END_ENUMS, }; #define IPU2_LOAD_ADDR (IPU1_LOAD_ADDR + MAX_REMOTECORE_BIN_SIZE) #define PAGE_SHIFT 12 #define PAGESIZE_1M 0x0 #define PAGESIZE_64K 0x1 #define PAGESIZE_4K 0x2 #define PAGESIZE_16M 0x3 #define LE 0 #define BE 1 #define ELEMSIZE_8 0x0 #define ELEMSIZE_16 0x1 #define ELEMSIZE_32 0x2 #define MIXED_TLB 0x0 #define MIXED_CPU 0x1 #define PGT_SMALLPAGE_SIZE 0x00001000 #define PGT_LARGEPAGE_SIZE 0x00010000 #define PGT_SECTION_SIZE 0x00100000 #define PGT_SUPERSECTION_SIZE 0x01000000 #define PGT_L1_DESC_PAGE 0x00001 #define PGT_L1_DESC_SECTION 0x00002 #define PGT_L1_DESC_SUPERSECTION 0x40002 #define PGT_L1_DESC_PAGE_MASK 0xfffffC00 #define PGT_L1_DESC_SECTION_MASK 0xfff00000 #define PGT_L1_DESC_SUPERSECTION_MASK 0xff000000 #define PGT_L1_DESC_SMALLPAGE_INDEX_SHIFT 12 #define PGT_L1_DESC_LARGEPAGE_INDEX_SHIFT 16 #define PGT_L1_DESC_SECTION_INDEX_SHIFT 20 #define PGT_L1_DESC_SUPERSECTION_INDEX_SHIFT 24 #define PGT_L2_DESC_SMALLPAGE 0x02 #define PGT_L2_DESC_LARGEPAGE 0x01 #define PGT_L2_DESC_SMALLPAGE_MASK 0xfffff000 #define PGT_L2_DESC_LARGEPAGE_MASK 0xffff0000 /* * The memory for the page tables (256 KB per IPU) is placed just before * the carveout memories for the remote processors. 16 KB of memory is * needed for the L1 page table (4096 entries * 4 bytes per 1 MB section). * Any smaller page (64 KB or 4 KB) entries are supported through L2 page * tables (1 KB per table). The remaining 240 KB can provide support for * 240 L2 page tables. Any remoteproc firmware image requiring more than * 240 L2 page table entries would need more memory to be reserved. */ #define PAGE_TABLE_SIZE_L1 (0x00004000) #define PAGE_TABLE_SIZE_L2 (0x400) #define MAX_NUM_L2_PAGE_TABLES (240) #define PAGE_TABLE_SIZE_L2_TOTAL (MAX_NUM_L2_PAGE_TABLES * PAGE_TABLE_SIZE_L2) #define PAGE_TABLE_SIZE (PAGE_TABLE_SIZE_L1 + (PAGE_TABLE_SIZE_L2_TOTAL)) /** * struct omap_rproc_mem - internal memory structure * @cpu_addr: MPU virtual address of the memory region * @bus_addr: bus address used to access the memory region * @dev_addr: device address of the memory region from DSP view * @size: size of the memory region */ struct omap_rproc_mem { void __iomem *cpu_addr; phys_addr_t bus_addr; u32 dev_addr; size_t size; }; struct ipu_privdata { struct omap_rproc_mem mem; struct list_head mappings; const char *fw_name; u32 bootaddr; int id; struct udevice *rdev; }; typedef int (*handle_resource_t) (void *, int offset, int avail); unsigned int *page_table_l1 = (unsigned int *)0x0; unsigned int *page_table_l2 = (unsigned int *)0x0; /* * Set maximum carveout size to 96 MB */ #define DRA7_RPROC_MAX_CO_SIZE (96 * 0x100000) /* * These global variables are used for deriving the MMU page tables. They * are initialized for each core with the appropriate values. The length * of the array mem_bitmap is set as per a 96 MB carveout which the * maximum set aside in the current memory map. */ unsigned long mem_base; unsigned long mem_size; unsigned long mem_bitmap[BITS_TO_LONGS(DRA7_RPROC_MAX_CO_SIZE >> PAGE_SHIFT)]; unsigned long mem_count; unsigned int pgtable_l2_map[MAX_NUM_L2_PAGE_TABLES]; unsigned int pgtable_l2_cnt; void *ipu_alloc_mem(struct udevice *dev, unsigned long len, unsigned long align) { unsigned long mask; unsigned long pageno; int count; count = ((len + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1)) >> PAGE_SHIFT; mask = (1 << align) - 1; pageno = bitmap_find_next_zero_area(mem_bitmap, mem_count, 0, count, mask); debug("%s: count %d mask %#lx pageno %#lx\n", __func__, count, mask, pageno); if (pageno >= mem_count) { debug("%s: %s Error allocating memory; " "Please check carveout size\n", __FILE__, __func__); return NULL; } bitmap_set(mem_bitmap, pageno, count); return (void *)(mem_base + (pageno << PAGE_SHIFT)); } int find_pagesz(unsigned int virt, unsigned int phys, unsigned int len) { int pg_sz_ind = -1; unsigned int min_align = __ffs(virt); if (min_align > __ffs(phys)) min_align = __ffs(phys); if (min_align >= PGT_L1_DESC_SUPERSECTION_INDEX_SHIFT && len >= 0x1000000) { pg_sz_ind = PAGESIZE_16M; goto ret_block; } if (min_align >= PGT_L1_DESC_SECTION_INDEX_SHIFT && len >= 0x100000) { pg_sz_ind = PAGESIZE_1M; goto ret_block; } if (min_align >= PGT_L1_DESC_LARGEPAGE_INDEX_SHIFT && len >= 0x10000) { pg_sz_ind = PAGESIZE_64K; goto ret_block; } if (min_align >= PGT_L1_DESC_SMALLPAGE_INDEX_SHIFT && len >= 0x1000) { pg_sz_ind = PAGESIZE_4K; goto ret_block; } ret_block: return pg_sz_ind; } int get_l2_pg_tbl_addr(unsigned int virt, unsigned int *pg_tbl_addr) { int ret = -1; int i = 0; int match_found = 0; unsigned int tag = (virt & PGT_L1_DESC_SECTION_MASK); *pg_tbl_addr = 0; for (i = 0; (i < pgtable_l2_cnt) && (match_found == 0); i++) { if (tag == pgtable_l2_map[i]) { *pg_tbl_addr = ((unsigned int)page_table_l2) + (i * PAGE_TABLE_SIZE_L2); match_found = 1; ret = 0; } } if (match_found == 0 && i < MAX_NUM_L2_PAGE_TABLES) { pgtable_l2_map[i] = tag; pgtable_l2_cnt++; *pg_tbl_addr = ((unsigned int)page_table_l2) + (i * PAGE_TABLE_SIZE_L2); ret = 0; } return ret; } int config_l2_pagetable(unsigned int virt, unsigned int phys, unsigned int pg_sz, unsigned int pg_tbl_addr) { int ret = -1; unsigned int desc = 0; int i = 0; unsigned int *pg_tbl = (unsigned int *)pg_tbl_addr; /* * Pick bit 19:12 of the virtual address as index */ unsigned int index = (virt & (~PGT_L1_DESC_SECTION_MASK)) >> PAGE_SHIFT; switch (pg_sz) { case PAGESIZE_64K: desc = (phys & PGT_L2_DESC_LARGEPAGE_MASK) | PGT_L2_DESC_LARGEPAGE; for (i = 0; i < 16; i++) pg_tbl[index + i] = desc; ret = 0; break; case PAGESIZE_4K: desc = (phys & PGT_L2_DESC_SMALLPAGE_MASK) | PGT_L2_DESC_SMALLPAGE; pg_tbl[index] = desc; ret = 0; break; default: break; } return ret; } unsigned int ipu_config_pagetable(struct udevice *dev, unsigned int virt, unsigned int phys, unsigned int len) { unsigned int index; unsigned int l = len; unsigned int desc; int pg_sz = 0; int i = 0, err = 0; unsigned int pg_tbl_l2_addr = 0; unsigned int tmp_pgsz; if ((len & 0x0FFF) != 0) return 0; while (l > 0) { pg_sz = find_pagesz(virt, phys, l); index = virt >> PGT_L1_DESC_SECTION_INDEX_SHIFT; switch (pg_sz) { /* * 16 MB super section */ case PAGESIZE_16M: /* * Program the next 16 descriptors */ desc = (phys & PGT_L1_DESC_SUPERSECTION_MASK) | PGT_L1_DESC_SUPERSECTION; for (i = 0; i < 16; i++) page_table_l1[index + i] = desc; l -= PGT_SUPERSECTION_SIZE; phys += PGT_SUPERSECTION_SIZE; virt += PGT_SUPERSECTION_SIZE; break; /* * 1 MB section */ case PAGESIZE_1M: desc = (phys & PGT_L1_DESC_SECTION_MASK) | PGT_L1_DESC_SECTION; page_table_l1[index] = desc; l -= PGT_SECTION_SIZE; phys += PGT_SECTION_SIZE; virt += PGT_SECTION_SIZE; break; /* * 64 KB large page */ case PAGESIZE_64K: case PAGESIZE_4K: if (pg_sz == PAGESIZE_64K) tmp_pgsz = 0x10000; else tmp_pgsz = 0x1000; err = get_l2_pg_tbl_addr(virt, &pg_tbl_l2_addr); if (err != 0) { debug ("Unable to get level 2 PT address\n"); hang(); } err = config_l2_pagetable(virt, phys, pg_sz, pg_tbl_l2_addr); desc = (pg_tbl_l2_addr & PGT_L1_DESC_PAGE_MASK) | PGT_L1_DESC_PAGE; page_table_l1[index] = desc; l -= tmp_pgsz; phys += tmp_pgsz; virt += tmp_pgsz; break; case -1: default: return 0; } } return len; } int da_to_pa(struct udevice *dev, int da) { struct rproc_mem_entry *maps = NULL; struct ipu_privdata *priv = dev_get_priv(dev); list_for_each_entry(maps, &priv->mappings, node) { if (da >= maps->da && da < (maps->da + maps->len)) return maps->dma + (da - maps->da); } return 0; } u32 ipu_config_mmu(u32 core_id, struct rproc *cfg) { u32 i = 0; u32 reg = 0; /* * Clear the entire pagetable location before programming the * address into the MMU */ memset((void *)cfg->page_table_addr, 0x00, PAGE_TABLE_SIZE); for (i = 0; i < cfg->num_iommus; i++) { u32 mmu_base = cfg->mmu_base_addr[i]; __raw_writel((int)cfg->page_table_addr, mmu_base + 0x4c); reg = __raw_readl(mmu_base + 0x88); /* * enable bus-error back */ __raw_writel(reg | 0x1, mmu_base + 0x88); /* * Enable the MMU IRQs during MMU programming for the * late attachcase. This is to allow the MMU fault to be * detected by the kernel. * * MULTIHITFAULT|EMMUMISS|TRANSLATIONFAULT|TABLEWALKFAULT */ __raw_writel(0x1E, mmu_base + 0x1c); /* * emutlbupdate|TWLENABLE|MMUENABLE */ __raw_writel(0x6, mmu_base + 0x44); } return 0; } /** * enum ipu_mem - PRU core memory range identifiers */ enum ipu_mem { PRU_MEM_IRAM = 0, PRU_MEM_CTRL, PRU_MEM_DEBUG, PRU_MEM_MAX, }; static int ipu_start(struct udevice *dev) { struct ipu_privdata *priv; struct reset_ctl reset; struct rproc *cfg = NULL; int ret; priv = dev_get_priv(dev); cfg = rproc_cfg_arr[priv->id]; if (cfg->config_peripherals) cfg->config_peripherals(priv->id, cfg); /* * Start running the remote core */ ret = reset_get_by_index(dev, 0, &reset); if (ret < 0) { dev_err(dev, "%s: error getting reset index %d\n", __func__, 0); return ret; } ret = reset_deassert(&reset); if (ret < 0) { dev_err(dev, "%s: error deasserting reset %d\n", __func__, 0); return ret; } ret = reset_get_by_index(dev, 1, &reset); if (ret < 0) { dev_err(dev, "%s: error getting reset index %d\n", __func__, 1); return ret; } ret = reset_deassert(&reset); if (ret < 0) { dev_err(dev, "%s: error deasserting reset %d\n", __func__, 1); return ret; } return 0; } static int ipu_stop(struct udevice *dev) { return 0; } /** * ipu_init() - Initialize the remote processor * @dev: rproc device pointer * * Return: 0 if all went ok, else return appropriate error */ static int ipu_init(struct udevice *dev) { return 0; } static int ipu_add_res(struct udevice *dev, struct rproc_mem_entry *mapping) { struct ipu_privdata *priv = dev_get_priv(dev); list_add_tail(&mapping->node, &priv->mappings); return 0; } static int ipu_load(struct udevice *dev, ulong addr, ulong size) { Elf32_Ehdr *ehdr; /* Elf header structure pointer */ Elf32_Phdr *phdr; /* Program header structure pointer */ Elf32_Phdr proghdr; int va; int pa; int i; ehdr = (Elf32_Ehdr *)addr; phdr = (Elf32_Phdr *)(addr + ehdr->e_phoff); /* * Load each program header */ for (i = 0; i < ehdr->e_phnum; ++i) { memcpy(&proghdr, phdr, sizeof(Elf32_Phdr)); if (proghdr.p_type != PT_LOAD) { ++phdr; continue; } va = proghdr.p_paddr; pa = da_to_pa(dev, va); if (pa) proghdr.p_paddr = pa; void *dst = (void *)(uintptr_t)proghdr.p_paddr; void *src = (void *)addr + proghdr.p_offset; debug("Loading phdr %i to 0x%p (%i bytes)\n", i, dst, proghdr.p_filesz); if (proghdr.p_filesz) memcpy(dst, src, proghdr.p_filesz); flush_cache((unsigned long)dst, proghdr.p_memsz); ++phdr; } return 0; } static const struct dm_rproc_ops ipu_ops = { .init = ipu_init, .start = ipu_start, .stop = ipu_stop, .load = ipu_load, .add_res = ipu_add_res, .config_pagetable = ipu_config_pagetable, .alloc_mem = ipu_alloc_mem, }; /* * If the remotecore binary expects any peripherals to be setup before it has * booted, configure them here. * * These functions are left empty by default as their operation is usecase * specific. */ u32 ipu1_config_peripherals(u32 core_id, struct rproc *cfg) { return 0; } u32 ipu2_config_peripherals(u32 core_id, struct rproc *cfg) { return 0; } struct rproc_intmem_to_l3_mapping ipu1_intmem_to_l3_mapping = { .num_entries = 1, .mappings = { /* * L2 SRAM */ { .priv_addr = 0x55020000, .l3_addr = 0x58820000, .len = (64 * 1024)}, } }; struct rproc_intmem_to_l3_mapping ipu2_intmem_to_l3_mapping = { .num_entries = 1, .mappings = { /* * L2 SRAM */ { .priv_addr = 0x55020000, .l3_addr = 0x55020000, .len = (64 * 1024)}, } }; struct rproc ipu1_config = { .num_iommus = 1, .mmu_base_addr = {0x58882000, 0}, .load_addr = IPU1_LOAD_ADDR, .core_name = "IPU1", .firmware_name = "dra7-ipu1-fw.xem4", .config_mmu = ipu_config_mmu, .config_peripherals = ipu1_config_peripherals, .intmem_to_l3_mapping = &ipu1_intmem_to_l3_mapping }; struct rproc ipu2_config = { .num_iommus = 1, .mmu_base_addr = {0x55082000, 0}, .load_addr = IPU2_LOAD_ADDR, .core_name = "IPU2", .firmware_name = "dra7-ipu2-fw.xem4", .config_mmu = ipu_config_mmu, .config_peripherals = ipu2_config_peripherals, .intmem_to_l3_mapping = &ipu2_intmem_to_l3_mapping }; struct rproc *rproc_cfg_arr[2] = { [IPU2] = &ipu2_config, [IPU1] = &ipu1_config, }; u32 spl_pre_boot_core(struct udevice *dev, u32 core_id) { struct rproc *cfg = NULL; unsigned long load_elf_status = 0; int tablesz; cfg = rproc_cfg_arr[core_id]; /* * Check for valid elf image */ if (!valid_elf_image(cfg->load_addr)) return 1; if (rproc_find_resource_table(dev, cfg->load_addr, &tablesz)) cfg->has_rsc_table = 1; else cfg->has_rsc_table = 0; /* * Configure the MMU */ if (cfg->config_mmu && cfg->has_rsc_table) cfg->config_mmu(core_id, cfg); /* * Load the remote core. Fill the page table of the first(possibly * only) IOMMU during ELF loading. Copy the page table to the second * IOMMU before running the remote core. */ page_table_l1 = (unsigned int *)cfg->page_table_addr; page_table_l2 = (unsigned int *)(cfg->page_table_addr + PAGE_TABLE_SIZE_L1); mem_base = cfg->cma_base; mem_size = cfg->cma_size; memset(mem_bitmap, 0x00, sizeof(mem_bitmap)); mem_count = (cfg->cma_size >> PAGE_SHIFT); /* * Clear variables used for level 2 page table allocation */ memset(pgtable_l2_map, 0x00, sizeof(pgtable_l2_map)); pgtable_l2_cnt = 0; load_elf_status = rproc_parse_resource_table(dev, cfg); if (load_elf_status == 0) { debug("load_elf_image_phdr returned error for core %s\n", cfg->core_name); return 1; } flush_cache(cfg->page_table_addr, PAGE_TABLE_SIZE); return 0; } static fdt_addr_t ipu_parse_mem_nodes(struct udevice *dev, char *name, int privid, fdt_size_t *sizep) { int ret; u32 sp; ofnode mem_node; ret = ofnode_read_u32(dev_ofnode(dev), name, &sp); if (ret) { dev_err(dev, "memory-region node fetch failed %d\n", ret); return ret; } mem_node = ofnode_get_by_phandle(sp); if (!ofnode_valid(mem_node)) return -EINVAL; return ofnode_get_addr_size_index(mem_node, 0, sizep); } /** * ipu_probe() - Basic probe * @dev: corresponding k3 remote processor device * * Return: 0 if all goes good, else appropriate error message. */ static int ipu_probe(struct udevice *dev) { struct ipu_privdata *priv; struct rproc *cfg = NULL; struct reset_ctl reset; static const char *const ipu_mem_names[] = { "l2ram" }; int ret; fdt_size_t sizep; priv = dev_get_priv(dev); priv->mem.bus_addr = devfdt_get_addr_size_name(dev, ipu_mem_names[0], (fdt_addr_t *)&priv->mem.size); ret = reset_get_by_index(dev, 2, &reset); if (ret < 0) { dev_err(dev, "%s: error getting reset index %d\n", __func__, 2); return ret; } ret = reset_deassert(&reset); if (ret < 0) { dev_err(dev, "%s: error deasserting reset %d\n", __func__, 2); return ret; } if (priv->mem.bus_addr == FDT_ADDR_T_NONE) { dev_err(dev, "%s bus address not found\n", ipu_mem_names[0]); return -EINVAL; } priv->mem.cpu_addr = map_physmem(priv->mem.bus_addr, priv->mem.size, MAP_NOCACHE); if (devfdt_get_addr(dev) == 0x58820000) priv->id = 0; else priv->id = 1; cfg = rproc_cfg_arr[priv->id]; cfg->cma_base = ipu_parse_mem_nodes(dev, "memory-region", priv->id, &sizep); cfg->cma_size = sizep; cfg->page_table_addr = ipu_parse_mem_nodes(dev, "pg-tbl", priv->id, &sizep); dev_info(dev, "ID %d memory %8s: bus addr %pa size 0x%zx va %p da 0x%x\n", priv->id, ipu_mem_names[0], &priv->mem.bus_addr, priv->mem.size, priv->mem.cpu_addr, priv->mem.dev_addr); INIT_LIST_HEAD(&priv->mappings); if (spl_pre_boot_core(dev, priv->id)) return -EINVAL; return 0; } static const struct udevice_id ipu_ids[] = { {.compatible = "ti,dra7-ipu"}, {} }; U_BOOT_DRIVER(ipu) = { .name = "ipu", .of_match = ipu_ids, .id = UCLASS_REMOTEPROC, .ops = &ipu_ops, .probe = ipu_probe, .priv_auto = sizeof(struct ipu_privdata), };