From 6c12ba5fbd0cca3619493365575b670fa86a857d Mon Sep 17 00:00:00 2001 From: Mayuresh Kulkarni Date: Wed, 9 Mar 2011 16:00:12 +0530 Subject: [ARM]: tegra: add support for mc/emc bandwidth statistics provide a mechanism to perform statistical sampling of the memory controller usage on a client-by-client basis, and report the log through sysfs original work by: gking@nvidia.com Original-Change-Id: I5d7f357af0353c55e14026b036ccf7e448df643b Signed-off-by: Mayuresh Kulkarni Reviewed-on: http://git-master/r/20383 Reviewed-by: Bharat Nihalani Rebase-Id: R591569f7b618508b34c2abe130700d2668779310 --- arch/arm/mach-tegra/Kconfig | 8 + arch/arm/mach-tegra/Makefile | 1 + arch/arm/mach-tegra/tegra2_mc.c | 1149 +++++++++++++++++++++++++++++++++++++++ arch/arm/mach-tegra/tegra2_mc.h | 271 +++++++++ 4 files changed, 1429 insertions(+) create mode 100644 arch/arm/mach-tegra/tegra2_mc.c create mode 100644 arch/arm/mach-tegra/tegra2_mc.h (limited to 'arch/arm') diff --git a/arch/arm/mach-tegra/Kconfig b/arch/arm/mach-tegra/Kconfig index 87b0f8a69fe3..abddc72f2fde 100644 --- a/arch/arm/mach-tegra/Kconfig +++ b/arch/arm/mach-tegra/Kconfig @@ -168,4 +168,12 @@ config TEGRA_CLOCK_DEBUG_WRITE depends on DEBUG_FS default n +config TEGRA_MC_PROFILE + tristate "Enable profiling memory controller utilization" + default n + help + When enabled, provides a mechanism to perform statistical + sampling of the memory controller usage on a client-by-client + basis, and report the log through sysfs. + endif diff --git a/arch/arm/mach-tegra/Makefile b/arch/arm/mach-tegra/Makefile index dd8893a9e364..d77b2d4e6954 100644 --- a/arch/arm/mach-tegra/Makefile +++ b/arch/arm/mach-tegra/Makefile @@ -46,6 +46,7 @@ obj-$(CONFIG_USB_SUPPORT) += usb_phy.o obj-$(CONFIG_CPU_IDLE) += cpuidle.o obj-$(CONFIG_TEGRA_IOVMM) += iovmm.o obj-$(CONFIG_TEGRA_IOVMM_GART) += iovmm-gart.o +obj-$(CONFIG_TEGRA_MC_PROFILE) += tegra2_mc.o obj-${CONFIG_MACH_HARMONY} += board-harmony.o obj-${CONFIG_MACH_HARMONY} += board-harmony-panel.o diff --git a/arch/arm/mach-tegra/tegra2_mc.c b/arch/arm/mach-tegra/tegra2_mc.c new file mode 100644 index 000000000000..dd49380bbed3 --- /dev/null +++ b/arch/arm/mach-tegra/tegra2_mc.c @@ -0,0 +1,1149 @@ +/* + * arch/arm/mach-tegra/tegra2_mc.c + * + * Memory controller bandwidth profiling interface + * + * Copyright (c) 2009-2011, NVIDIA Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tegra2_mc.h" + +static void stat_start(void); +static void stat_stop(void); +static void stat_log(void); + +static struct hrtimer sample_timer; + +#define MC_COUNTER_INITIALIZER() \ + { \ + .enabled = false, \ + .reschedule = false, \ + .period = 10, \ + .mode = FILTER_CLIENT, \ + .address_low = 0, \ + .address_length_1 = 0xfffffffful, \ + .address_window_size_1 = PAGE_SIZE, \ + .client_number = 0, \ + } + +static struct tegra_mc_counter mc_counter0 = MC_COUNTER_INITIALIZER(); +static struct tegra_mc_counter mc_counter1 = MC_COUNTER_INITIALIZER(); +static struct tegra_mc_counter emc_llp_counter = MC_COUNTER_INITIALIZER(); + +/* /sys/devices/system/tegra_mc */ +static bool sample_enable = SAMPLE_ENABLE_DEFAULT; +static u16 sample_quantum = SAMPLE_QUANTUM_DEFAULT; +static u8 sample_log[SAMPLE_LOG_SIZE]; + +static DEFINE_SPINLOCK(sample_enable_lock); +static DEFINE_SPINLOCK(sample_log_lock); + +static u8 *sample_log_wptr = sample_log, *sample_log_rptr = sample_log; +static int sample_log_size = SAMPLE_LOG_SIZE - 1; + +static bool sampling(void) +{ + bool ret; + + spin_lock_bh(&sample_enable_lock); + ret = (sample_enable == true)? true : false; + spin_unlock_bh(&sample_enable_lock); + + return ret; +} + +static struct sysdev_class tegra_mc_sysclass = { + .name = "tegra_mc", +}; + +static ssize_t tegra_mc_enable_show(struct sysdev_class *class, + struct sysdev_class_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", sample_enable); +} + +static ssize_t tegra_mc_enable_store(struct sysdev_class *class, + struct sysdev_class_attribute *attr, + const char *buf, size_t count) +{ + int value, i; + struct tegra_mc_counter *counters[] = { + &mc_counter0, + &mc_counter1, + &emc_llp_counter + }; + + sscanf(buf, "%d", &value); + + if (value == 0 || value == 1) + sample_enable = value; + else + return -EINVAL; + + if (!sample_enable) { + stat_stop(); + hrtimer_cancel(&sample_timer); + return count; + } + + hrtimer_cancel(&sample_timer); + + /* we need to initialize variables that change during sampling */ + sample_log_wptr = sample_log_rptr = sample_log; + sample_log_size = SAMPLE_LOG_SIZE - 1; + + for (i = 0; i < ARRAY_SIZE(counters); i++) { + struct tegra_mc_counter *c = counters[i]; + + if (!c->enabled) + continue; + + c->current_address_low = c->address_low; + c->current_address_high = c->address_low; + c->address_range_change = (c->mode == FILTER_ADDR); + if (c->address_range_change) + c->current_address_high += c->address_window_size_1; + else + c->current_address_high += c->address_length_1; + + c->current_client = 0; + c->sample_count = 0; + } + + stat_start(); + + hrtimer_start(&sample_timer, + ktime_add_ns(ktime_get(), (u64)sample_quantum * 1000000), + HRTIMER_MODE_ABS); + + return count; +} + +static ssize_t tegra_mc_log_show(struct sysdev_class *class, + struct sysdev_class_attribute *attr, char *buf) +{ + int index = 0, count = 0; + unsigned long flags; + + spin_lock_irqsave(&sample_log_lock, flags); + + while (sample_log_rptr != sample_log_wptr) { + if (sample_log_rptr < sample_log_wptr) { + count = sample_log_wptr - sample_log_rptr; + memcpy(buf + index, sample_log_rptr, count); + sample_log_rptr = sample_log_wptr; + sample_log_size += count; + } else { + count = SAMPLE_LOG_SIZE - + (sample_log_rptr - sample_log); + memcpy(buf + index, sample_log_rptr, count); + sample_log_rptr = sample_log; + sample_log_size += count; + } + index += count; + } + + spin_unlock_irqrestore(&sample_log_lock, flags); + + return index; +} + +static ssize_t tegra_mc_log_store(struct sysdev_class *class, + struct sysdev_class_attribute *attr, + const char *buf, size_t count) +{ + return -EPERM; +} + +static ssize_t tegra_mc_quantum_show(struct sysdev_class *class, + struct sysdev_class_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", sample_quantum); +} + +static ssize_t tegra_mc_quantum_store(struct sysdev_class *class, + struct sysdev_class_attribute *attr, + const char *buf, size_t count) +{ + int value; + + if (sampling()) + return -EINVAL; + + sscanf(buf, "%d", &value); + sample_quantum = value; + + return count; +} + +#define TEGRA_MC_EXPAND(_attr,_mode) \ + static SYSDEV_CLASS_ATTR( \ + _attr, _mode, tegra_mc_##_attr##_show, tegra_mc_##_attr##_store); + +#define TEGRA_MC_ATTRIBUTES(_attr1,_mode1,_attr2,_mode2,_attr3,_mode3) \ + TEGRA_MC_EXPAND(_attr1,_mode1) \ + TEGRA_MC_EXPAND(_attr2,_mode2) \ + TEGRA_MC_EXPAND(_attr3,_mode3) + +TEGRA_MC_ATTRIBUTES(enable,0666,log,0444,quantum,0666) + +#undef TEGRA_MC_EXPAND + +#define TEGRA_MC_EXPAND(_attr,_mode) \ + &attr_##_attr, + +static struct sysdev_class_attribute *tegra_mc_attrs[] = { + TEGRA_MC_ATTRIBUTES(enable,0666,log,0444,quantum,0666) + NULL +}; + +/* /sys/devices/system/tegra_mc/client */ +static bool tegra_mc_client_0_enabled = CLIENT_ENABLED_DEFAULT; +static u8 tegra_mc_client_0_on_schedule_buffer[CLIENT_ON_SCHEDULE_LENGTH]; +static struct kobject *tegra_mc_client_kobj, *tegra_mc_client_0_kobj; + +struct match_mode { + const char *name; + int mode; +}; + +static const struct match_mode mode_list[] = { + [0] = { + .name = "none", + .mode = FILTER_NONE, + }, + [1] = { + .name = "address", + .mode = FILTER_ADDR, + }, + [2] = { + .name = "client", + .mode = FILTER_CLIENT, + }, +}; + +static int tegra_mc_parse_mode(const char* str) { + int i; + + for (i = 0; i < ARRAY_SIZE(mode_list); i++) { + if (!strncmp(str, mode_list[i].name, strlen(mode_list[i].name))) + return mode_list[i].mode; + } + return -EINVAL; +} + +static int tegra_mc_client_parse(const char *buf, size_t count, + tegra_mc_counter_t *counter0, tegra_mc_counter_t *counter1, + tegra_mc_counter_t *llp) +{ + char *options, *p, *ptr; + tegra_mc_counter_t *counter; + substring_t args[MAX_OPT_ARGS]; + enum { + opt_period, + opt_mode, + opt_client, + opt_address_low, + opt_address_length, + opt_address_window_size, + opt_err, + }; + const match_table_t tokens = { + {opt_period, "period=%s"}, + {opt_mode, "mode=%s"}, + {opt_client, "client=%s"}, + {opt_address_low, "address_low=%s"}, + {opt_address_length, "address_length=%s"}, + {opt_address_window_size, "address_window_size=%s"}, + {opt_err, NULL}, + }; + int ret = 0, i, token, client_number; + bool aggregate = false; + int period, *client_ids, mode; + bool fperiod = false, fmode = false, fclient = false; + u64 address_low = 0; + u64 address_length = 1ull<<32; + u64 address_window_size = PAGE_SIZE; + + client_ids = kmalloc(sizeof(int) * (MC_COUNTER_CLIENT_SIZE + 1), + GFP_KERNEL); + if (!client_ids) + return -ENOMEM; + + options = kstrdup(buf, GFP_KERNEL); + if (!options) { + ret = -ENOMEM; + goto end; + } + + while ((p = strsep(&options, " ")) != NULL) { + if (!*p) + continue; + + pr_debug("\t %s\n", p); + + token = match_token(p, tokens, args); + switch (token) { + case opt_period: + if (match_int(&args[0], &period) || period<=0) { + ret = -EINVAL; + goto end; + } + fperiod = true; + break; + + case opt_mode: + mode = tegra_mc_parse_mode(args[0].from); + if (mode<0) { + ret = mode; + goto end; + } + fmode = true; + break; + + case opt_client: + client_ids[0] = 0; + + ptr = get_options(args[0].from, + MC_COUNTER_CLIENT_SIZE+1 , client_ids); + + if (client_ids[0] <= 0) { + ret = -EINVAL; + goto end; + } + + for (i = 1; i <= client_ids[0]; i++) { + if (client_ids[i] < MC_STAT_END) + continue; + + if ((client_ids[i] != MC_STAT_AGGREGATE) || + client_ids[0] != 1) { + ret = -EINVAL; + goto end; + } else + aggregate = true; + } + + client_number = client_ids[0]; + fclient = true; + break; + + case opt_address_low: + address_low = simple_strtoull(args[0].from, NULL, 0); + break; + + case opt_address_length: + address_length = simple_strtoull(args[0].from, NULL, 0); + break; + + case opt_address_window_size: + address_window_size = simple_strtoull(args[0].from, + NULL, 0); + break; + + default: + ret = -EINVAL; + goto end; + } + } + + if (!fmode || !fclient || (mode == FILTER_CLIENT && aggregate)) { + ret = -EINVAL; + goto end; + } + + address_low &= PAGE_MASK; + address_length += PAGE_SIZE-1; + address_length &= ~((1ull << PAGE_SHIFT)-1ull); + + address_window_size += PAGE_SIZE-1; + address_window_size &= ~((1ull << PAGE_SHIFT)-1ull); + + if (mode == FILTER_CLIENT) { + counter = counter0; + counter->reschedule = (client_number>1); + counter->client_number = client_number; + llp->enabled = false; + counter1->enabled = false; + for (i = 1; i <= client_number && i < MC_COUNTER_CLIENT_SIZE; i++) + counter->clients[i - 1] = client_ids[i]; + } else if (mode == FILTER_ADDR || mode == FILTER_NONE) { + if (aggregate) { + counter = counter1; + llp->enabled = true; + counter0->enabled = false; + } else { + counter = counter0; + counter1->enabled = false; + llp->enabled = false; + } + counter->client_number = 1; + counter->clients[0] = client_ids[1]; + counter->reschedule = (mode != FILTER_NONE); + } else { + ret = -EINVAL; + goto end; + } + + counter->mode = mode; + counter->enabled = true; + counter->address_low = (u32)address_low; + counter->address_length_1 = (u32)(address_length-1); + counter->address_window_size_1 = (u32)(address_window_size-1); + + if (llp->enabled) { + llp->mode = counter->mode; + llp->reschedule = counter->reschedule; + llp->period = counter->period; + llp->address_low = counter->address_low; + llp->address_length_1 = counter->address_length_1; + llp->address_window_size_1 = counter->address_window_size_1; + } + +end: + if (options) + kfree(options); + if (client_ids) + kfree(client_ids); + + return ret; +} + +static ssize_t tegra_mc_client_0_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + if (strcmp(attr->attr.name, "enable") == 0) + return sprintf(buf, "%d\n", tegra_mc_client_0_enabled); + else if (strcmp(attr->attr.name, "on_schedule") == 0) + return sprintf(buf, "%s", tegra_mc_client_0_on_schedule_buffer); + else + return -EINVAL; +} + +static ssize_t tegra_mc_client_0_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + int value; + + if (sampling()) + return -EINVAL; + + if (strcmp(attr->attr.name, "enable") == 0) { + sscanf(buf, "%d\n", &value); + if (value == 0 || value == 1) + tegra_mc_client_0_enabled = value; + else + return -EINVAL; + + return count; + } else if (strcmp(attr->attr.name, "on_schedule") == 0) { + if (tegra_mc_client_parse(buf, count, + &mc_counter0, &mc_counter1, + &emc_llp_counter)== 0) { + + strncpy(tegra_mc_client_0_on_schedule_buffer, + buf, count); + + return count; + } else + return -EINVAL; + } else + return -EINVAL; +} + +static struct kobj_attribute tegra_mc_client_0_enable = + __ATTR(enable, 0660, tegra_mc_client_0_show, tegra_mc_client_0_store); + +static struct kobj_attribute tegra_mc_client_0_on_schedule = + __ATTR(on_schedule, 0660, tegra_mc_client_0_show, tegra_mc_client_0_store); + +static struct attribute *tegra_mc_client_0_attrs[] = { + &tegra_mc_client_0_enable.attr, + &tegra_mc_client_0_on_schedule.attr, + NULL, +}; + +static struct attribute_group tegra_mc_client_0_attr_group = { + .attrs = tegra_mc_client_0_attrs +}; + +/* /sys/devices/system/tegra_mc/dram */ +#define dram_counters(_x) \ + _x(activate_cnt, ACTIVATE_CNT) \ + _x(read_cnt, READ_CNT) \ + _x(write_cnt, WRITE_CNT) \ + _x(ref_cnt, REF_CNT) \ + _x(cumm_banks_active_cke_eq1, CUMM_BANKS_ACTIVE_CKE_EQ1) \ + _x(cumm_banks_active_cke_eq0, CUMM_BANKS_ACTIVE_CKE_EQ0) \ + _x(cke_eq1_clks, CKE_EQ1_CLKS) \ + _x(extclks_cke_eq1, EXTCLKS_CKE_EQ1) \ + _x(extclks_cke_eq0, EXTCLKS_CKE_EQ0) \ + _x(no_banks_active_cke_eq1, NO_BANKS_ACTIVE_CKE_EQ1) \ + _x(no_banks_active_cke_eq0, NO_BANKS_ACTIVE_CKE_EQ0) + +#define DEFINE_COUNTER(_name, _val) { .enabled = false, .device_mask = 0, }, + +static tegra_emc_dram_counter_t dram_counters[] = { + dram_counters(DEFINE_COUNTER) +}; + +#define DEFINE_SYSFS(_name, _val) \ + \ +static struct kobject *tegra_mc_dram_##_name##_kobj; \ + \ +static ssize_t tegra_mc_dram_##_name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return tegra_mc_dram_show(kobj, attr, buf, \ + _val - EMC_DRAM_STAT_BEGIN); \ +} \ + \ +static ssize_t tegra_mc_dram_##_name##_store(struct kobject *kobj, \ + struct kobj_attribute *attr, const char *buf, size_t count) \ +{ \ + if (sampling()) \ + return 0; \ + \ + return tegra_mc_dram_store(kobj, attr, buf, count, \ + _val - EMC_DRAM_STAT_BEGIN); \ +} \ + \ + \ +static struct kobj_attribute tegra_mc_dram_##_name##_enable = \ + __ATTR(enable, 0660, tegra_mc_dram_##_name##_show, \ + tegra_mc_dram_##_name##_store); \ + \ +static struct kobj_attribute tegra_mc_dram_##_name##_device_mask = \ + __ATTR(device_mask, 0660, tegra_mc_dram_##_name##_show, \ + tegra_mc_dram_##_name##_store); \ + \ +static struct attribute *tegra_mc_dram_##_name##_attrs[] = { \ + &tegra_mc_dram_##_name##_enable.attr, \ + &tegra_mc_dram_##_name##_device_mask.attr, \ + NULL, \ +}; \ + \ +static struct attribute_group tegra_mc_dram_##_name##_attr_group = { \ + .attrs = tegra_mc_dram_##_name##_attrs, \ +}; + +static struct kobject *tegra_mc_dram_kobj; + +static ssize_t tegra_mc_dram_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf, int index) +{ + if (index >= EMC_DRAM_STAT_END - EMC_DRAM_STAT_BEGIN) + return -EINVAL; + + if (strcmp(attr->attr.name, "enable") == 0) + return sprintf(buf, "%d\n", dram_counters[index].enabled); + else if (strcmp(attr->attr.name, "device_mask") == 0) + return sprintf(buf, "%d\n", dram_counters[index].device_mask); + else + return -EINVAL; +} +static ssize_t tegra_mc_dram_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count, int index) +{ + int value; + + if (index >= EMC_DRAM_STAT_END - EMC_DRAM_STAT_BEGIN) + return -EINVAL; + + if (strcmp(attr->attr.name, "enable") == 0) { + sscanf(buf, "%d\n", &value); + if (value == 0 || value == 1) + dram_counters[index].enabled = value; + else + return -EINVAL; + + return count; + } else if (strcmp(attr->attr.name, "device_mask") == 0) { + sscanf(buf, "%d\n", &value); + dram_counters[index].device_mask = (u8)value; + + return count; + } else + return -EINVAL; +} + +dram_counters(DEFINE_SYSFS) + +/* Tegra Statistics */ +typedef struct { + void __iomem *mmio; +} tegra_device_t; + +static tegra_device_t mc = { + .mmio = IO_ADDRESS(TEGRA_MC_BASE), +}; + +static tegra_device_t emc = { + .mmio = IO_ADDRESS(TEGRA_EMC_BASE), +}; + +void mc_stat_start(tegra_mc_counter_t *counter0, tegra_mc_counter_t *counter1) +{ + struct tegra_mc_counter *c; + u32 filter_client = ARMC_STAT_CONTROL_FILTER_CLIENT_DISABLE; + u32 filter_addr = ARMC_STAT_CONTROL_FILTER_ADDR_DISABLE; + + if (!tegra_mc_client_0_enabled) + return; + + c = (counter0->enabled) ? counter0 : counter1; + + /* disable statistics */ + writel((MC_STAT_CONTROL_0_EMC_GATHER_DISABLE << MC_STAT_CONTROL_0_EMC_GATHER_SHIFT), + mc.mmio + MC_STAT_CONTROL_0); + + if (c->enabled && c->mode == FILTER_ADDR) + filter_addr = ARMC_STAT_CONTROL_FILTER_ADDR_ENABLE; + else if (c->enabled && c->mode == FILTER_CLIENT) + filter_client = ARMC_STAT_CONTROL_FILTER_CLIENT_ENABLE; + + filter_addr <<= ARMC_STAT_CONTROL_FILTER_ADDR_SHIFT; + filter_client <<= ARMC_STAT_CONTROL_FILTER_CLIENT_SHIFT; + + if (c->enabled) { + u32 reg = 0; + reg |= (ARMC_STAT_CONTROL_MODE_BANDWIDTH << + ARMC_STAT_CONTROL_MODE_SHIFT); + reg |= (ARMC_STAT_CONTROL_EVENT_QUALIFIED << + ARMC_STAT_CONTROL_EVENT_SHIFT); + reg |= (ARMC_STAT_CONTROL_FILTER_PRI_DISABLE << + ARMC_STAT_CONTROL_FILTER_PRI_SHIFT); + reg |= (ARMC_STAT_CONTROL_FILTER_COALESCED_DISABLE << + ARMC_STAT_CONTROL_FILTER_COALESCED_SHIFT); + reg |= filter_client; + reg |= filter_addr; + reg |= (c->clients[c->current_client] << + ARMC_STAT_CONTROL_CLIENT_ID_SHIFT); + + /* note these registers are shared */ + writel(c->current_address_low, + mc.mmio + MC_STAT_EMC_ADDR_LOW_0); + writel(c->current_address_high, + mc.mmio + MC_STAT_EMC_ADDR_HIGH_0); + writel(0xFFFFFFFF, mc.mmio + MC_STAT_EMC_CLOCK_LIMIT_0); + + writel(reg, mc.mmio + MC_STAT_EMC_CONTROL_0_0); + } + + /* reset then enable statistics */ + writel((MC_STAT_CONTROL_0_EMC_GATHER_CLEAR << MC_STAT_CONTROL_0_EMC_GATHER_SHIFT), + mc.mmio + MC_STAT_CONTROL_0); + + writel((MC_STAT_CONTROL_0_EMC_GATHER_ENABLE << MC_STAT_CONTROL_0_EMC_GATHER_SHIFT), + mc.mmio + MC_STAT_CONTROL_0); +} + +void mc_stat_stop(tegra_mc_counter_t *counter0, + tegra_mc_counter_t *counter1) +{ + /* Disable statistics */ + writel((MC_STAT_CONTROL_0_EMC_GATHER_DISABLE << MC_STAT_CONTROL_0_EMC_GATHER_SHIFT), + mc.mmio + MC_STAT_CONTROL_0); + + if (counter0->enabled) + counter0->value = readl(mc.mmio + MC_STAT_EMC_COUNT_0_0); + else + counter1->value = readl(mc.mmio + MC_STAT_EMC_COUNT_1_0); +} + +void emc_stat_start(tegra_mc_counter_t *llp_counter, + tegra_emc_dram_counter_t *dram_counter) +{ + u32 llmc_stat = 0; + u32 llmc_ctrl = + (AREMC_STAT_CONTROL_MODE_BANDWIDTH << + AREMC_STAT_CONTROL_MODE_SHIFT) | + (AREMC_STAT_CONTROL_CLIENT_TYPE_MPCORER << + AREMC_STAT_CONTROL_CLIENT_TYPE_SHIFT) | + (AREMC_STAT_CONTROL_EVENT_QUALIFIED << + AREMC_STAT_CONTROL_EVENT_SHIFT); + + /* disable statistics */ + llmc_stat |= (EMC_STAT_CONTROL_0_LLMC_GATHER_DISABLE << + EMC_STAT_CONTROL_0_LLMC_GATHER_SHIFT); + llmc_stat |= (EMC_STAT_CONTROL_0_DRAM_GATHER_DISABLE << + EMC_STAT_CONTROL_0_DRAM_GATHER_SHIFT); + writel(llmc_stat, emc.mmio + EMC_STAT_CONTROL_0); + + if (tegra_mc_client_0_enabled && llp_counter->enabled) { + if (llp_counter->mode == FILTER_ADDR) { + llmc_ctrl |= + (AREMC_STAT_CONTROL_FILTER_ADDR_ENABLE << + AREMC_STAT_CONTROL_FILTER_ADDR_SHIFT); + llmc_ctrl |= + (AREMC_STAT_CONTROL_FILTER_CLIENT_DISABLE << + AREMC_STAT_CONTROL_FILTER_CLIENT_SHIFT); + } else if (llp_counter->mode == FILTER_CLIENT) { + /* not allow aggregate client in client mode */ + llmc_ctrl |= + (AREMC_STAT_CONTROL_FILTER_ADDR_DISABLE << + AREMC_STAT_CONTROL_FILTER_ADDR_SHIFT); + llmc_ctrl |= + (AREMC_STAT_CONTROL_FILTER_CLIENT_DISABLE << + AREMC_STAT_CONTROL_FILTER_CLIENT_SHIFT); + } else if (llp_counter->mode == FILTER_NONE) { + llmc_ctrl |= + (AREMC_STAT_CONTROL_FILTER_ADDR_DISABLE << + AREMC_STAT_CONTROL_FILTER_ADDR_SHIFT); + llmc_ctrl |= + (AREMC_STAT_CONTROL_FILTER_CLIENT_DISABLE << + AREMC_STAT_CONTROL_FILTER_CLIENT_SHIFT); + } + + writel(llp_counter->current_address_low, + emc.mmio + EMC_STAT_LLMC_ADDR_LOW_0); + writel(llp_counter->current_address_high, + emc.mmio + EMC_STAT_LLMC_ADDR_HIGH_0); + writel(0xFFFFFFFF, emc.mmio + EMC_STAT_LLMC_CLOCK_LIMIT_0); + writel(llmc_ctrl, emc.mmio + EMC_STAT_LLMC_CONTROL_0_0); + } + + writel(0xFFFFFFFF, emc.mmio + EMC_STAT_DRAM_CLOCK_LIMIT_LO_0); + writel(0xFF, emc.mmio + EMC_STAT_DRAM_CLOCK_LIMIT_HI_0); + + /* Reset then enable statistics */ + llmc_stat |= (EMC_STAT_CONTROL_0_LLMC_GATHER_CLEAR << + EMC_STAT_CONTROL_0_LLMC_GATHER_SHIFT); + llmc_stat |= (EMC_STAT_CONTROL_0_DRAM_GATHER_CLEAR << + EMC_STAT_CONTROL_0_DRAM_GATHER_SHIFT); + writel(llmc_stat, emc.mmio + EMC_STAT_CONTROL_0); + + llmc_stat |= (EMC_STAT_CONTROL_0_LLMC_GATHER_ENABLE << + EMC_STAT_CONTROL_0_LLMC_GATHER_SHIFT); + llmc_stat |= (EMC_STAT_CONTROL_0_DRAM_GATHER_ENABLE << + EMC_STAT_CONTROL_0_DRAM_GATHER_SHIFT); + writel(llmc_stat, emc.mmio + EMC_STAT_CONTROL_0); +} + +void emc_stat_stop(tegra_mc_counter_t *llp_counter, + tegra_emc_dram_counter_t *dram_counter) +{ + u32 llmc_stat = 0; + int i; + int dev0_offsets_lo[] = { + EMC_STAT_DRAM_DEV0_ACTIVATE_CNT_LO_0, + EMC_STAT_DRAM_DEV0_READ_CNT_LO_0, + EMC_STAT_DRAM_DEV0_WRITE_CNT_LO_0, + EMC_STAT_DRAM_DEV0_REF_CNT_LO_0, + EMC_STAT_DRAM_DEV0_CUMM_BANKS_ACTIVE_CKE_EQ1_LO_0, + EMC_STAT_DRAM_DEV0_CUMM_BANKS_ACTIVE_CKE_EQ0_LO_0, + EMC_STAT_DRAM_DEV0_CKE_EQ1_CLKS_LO_0, + EMC_STAT_DRAM_DEV0_EXTCLKS_CKE_EQ1_LO_0, + EMC_STAT_DRAM_DEV0_EXTCLKS_CKE_EQ0_LO_0, + EMC_STAT_DRAM_DEV0_NO_BANKS_ACTIVE_CKE_EQ1_LO_0, + EMC_STAT_DRAM_DEV0_NO_BANKS_ACTIVE_CKE_EQ0_LO_0, + }; + int dev0_offsets_hi[] = { + EMC_STAT_DRAM_DEV0_ACTIVATE_CNT_HI_0, + EMC_STAT_DRAM_DEV0_READ_CNT_HI_0, + EMC_STAT_DRAM_DEV0_WRITE_CNT_HI_0, + EMC_STAT_DRAM_DEV0_REF_CNT_HI_0, + EMC_STAT_DRAM_DEV0_CUMM_BANKS_ACTIVE_CKE_EQ1_HI_0, + EMC_STAT_DRAM_DEV0_CUMM_BANKS_ACTIVE_CKE_EQ0_HI_0, + EMC_STAT_DRAM_DEV0_CKE_EQ1_CLKS_HI_0, + EMC_STAT_DRAM_DEV0_EXTCLKS_CKE_EQ1_HI_0, + EMC_STAT_DRAM_DEV0_EXTCLKS_CKE_EQ0_HI_0, + EMC_STAT_DRAM_DEV0_NO_BANKS_ACTIVE_CKE_EQ1_HI_0, + EMC_STAT_DRAM_DEV0_NO_BANKS_ACTIVE_CKE_EQ0_HI_0, + }; + int dev1_offsets_lo[] = { + EMC_STAT_DRAM_DEV1_ACTIVATE_CNT_LO_0, + EMC_STAT_DRAM_DEV1_READ_CNT_LO_0, + EMC_STAT_DRAM_DEV1_WRITE_CNT_LO_0, + EMC_STAT_DRAM_DEV1_REF_CNT_LO_0, + EMC_STAT_DRAM_DEV1_CUMM_BANKS_ACTIVE_CKE_EQ1_LO_0, + EMC_STAT_DRAM_DEV1_CUMM_BANKS_ACTIVE_CKE_EQ0_LO_0, + EMC_STAT_DRAM_DEV1_CKE_EQ1_CLKS_LO_0, + EMC_STAT_DRAM_DEV1_EXTCLKS_CKE_EQ1_LO_0, + EMC_STAT_DRAM_DEV1_EXTCLKS_CKE_EQ0_LO_0, + EMC_STAT_DRAM_DEV1_NO_BANKS_ACTIVE_CKE_EQ1_LO_0, + EMC_STAT_DRAM_DEV1_NO_BANKS_ACTIVE_CKE_EQ0_LO_0, + }; + int dev1_offsets_hi[] = { + EMC_STAT_DRAM_DEV1_ACTIVATE_CNT_HI_0, + EMC_STAT_DRAM_DEV1_READ_CNT_HI_0, + EMC_STAT_DRAM_DEV1_WRITE_CNT_HI_0, + EMC_STAT_DRAM_DEV1_REF_CNT_HI_0, + EMC_STAT_DRAM_DEV1_CUMM_BANKS_ACTIVE_CKE_EQ1_HI_0, + EMC_STAT_DRAM_DEV1_CUMM_BANKS_ACTIVE_CKE_EQ0_HI_0, + EMC_STAT_DRAM_DEV1_CKE_EQ1_CLKS_HI_0, + EMC_STAT_DRAM_DEV1_EXTCLKS_CKE_EQ1_HI_0, + EMC_STAT_DRAM_DEV1_EXTCLKS_CKE_EQ0_HI_0, + EMC_STAT_DRAM_DEV1_NO_BANKS_ACTIVE_CKE_EQ1_HI_0, + EMC_STAT_DRAM_DEV1_NO_BANKS_ACTIVE_CKE_EQ0_HI_0, + }; + + /* Disable statistics */ + llmc_stat |= (EMC_STAT_CONTROL_0_LLMC_GATHER_DISABLE << + EMC_STAT_CONTROL_0_LLMC_GATHER_SHIFT); + llmc_stat |= (EMC_STAT_CONTROL_0_DRAM_GATHER_DISABLE << + EMC_STAT_CONTROL_0_DRAM_GATHER_SHIFT); + writel(llmc_stat, emc.mmio + EMC_STAT_CONTROL_0); + + if (tegra_mc_client_0_enabled == true) + llp_counter->value = readl(emc.mmio + EMC_STAT_LLMC_COUNT_0_0); + + for (i = 0; i < EMC_DRAM_STAT_END - EMC_DRAM_STAT_BEGIN; i++) { + if (dram_counter[i].enabled) { + dram_counter[i].value = 0; + if (!(dram_counter[i].device_mask & 0x1)) { + if (readl(emc.mmio + dev0_offsets_hi[i]) != 0) { + dram_counter[i].value = 0xFFFFFFFF; + continue; + } + dram_counter[i].value += + readl(emc.mmio + dev0_offsets_lo[i]); + } + if (!(dram_counter[i].device_mask & 0x2)) { + if (readl(emc.mmio + dev1_offsets_hi[i]) != 0) { + dram_counter[i].value = 0xFFFFFFFF; + continue; + } + dram_counter[i].value += + readl(emc.mmio + dev1_offsets_lo[i]); + } + } + } +} + +static void stat_reschedule(tegra_mc_counter_t *counter0, + tegra_mc_counter_t *counter1, + tegra_mc_counter_t *llp) +{ + int i; + struct tegra_mc_counter *counters[] = { + counter0, + counter1, + llp + }; + + if (!tegra_mc_client_0_enabled) + return; + + for (i = 0; i < ARRAY_SIZE(counters); i++) { + struct tegra_mc_counter *c = counters[i]; + + c->address_range_change = false; + if (!c->enabled || !c->reschedule) + continue; + + c->sample_count++; + + if (c->sample_count < c->period) + continue; + + c->sample_count = 0; + + if (c->mode == FILTER_CLIENT) { + c->current_client++; + if (c->current_client == c->client_number) + c->current_client = 0; + continue; + } + + c->address_range_change = true; + c->current_address_low = c->current_address_high+1; + + if (c->current_address_low >= c->address_low+c->address_length_1) + c->current_address_low = c->address_low; + + c->current_address_high = c->current_address_low + + c->address_window_size_1; + } +} + +static void stat_start(void) +{ + mc_stat_start(&mc_counter0, &mc_counter1); + emc_stat_start(&emc_llp_counter, dram_counters); +} + +static void stat_stop(void) +{ + mc_stat_stop(&mc_counter0, &mc_counter1); + emc_stat_stop(&emc_llp_counter, dram_counters); +} + +static size_t stat_log_counter(struct tegra_mc_counter *c, + struct tegra_mc_counter *l, log_event_t *e, u32* value) +{ + size_t size = 0; + + *value = c->value; + if (l) + *value += l->value; + + if (!c->enabled || (l && !l->enabled) || !*value) + return 0; + + e->word0.enabled = 1; + e->word0.address_range_change = c->address_range_change; + e->word0.event_id = (l) ? MC_STAT_AGGREGATE : + c->clients[c->current_client]; + e->word0.address_range_low_pfn = __phys_to_pfn(c->current_address_low); + size += sizeof(e->word0); + + if (c->address_range_change) { + e->word1.address_range_length_pfn = + __phys_to_pfn(c->address_window_size_1+1); + size += sizeof(e->word1); + } + + size += sizeof(*value); + return size; +} + +#define statcpy(_buf, _bufstart, _buflen, _elem) \ + do { \ + size_t s = sizeof(_elem); \ + memcpy(_buf, &_elem, s); \ + _buf += s; \ + if (_buf >= _bufstart + _buflen) \ + _buf = _bufstart; \ + } while (0); + +static void stat_log(void) +{ + log_header_t header = {0, 0}; + log_event_t event[LOG_EVENT_NUMBER_MAX]; + u32 value[LOG_EVENT_NUMBER_MAX]; + int i, count = 0; + unsigned long flags; + size_t elem; + int required_log_size = 0; + + required_log_size += sizeof(header); + + if (tegra_mc_client_0_enabled) { + elem = stat_log_counter(&mc_counter0, NULL, &event[count], + &value[count]); + if (elem) { + required_log_size += elem; + count++; + } + + elem = stat_log_counter(&mc_counter1, &emc_llp_counter, + &event[count], &value[count]); + + if (elem) { + required_log_size += elem; + count++; + } + } + + for (i = 0; i < EMC_DRAM_STAT_END - EMC_DRAM_STAT_BEGIN && + count < LOG_EVENT_NUMBER_MAX; i++) { + if (dram_counters[i].enabled && dram_counters[i].value != 0) { + event[count].word0.enabled = 1; + event[count].word0.address_range_change = false; + event[count].word0.event_id = i + EMC_DRAM_STAT_BEGIN; + event[count].word0.address_range_low_pfn = 0; + required_log_size += sizeof(event[count].word0); + + event[count].word1.address_range_length_pfn = + 0xFFFFFFFFUL >> SHIFT_4K; + + value[count] = dram_counters[i].value; + required_log_size += sizeof(value[count]); + + count++; + } + } + + header.time_quantum = sample_quantum * MILLISECONDS_TO_TIME_QUANTUM; + for (i = 0; i < count; i++) { + header.event_state_change |= 1 << i; + } + + if (header.event_state_change != 0) { + spin_lock_irqsave(&sample_log_lock, flags); + if (unlikely(required_log_size > sample_log_size)) { + pr_err("%s: sample log too small!\n", __func__); + WARN_ON(1); + spin_unlock_irqrestore(&sample_log_lock, flags); + goto reschedule; + } + + statcpy(sample_log_wptr, sample_log, SAMPLE_LOG_SIZE, header); + + for (i=0; i