summaryrefslogtreecommitdiff
path: root/drivers/hwtracing
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/hwtracing')
-rw-r--r--drivers/hwtracing/coresight/coresight-cpu-debug.c33
-rw-r--r--drivers/hwtracing/coresight/coresight-etm4x-sysfs.c13
-rw-r--r--drivers/hwtracing/coresight/coresight-etm4x.c38
-rw-r--r--drivers/hwtracing/coresight/coresight-funnel.c11
-rw-r--r--drivers/hwtracing/coresight/coresight-priv.h10
-rw-r--r--drivers/hwtracing/coresight/coresight-replicator.c3
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc-etf.c23
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc-etr.c93
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc.c40
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc.h11
-rw-r--r--drivers/hwtracing/intel_th/Makefile3
-rw-r--r--drivers/hwtracing/intel_th/msu-sink.c116
-rw-r--r--drivers/hwtracing/intel_th/msu.c537
-rw-r--r--drivers/hwtracing/intel_th/msu.h20
14 files changed, 752 insertions, 199 deletions
diff --git a/drivers/hwtracing/coresight/coresight-cpu-debug.c b/drivers/hwtracing/coresight/coresight-cpu-debug.c
index 2463aa7ab4f6..96544b348c27 100644
--- a/drivers/hwtracing/coresight/coresight-cpu-debug.c
+++ b/drivers/hwtracing/coresight/coresight-cpu-debug.c
@@ -646,24 +646,23 @@ static int debug_remove(struct amba_device *adev)
return 0;
}
+static const struct amba_cs_uci_id uci_id_debug[] = {
+ {
+ /* CPU Debug UCI data */
+ .devarch = 0x47706a15,
+ .devarch_mask = 0xfff0ffff,
+ .devtype = 0x00000015,
+ }
+};
+
static const struct amba_id debug_ids[] = {
- { /* Debug for Cortex-A53 */
- .id = 0x000bbd03,
- .mask = 0x000fffff,
- },
- { /* Debug for Cortex-A57 */
- .id = 0x000bbd07,
- .mask = 0x000fffff,
- },
- { /* Debug for Cortex-A72 */
- .id = 0x000bbd08,
- .mask = 0x000fffff,
- },
- { /* Debug for Cortex-A73 */
- .id = 0x000bbd09,
- .mask = 0x000fffff,
- },
- { 0, 0 },
+ CS_AMBA_ID(0x000bbd03), /* Cortex-A53 */
+ CS_AMBA_ID(0x000bbd07), /* Cortex-A57 */
+ CS_AMBA_ID(0x000bbd08), /* Cortex-A72 */
+ CS_AMBA_ID(0x000bbd09), /* Cortex-A73 */
+ CS_AMBA_UCI_ID(0x000f0205, uci_id_debug), /* Qualcomm Kryo */
+ CS_AMBA_UCI_ID(0x000f0211, uci_id_debug), /* Qualcomm Kryo */
+ {},
};
static struct amba_driver debug_driver = {
diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
index a0365e23678e..219c10eb752c 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
@@ -296,11 +296,8 @@ static ssize_t mode_store(struct device *dev,
spin_lock(&drvdata->spinlock);
config->mode = val & ETMv4_MODE_ALL;
-
- if (config->mode & ETM_MODE_EXCLUDE)
- etm4_set_mode_exclude(drvdata, true);
- else
- etm4_set_mode_exclude(drvdata, false);
+ etm4_set_mode_exclude(drvdata,
+ config->mode & ETM_MODE_EXCLUDE ? true : false);
if (drvdata->instrp0 == true) {
/* start by clearing instruction P0 field */
@@ -999,10 +996,8 @@ static ssize_t addr_range_store(struct device *dev,
* Program include or exclude control bits for vinst or vdata
* whenever we change addr comparators to ETM_ADDR_TYPE_RANGE
*/
- if (config->mode & ETM_MODE_EXCLUDE)
- etm4_set_mode_exclude(drvdata, true);
- else
- etm4_set_mode_exclude(drvdata, false);
+ etm4_set_mode_exclude(drvdata,
+ config->mode & ETM_MODE_EXCLUDE ? true : false);
spin_unlock(&drvdata->spinlock);
return size;
diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
index 7bcac8896fc1..a128b5063f46 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x.c
@@ -34,7 +34,8 @@
#include "coresight-etm-perf.h"
static int boot_enable;
-module_param_named(boot_enable, boot_enable, int, S_IRUGO);
+module_param(boot_enable, int, 0444);
+MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
/* The number of ETMv4 currently registered */
static int etm4_count;
@@ -47,7 +48,7 @@ static enum cpuhp_state hp_online;
static void etm4_os_unlock(struct etmv4_drvdata *drvdata)
{
- /* Writing any value to ETMOSLAR unlocks the trace registers */
+ /* Writing 0 to TRCOSLAR unlocks the trace registers */
writel_relaxed(0x0, drvdata->base + TRCOSLAR);
drvdata->os_unlock = true;
isb();
@@ -188,6 +189,13 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
dev_err(etm_dev,
"timeout while waiting for Idle Trace Status\n");
+ /*
+ * As recommended by section 4.3.7 ("Synchronization when using the
+ * memory-mapped interface") of ARM IHI 0064D
+ */
+ dsb(sy);
+ isb();
+
done:
CS_LOCK(drvdata->base);
@@ -453,8 +461,12 @@ static void etm4_disable_hw(void *info)
/* EN, bit[0] Trace unit enable bit */
control &= ~0x1;
- /* make sure everything completes before disabling */
- mb();
+ /*
+ * Make sure everything completes before disabling, as recommended
+ * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register,
+ * SSTATUS") of ARM IHI 0064D
+ */
+ dsb(sy);
isb();
writel_relaxed(control, drvdata->base + TRCPRGCTLR);
@@ -1047,10 +1059,8 @@ static int etm4_starting_cpu(unsigned int cpu)
return 0;
spin_lock(&etmdrvdata[cpu]->spinlock);
- if (!etmdrvdata[cpu]->os_unlock) {
+ if (!etmdrvdata[cpu]->os_unlock)
etm4_os_unlock(etmdrvdata[cpu]);
- etmdrvdata[cpu]->os_unlock = true;
- }
if (local_read(&etmdrvdata[cpu]->mode))
etm4_enable_hw(etmdrvdata[cpu]);
@@ -1192,11 +1202,15 @@ static struct amba_cs_uci_id uci_id_etm4[] = {
};
static const struct amba_id etm4_ids[] = {
- CS_AMBA_ID(0x000bb95d), /* Cortex-A53 */
- CS_AMBA_ID(0x000bb95e), /* Cortex-A57 */
- CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */
- CS_AMBA_ID(0x000bb959), /* Cortex-A73 */
- CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4), /* Cortex-A35 */
+ CS_AMBA_ID(0x000bb95d), /* Cortex-A53 */
+ CS_AMBA_ID(0x000bb95e), /* Cortex-A57 */
+ CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */
+ CS_AMBA_ID(0x000bb959), /* Cortex-A73 */
+ CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */
+ CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */
+ CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */
+ CS_AMBA_ID(0x000bb802), /* Qualcomm Kryo 385 Cortex-A55 */
+ CS_AMBA_ID(0x000bb803), /* Qualcomm Kryo 385 Cortex-A75 */
{},
};
diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c
index fa97cb9ab4f9..05f7896c3a01 100644
--- a/drivers/hwtracing/coresight/coresight-funnel.c
+++ b/drivers/hwtracing/coresight/coresight-funnel.c
@@ -5,6 +5,7 @@
* Description: CoreSight Funnel driver
*/
+#include <linux/acpi.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/types.h>
@@ -192,7 +193,7 @@ static int funnel_probe(struct device *dev, struct resource *res)
if (is_of_node(dev_fwnode(dev)) &&
of_device_is_compatible(dev->of_node, "arm,coresight-funnel"))
- pr_warn_once("Uses OBSOLETE CoreSight funnel binding\n");
+ dev_warn_once(dev, "Uses OBSOLETE CoreSight funnel binding\n");
desc.name = coresight_alloc_device_name(&funnel_devs, dev);
if (!desc.name)
@@ -302,11 +303,19 @@ static const struct of_device_id static_funnel_match[] = {
{}
};
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id static_funnel_ids[] = {
+ {"ARMHC9FE", 0},
+ {},
+};
+#endif
+
static struct platform_driver static_funnel_driver = {
.probe = static_funnel_probe,
.driver = {
.name = "coresight-static-funnel",
.of_match_table = static_funnel_match,
+ .acpi_match_table = ACPI_PTR(static_funnel_ids),
.pm = &funnel_dev_pm_ops,
.suppress_bind_attrs = true,
},
diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h
index 61d7f9ff054d..82e563cdc879 100644
--- a/drivers/hwtracing/coresight/coresight-priv.h
+++ b/drivers/hwtracing/coresight/coresight-priv.h
@@ -185,11 +185,11 @@ static inline int etm_writel_cp14(u32 off, u32 val) { return 0; }
}
/* coresight AMBA ID, full UCI structure: id table entry. */
-#define CS_AMBA_UCI_ID(pid, uci_ptr) \
- { \
- .id = pid, \
- .mask = 0x000fffff, \
- .data = uci_ptr \
+#define CS_AMBA_UCI_ID(pid, uci_ptr) \
+ { \
+ .id = pid, \
+ .mask = 0x000fffff, \
+ .data = (void *)uci_ptr \
}
/* extract the data value from a UCI structure given amba_id pointer. */
diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c
index b7d6d59d56db..b29ba640eb25 100644
--- a/drivers/hwtracing/coresight/coresight-replicator.c
+++ b/drivers/hwtracing/coresight/coresight-replicator.c
@@ -184,7 +184,8 @@ static int replicator_probe(struct device *dev, struct resource *res)
if (is_of_node(dev_fwnode(dev)) &&
of_device_is_compatible(dev->of_node, "arm,coresight-replicator"))
- pr_warn_once("Uses OBSOLETE CoreSight replicator binding\n");
+ dev_warn_once(dev,
+ "Uses OBSOLETE CoreSight replicator binding\n");
desc.name = coresight_alloc_device_name(&replicator_devs, dev);
if (!desc.name)
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c
index 23b7ff00af5c..807416b75ecc 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etf.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c
@@ -479,30 +479,11 @@ static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,
* traces.
*/
if (!buf->snapshot && to_read > handle->size) {
- u32 mask = 0;
-
- /*
- * The value written to RRP must be byte-address aligned to
- * the width of the trace memory databus _and_ to a frame
- * boundary (16 byte), whichever is the biggest. For example,
- * for 32-bit, 64-bit and 128-bit wide trace memory, the four
- * LSBs must be 0s. For 256-bit wide trace memory, the five
- * LSBs must be 0s.
- */
- switch (drvdata->memwidth) {
- case TMC_MEM_INTF_WIDTH_32BITS:
- case TMC_MEM_INTF_WIDTH_64BITS:
- case TMC_MEM_INTF_WIDTH_128BITS:
- mask = GENMASK(31, 4);
- break;
- case TMC_MEM_INTF_WIDTH_256BITS:
- mask = GENMASK(31, 5);
- break;
- }
+ u32 mask = tmc_get_memwidth_mask(drvdata);
/*
* Make sure the new size is aligned in accordance with the
- * requirement explained above.
+ * requirement explained in function tmc_get_memwidth_mask().
*/
to_read = handle->size & mask;
/* Move the RAM read pointer up */
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c
index 17006705287a..625882bc8b08 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etr.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c
@@ -871,6 +871,7 @@ static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
return ERR_PTR(rc);
}
+ refcount_set(&etr_buf->refcount, 1);
dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
(unsigned long)size >> 10, etr_buf->mode);
return etr_buf;
@@ -927,15 +928,24 @@ static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
rrp = tmc_read_rrp(drvdata);
rwp = tmc_read_rwp(drvdata);
status = readl_relaxed(drvdata->base + TMC_STS);
+
+ /*
+ * If there were memory errors in the session, truncate the
+ * buffer.
+ */
+ if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
+ dev_dbg(&drvdata->csdev->dev,
+ "tmc memory error detected, truncating buffer\n");
+ etr_buf->len = 0;
+ etr_buf->full = 0;
+ return;
+ }
+
etr_buf->full = status & TMC_STS_FULL;
WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
etr_buf->ops->sync(etr_buf, rrp, rwp);
-
- /* Insert barrier packets at the beginning, if there was an overflow */
- if (etr_buf->full)
- tmc_etr_buf_insert_barrier_packet(etr_buf, etr_buf->offset);
}
static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
@@ -1072,6 +1082,13 @@ static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
drvdata->sysfs_buf = NULL;
} else {
tmc_sync_etr_buf(drvdata);
+ /*
+ * Insert barrier packets at the beginning, if there was
+ * an overflow.
+ */
+ if (etr_buf->full)
+ tmc_etr_buf_insert_barrier_packet(etr_buf,
+ etr_buf->offset);
}
}
@@ -1263,8 +1280,6 @@ retry:
if (IS_ERR(etr_buf))
return etr_buf;
- refcount_set(&etr_buf->refcount, 1);
-
/* Now that we have a buffer, add it to the IDR. */
mutex_lock(&drvdata->idr_mutex);
ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
@@ -1291,19 +1306,11 @@ get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
struct perf_event *event, int nr_pages,
void **pages, bool snapshot)
{
- struct etr_buf *etr_buf;
-
/*
* In per-thread mode the etr_buf isn't shared, so just go ahead
* with memory allocation.
*/
- etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
- if (IS_ERR(etr_buf))
- goto out;
-
- refcount_set(&etr_buf->refcount, 1);
-out:
- return etr_buf;
+ return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
}
static struct etr_buf *
@@ -1410,10 +1417,12 @@ free_etr_perf_buffer:
* tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
* buffer to the perf ring buffer.
*/
-static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf)
+static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
+ unsigned long src_offset,
+ unsigned long to_copy)
{
- long bytes, to_copy;
- long pg_idx, pg_offset, src_offset;
+ long bytes;
+ long pg_idx, pg_offset;
unsigned long head = etr_perf->head;
char **dst_pages, *src_buf;
struct etr_buf *etr_buf = etr_perf->etr_buf;
@@ -1422,8 +1431,6 @@ static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf)
pg_idx = head >> PAGE_SHIFT;
pg_offset = head & (PAGE_SIZE - 1);
dst_pages = (char **)etr_perf->pages;
- src_offset = etr_buf->offset;
- to_copy = etr_buf->len;
while (to_copy > 0) {
/*
@@ -1434,6 +1441,8 @@ static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf)
* 3) what is available in the destination page.
* in one iteration.
*/
+ if (src_offset >= etr_buf->size)
+ src_offset -= etr_buf->size;
bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
&src_buf);
if (WARN_ON_ONCE(bytes <= 0))
@@ -1454,8 +1463,6 @@ static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf)
/* Move source pointers */
src_offset += bytes;
- if (src_offset >= etr_buf->size)
- src_offset -= etr_buf->size;
}
}
@@ -1471,7 +1478,7 @@ tmc_update_etr_buffer(struct coresight_device *csdev,
void *config)
{
bool lost = false;
- unsigned long flags, size = 0;
+ unsigned long flags, offset, size = 0;
struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
struct etr_perf_buffer *etr_perf = config;
struct etr_buf *etr_buf = etr_perf->etr_buf;
@@ -1484,7 +1491,7 @@ tmc_update_etr_buffer(struct coresight_device *csdev,
goto out;
}
- if (WARN_ON(drvdata->perf_data != etr_perf)) {
+ if (WARN_ON(drvdata->perf_buf != etr_buf)) {
lost = true;
spin_unlock_irqrestore(&drvdata->spinlock, flags);
goto out;
@@ -1496,12 +1503,38 @@ tmc_update_etr_buffer(struct coresight_device *csdev,
tmc_sync_etr_buf(drvdata);
CS_LOCK(drvdata->base);
- /* Reset perf specific data */
- drvdata->perf_data = NULL;
spin_unlock_irqrestore(&drvdata->spinlock, flags);
+ lost = etr_buf->full;
+ offset = etr_buf->offset;
size = etr_buf->len;
- tmc_etr_sync_perf_buffer(etr_perf);
+
+ /*
+ * The ETR buffer may be bigger than the space available in the
+ * perf ring buffer (handle->size). If so advance the offset so that we
+ * get the latest trace data. In snapshot mode none of that matters
+ * since we are expected to clobber stale data in favour of the latest
+ * traces.
+ */
+ if (!etr_perf->snapshot && size > handle->size) {
+ u32 mask = tmc_get_memwidth_mask(drvdata);
+
+ /*
+ * Make sure the new size is aligned in accordance with the
+ * requirement explained in function tmc_get_memwidth_mask().
+ */
+ size = handle->size & mask;
+ offset = etr_buf->offset + etr_buf->len - size;
+
+ if (offset >= etr_buf->size)
+ offset -= etr_buf->size;
+ lost = true;
+ }
+
+ /* Insert barrier packets at the beginning, if there was an overflow */
+ if (lost)
+ tmc_etr_buf_insert_barrier_packet(etr_buf, etr_buf->offset);
+ tmc_etr_sync_perf_buffer(etr_perf, offset, size);
/*
* In snapshot mode we simply increment the head by the number of byte
@@ -1511,8 +1544,6 @@ tmc_update_etr_buffer(struct coresight_device *csdev,
*/
if (etr_perf->snapshot)
handle->head += size;
-
- lost |= etr_buf->full;
out:
/*
* Don't set the TRUNCATED flag in snapshot mode because 1) the
@@ -1556,7 +1587,6 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
}
etr_perf->head = PERF_IDX2OFF(handle->head, etr_perf);
- drvdata->perf_data = etr_perf;
/*
* No HW configuration is needed if the sink is already in
@@ -1572,6 +1602,7 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
/* Associate with monitored process. */
drvdata->pid = pid;
drvdata->mode = CS_MODE_PERF;
+ drvdata->perf_buf = etr_perf->etr_buf;
atomic_inc(csdev->refcnt);
}
@@ -1617,6 +1648,8 @@ static int tmc_disable_etr_sink(struct coresight_device *csdev)
/* Dissociate from monitored process. */
drvdata->pid = -1;
drvdata->mode = CS_MODE_DISABLED;
+ /* Reset perf specific data */
+ drvdata->perf_buf = NULL;
spin_unlock_irqrestore(&drvdata->spinlock, flags);
diff --git a/drivers/hwtracing/coresight/coresight-tmc.c b/drivers/hwtracing/coresight/coresight-tmc.c
index be37aff573b4..1cf82fa58289 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.c
+++ b/drivers/hwtracing/coresight/coresight-tmc.c
@@ -70,6 +70,34 @@ void tmc_disable_hw(struct tmc_drvdata *drvdata)
writel_relaxed(0x0, drvdata->base + TMC_CTL);
}
+u32 tmc_get_memwidth_mask(struct tmc_drvdata *drvdata)
+{
+ u32 mask = 0;
+
+ /*
+ * When moving RRP or an offset address forward, the new values must
+ * be byte-address aligned to the width of the trace memory databus
+ * _and_ to a frame boundary (16 byte), whichever is the biggest. For
+ * example, for 32-bit, 64-bit and 128-bit wide trace memory, the four
+ * LSBs must be 0s. For 256-bit wide trace memory, the five LSBs must
+ * be 0s.
+ */
+ switch (drvdata->memwidth) {
+ case TMC_MEM_INTF_WIDTH_32BITS:
+ /* fallthrough */
+ case TMC_MEM_INTF_WIDTH_64BITS:
+ /* fallthrough */
+ case TMC_MEM_INTF_WIDTH_128BITS:
+ mask = GENMASK(31, 4);
+ break;
+ case TMC_MEM_INTF_WIDTH_256BITS:
+ mask = GENMASK(31, 5);
+ break;
+ }
+
+ return mask;
+}
+
static int tmc_read_prepare(struct tmc_drvdata *drvdata)
{
int ret = 0;
@@ -236,6 +264,7 @@ coresight_tmc_reg(ffcr, TMC_FFCR);
coresight_tmc_reg(mode, TMC_MODE);
coresight_tmc_reg(pscr, TMC_PSCR);
coresight_tmc_reg(axictl, TMC_AXICTL);
+coresight_tmc_reg(authstatus, TMC_AUTHSTATUS);
coresight_tmc_reg(devid, CORESIGHT_DEVID);
coresight_tmc_reg64(rrp, TMC_RRP, TMC_RRPHI);
coresight_tmc_reg64(rwp, TMC_RWP, TMC_RWPHI);
@@ -255,6 +284,7 @@ static struct attribute *coresight_tmc_mgmt_attrs[] = {
&dev_attr_devid.attr,
&dev_attr_dba.attr,
&dev_attr_axictl.attr,
+ &dev_attr_authstatus.attr,
NULL,
};
@@ -342,6 +372,13 @@ static inline bool tmc_etr_can_use_sg(struct device *dev)
return fwnode_property_present(dev->fwnode, "arm,scatter-gather");
}
+static inline bool tmc_etr_has_non_secure_access(struct tmc_drvdata *drvdata)
+{
+ u32 auth = readl_relaxed(drvdata->base + TMC_AUTHSTATUS);
+
+ return (auth & TMC_AUTH_NSID_MASK) == 0x3;
+}
+
/* Detect and initialise the capabilities of a TMC ETR */
static int tmc_etr_setup_caps(struct device *parent, u32 devid, void *dev_caps)
{
@@ -349,6 +386,9 @@ static int tmc_etr_setup_caps(struct device *parent, u32 devid, void *dev_caps)
u32 dma_mask = 0;
struct tmc_drvdata *drvdata = dev_get_drvdata(parent);
+ if (!tmc_etr_has_non_secure_access(drvdata))
+ return -EACCES;
+
/* Set the unadvertised capabilities */
tmc_etr_init_caps(drvdata, (u32)(unsigned long)dev_caps);
diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h
index 1ed50411cc3c..71de978575f3 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.h
+++ b/drivers/hwtracing/coresight/coresight-tmc.h
@@ -39,6 +39,7 @@
#define TMC_ITATBCTR2 0xef0
#define TMC_ITATBCTR1 0xef4
#define TMC_ITATBCTR0 0xef8
+#define TMC_AUTHSTATUS 0xfb8
/* register description */
/* TMC_CTL - 0x020 */
@@ -47,6 +48,7 @@
#define TMC_STS_TMCREADY_BIT 2
#define TMC_STS_FULL BIT(0)
#define TMC_STS_TRIGGERED BIT(1)
+#define TMC_STS_MEMERR BIT(5)
/*
* TMC_AXICTL - 0x110
*
@@ -89,6 +91,8 @@
#define TMC_DEVID_AXIAW_SHIFT 17
#define TMC_DEVID_AXIAW_MASK 0x7f
+#define TMC_AUTH_NSID_MASK GENMASK(1, 0)
+
enum tmc_config_type {
TMC_CONFIG_TYPE_ETB,
TMC_CONFIG_TYPE_ETR,
@@ -178,8 +182,8 @@ struct etr_buf {
* device configuration register (DEVID)
* @idr: Holds etr_bufs allocated for this ETR.
* @idr_mutex: Access serialisation for idr.
- * @perf_data: PERF buffer for ETR.
- * @sysfs_data: SYSFS buffer for ETR.
+ * @sysfs_buf: SYSFS buffer for ETR.
+ * @perf_buf: PERF buffer for ETR.
*/
struct tmc_drvdata {
void __iomem *base;
@@ -202,7 +206,7 @@ struct tmc_drvdata {
struct idr idr;
struct mutex idr_mutex;
struct etr_buf *sysfs_buf;
- void *perf_data;
+ struct etr_buf *perf_buf;
};
struct etr_buf_operations {
@@ -251,6 +255,7 @@ void tmc_wait_for_tmcready(struct tmc_drvdata *drvdata);
void tmc_flush_and_stop(struct tmc_drvdata *drvdata);
void tmc_enable_hw(struct tmc_drvdata *drvdata);
void tmc_disable_hw(struct tmc_drvdata *drvdata);
+u32 tmc_get_memwidth_mask(struct tmc_drvdata *drvdata);
/* ETB/ETF functions */
int tmc_read_prepare_etb(struct tmc_drvdata *drvdata);
diff --git a/drivers/hwtracing/intel_th/Makefile b/drivers/hwtracing/intel_th/Makefile
index d9252fa8d9ca..b63eb8f309ad 100644
--- a/drivers/hwtracing/intel_th/Makefile
+++ b/drivers/hwtracing/intel_th/Makefile
@@ -20,3 +20,6 @@ intel_th_msu-y := msu.o
obj-$(CONFIG_INTEL_TH_PTI) += intel_th_pti.o
intel_th_pti-y := pti.o
+
+obj-$(CONFIG_INTEL_TH_MSU) += intel_th_msu_sink.o
+intel_th_msu_sink-y := msu-sink.o
diff --git a/drivers/hwtracing/intel_th/msu-sink.c b/drivers/hwtracing/intel_th/msu-sink.c
new file mode 100644
index 000000000000..2c7f5116be12
--- /dev/null
+++ b/drivers/hwtracing/intel_th/msu-sink.c
@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * An example software sink buffer for Intel TH MSU.
+ *
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include <linux/intel_th.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+
+#define MAX_SGTS 16
+
+struct msu_sink_private {
+ struct device *dev;
+ struct sg_table **sgts;
+ unsigned int nr_sgts;
+};
+
+static void *msu_sink_assign(struct device *dev, int *mode)
+{
+ struct msu_sink_private *priv;
+
+ priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return NULL;
+
+ priv->sgts = kcalloc(MAX_SGTS, sizeof(void *), GFP_KERNEL);
+ if (!priv->sgts) {
+ kfree(priv);
+ return NULL;
+ }
+
+ priv->dev = dev;
+ *mode = MSC_MODE_MULTI;
+
+ return priv;
+}
+
+static void msu_sink_unassign(void *data)
+{
+ struct msu_sink_private *priv = data;
+
+ kfree(priv->sgts);
+ kfree(priv);
+}
+
+/* See also: msc.c: __msc_buffer_win_alloc() */
+static int msu_sink_alloc_window(void *data, struct sg_table **sgt, size_t size)
+{
+ struct msu_sink_private *priv = data;
+ unsigned int nents;
+ struct scatterlist *sg_ptr;
+ void *block;
+ int ret, i;
+
+ if (priv->nr_sgts == MAX_SGTS)
+ return -ENOMEM;
+
+ nents = DIV_ROUND_UP(size, PAGE_SIZE);
+
+ ret = sg_alloc_table(*sgt, nents, GFP_KERNEL);
+ if (ret)
+ return -ENOMEM;
+
+ priv->sgts[priv->nr_sgts++] = *sgt;
+
+ for_each_sg((*sgt)->sgl, sg_ptr, nents, i) {
+ block = dma_alloc_coherent(priv->dev->parent->parent,
+ PAGE_SIZE, &sg_dma_address(sg_ptr),
+ GFP_KERNEL);
+ sg_set_buf(sg_ptr, block, PAGE_SIZE);
+ }
+
+ return nents;
+}
+
+/* See also: msc.c: __msc_buffer_win_free() */
+static void msu_sink_free_window(void *data, struct sg_table *sgt)
+{
+ struct msu_sink_private *priv = data;
+ struct scatterlist *sg_ptr;
+ int i;
+
+ for_each_sg(sgt->sgl, sg_ptr, sgt->nents, i) {
+ dma_free_coherent(priv->dev->parent->parent, PAGE_SIZE,
+ sg_virt(sg_ptr), sg_dma_address(sg_ptr));
+ }
+
+ sg_free_table(sgt);
+ priv->nr_sgts--;
+}
+
+static int msu_sink_ready(void *data, struct sg_table *sgt, size_t bytes)
+{
+ struct msu_sink_private *priv = data;
+
+ intel_th_msc_window_unlock(priv->dev, sgt);
+
+ return 0;
+}
+
+static const struct msu_buffer sink_mbuf = {
+ .name = "sink",
+ .assign = msu_sink_assign,
+ .unassign = msu_sink_unassign,
+ .alloc_window = msu_sink_alloc_window,
+ .free_window = msu_sink_free_window,
+ .ready = msu_sink_ready,
+};
+
+module_intel_th_msu_buffer(sink_mbuf);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/hwtracing/intel_th/msu.c b/drivers/hwtracing/intel_th/msu.c
index 8ab28e5fb366..fc9f15f36ad4 100644
--- a/drivers/hwtracing/intel_th/msu.c
+++ b/drivers/hwtracing/intel_th/msu.c
@@ -17,21 +17,48 @@
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/io.h>
+#include <linux/workqueue.h>
#include <linux/dma-mapping.h>
#ifdef CONFIG_X86
#include <asm/set_memory.h>
#endif
+#include <linux/intel_th.h>
#include "intel_th.h"
#include "msu.h"
#define msc_dev(x) (&(x)->thdev->dev)
+/*
+ * Lockout state transitions:
+ * READY -> INUSE -+-> LOCKED -+-> READY -> etc.
+ * \-----------/
+ * WIN_READY: window can be used by HW
+ * WIN_INUSE: window is in use
+ * WIN_LOCKED: window is filled up and is being processed by the buffer
+ * handling code
+ *
+ * All state transitions happen automatically, except for the LOCKED->READY,
+ * which needs to be signalled by the buffer code by calling
+ * intel_th_msc_window_unlock().
+ *
+ * When the interrupt handler has to switch to the next window, it checks
+ * whether it's READY, and if it is, it performs the switch and tracing
+ * continues. If it's LOCKED, it stops the trace.
+ */
+enum lockout_state {
+ WIN_READY = 0,
+ WIN_INUSE,
+ WIN_LOCKED
+};
+
/**
* struct msc_window - multiblock mode window descriptor
* @entry: window list linkage (msc::win_list)
* @pgoff: page offset into the buffer that this window starts at
+ * @lockout: lockout state, see comment below
+ * @lo_lock: lockout state serialization
* @nr_blocks: number of blocks (pages) in this window
* @nr_segs: number of segments in this window (<= @nr_blocks)
* @_sgt: array of block descriptors
@@ -40,6 +67,8 @@
struct msc_window {
struct list_head entry;
unsigned long pgoff;
+ enum lockout_state lockout;
+ spinlock_t lo_lock;
unsigned int nr_blocks;
unsigned int nr_segs;
struct msc *msc;
@@ -66,8 +95,8 @@ struct msc_iter {
struct msc_window *start_win;
struct msc_window *win;
unsigned long offset;
- int start_block;
- int block;
+ struct scatterlist *start_block;
+ struct scatterlist *block;
unsigned int block_off;
unsigned int wrap_count;
unsigned int eof;
@@ -77,6 +106,8 @@ struct msc_iter {
* struct msc - MSC device representation
* @reg_base: register window base address
* @thdev: intel_th_device pointer
+ * @mbuf: MSU buffer, if assigned
+ * @mbuf_priv MSU buffer's private data, if @mbuf
* @win_list: list of windows in multiblock mode
* @single_sgt: single mode buffer
* @cur_win: current window
@@ -100,6 +131,10 @@ struct msc {
void __iomem *msu_base;
struct intel_th_device *thdev;
+ const struct msu_buffer *mbuf;
+ void *mbuf_priv;
+
+ struct work_struct work;
struct list_head win_list;
struct sg_table single_sgt;
struct msc_window *cur_win;
@@ -108,6 +143,8 @@ struct msc {
unsigned int single_wrap : 1;
void *base;
dma_addr_t base_addr;
+ u32 orig_addr;
+ u32 orig_sz;
/* <0: no buffer, 0: no users, >0: active users */
atomic_t user_count;
@@ -126,6 +163,101 @@ struct msc {
unsigned int index;
};
+static LIST_HEAD(msu_buffer_list);
+static struct mutex msu_buffer_mutex;
+
+/**
+ * struct msu_buffer_entry - internal MSU buffer bookkeeping
+ * @entry: link to msu_buffer_list
+ * @mbuf: MSU buffer object
+ * @owner: module that provides this MSU buffer
+ */
+struct msu_buffer_entry {
+ struct list_head entry;
+ const struct msu_buffer *mbuf;
+ struct module *owner;
+};
+
+static struct msu_buffer_entry *__msu_buffer_entry_find(const char *name)
+{
+ struct msu_buffer_entry *mbe;
+
+ lockdep_assert_held(&msu_buffer_mutex);
+
+ list_for_each_entry(mbe, &msu_buffer_list, entry) {
+ if (!strcmp(mbe->mbuf->name, name))
+ return mbe;
+ }
+
+ return NULL;
+}
+
+static const struct msu_buffer *
+msu_buffer_get(const char *name)
+{
+ struct msu_buffer_entry *mbe;
+
+ mutex_lock(&msu_buffer_mutex);
+ mbe = __msu_buffer_entry_find(name);
+ if (mbe && !try_module_get(mbe->owner))
+ mbe = NULL;
+ mutex_unlock(&msu_buffer_mutex);
+
+ return mbe ? mbe->mbuf : NULL;
+}
+
+static void msu_buffer_put(const struct msu_buffer *mbuf)
+{
+ struct msu_buffer_entry *mbe;
+
+ mutex_lock(&msu_buffer_mutex);
+ mbe = __msu_buffer_entry_find(mbuf->name);
+ if (mbe)
+ module_put(mbe->owner);
+ mutex_unlock(&msu_buffer_mutex);
+}
+
+int intel_th_msu_buffer_register(const struct msu_buffer *mbuf,
+ struct module *owner)
+{
+ struct msu_buffer_entry *mbe;
+ int ret = 0;
+
+ mbe = kzalloc(sizeof(*mbe), GFP_KERNEL);
+ if (!mbe)
+ return -ENOMEM;
+
+ mutex_lock(&msu_buffer_mutex);
+ if (__msu_buffer_entry_find(mbuf->name)) {
+ ret = -EEXIST;
+ kfree(mbe);
+ goto unlock;
+ }
+
+ mbe->mbuf = mbuf;
+ mbe->owner = owner;
+ list_add_tail(&mbe->entry, &msu_buffer_list);
+unlock:
+ mutex_unlock(&msu_buffer_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(intel_th_msu_buffer_register);
+
+void intel_th_msu_buffer_unregister(const struct msu_buffer *mbuf)
+{
+ struct msu_buffer_entry *mbe;
+
+ mutex_lock(&msu_buffer_mutex);
+ mbe = __msu_buffer_entry_find(mbuf->name);
+ if (mbe) {
+ list_del(&mbe->entry);
+ kfree(mbe);
+ }
+ mutex_unlock(&msu_buffer_mutex);
+}
+EXPORT_SYMBOL_GPL(intel_th_msu_buffer_unregister);
+
static inline bool msc_block_is_empty(struct msc_block_desc *bdesc)
{
/* header hasn't been written */
@@ -139,28 +271,25 @@ static inline bool msc_block_is_empty(struct msc_block_desc *bdesc)
return false;
}
-static inline struct msc_block_desc *
-msc_win_block(struct msc_window *win, unsigned int block)
+static inline struct scatterlist *msc_win_base_sg(struct msc_window *win)
{
- return sg_virt(&win->sgt->sgl[block]);
+ return win->sgt->sgl;
}
-static inline size_t
-msc_win_actual_bsz(struct msc_window *win, unsigned int block)
+static inline struct msc_block_desc *msc_win_base(struct msc_window *win)
{
- return win->sgt->sgl[block].length;
+ return sg_virt(msc_win_base_sg(win));
}
-static inline dma_addr_t
-msc_win_baddr(struct msc_window *win, unsigned int block)
+static inline dma_addr_t msc_win_base_dma(struct msc_window *win)
{
- return sg_dma_address(&win->sgt->sgl[block]);
+ return sg_dma_address(msc_win_base_sg(win));
}
static inline unsigned long
-msc_win_bpfn(struct msc_window *win, unsigned int block)
+msc_win_base_pfn(struct msc_window *win)
{
- return msc_win_baddr(win, block) >> PAGE_SHIFT;
+ return PFN_DOWN(msc_win_base_dma(win));
}
/**
@@ -188,6 +317,26 @@ static struct msc_window *msc_next_window(struct msc_window *win)
return list_next_entry(win, entry);
}
+static size_t msc_win_total_sz(struct msc_window *win)
+{
+ struct scatterlist *sg;
+ unsigned int blk;
+ size_t size = 0;
+
+ for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
+ struct msc_block_desc *bdesc = sg_virt(sg);
+
+ if (msc_block_wrapped(bdesc))
+ return win->nr_blocks << PAGE_SHIFT;
+
+ size += msc_total_sz(bdesc);
+ if (msc_block_last_written(bdesc))
+ break;
+ }
+
+ return size;
+}
+
/**
* msc_find_window() - find a window matching a given sg_table
* @msc: MSC device
@@ -216,7 +365,7 @@ msc_find_window(struct msc *msc, struct sg_table *sgt, bool nonempty)
found++;
/* skip the empty ones */
- if (nonempty && msc_block_is_empty(msc_win_block(win, 0)))
+ if (nonempty && msc_block_is_empty(msc_win_base(win)))
continue;
if (found)
@@ -250,44 +399,38 @@ static struct msc_window *msc_oldest_window(struct msc *msc)
}
/**
- * msc_win_oldest_block() - locate the oldest block in a given window
+ * msc_win_oldest_sg() - locate the oldest block in a given window
* @win: window to look at
*
* Return: index of the block with the oldest data
*/
-static unsigned int msc_win_oldest_block(struct msc_window *win)
+static struct scatterlist *msc_win_oldest_sg(struct msc_window *win)
{
unsigned int blk;
- struct msc_block_desc *bdesc = msc_win_block(win, 0);
+ struct scatterlist *sg;
+ struct msc_block_desc *bdesc = msc_win_base(win);
/* without wrapping, first block is the oldest */
if (!msc_block_wrapped(bdesc))
- return 0;
+ return msc_win_base_sg(win);
/*
* with wrapping, last written block contains both the newest and the
* oldest data for this window.
*/
- for (blk = 0; blk < win->nr_segs; blk++) {
- bdesc = msc_win_block(win, blk);
+ for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
+ struct msc_block_desc *bdesc = sg_virt(sg);
if (msc_block_last_written(bdesc))
- return blk;
+ return sg;
}
- return 0;
+ return msc_win_base_sg(win);
}
static struct msc_block_desc *msc_iter_bdesc(struct msc_iter *iter)
{
- return msc_win_block(iter->win, iter->block);
-}
-
-static void msc_iter_init(struct msc_iter *iter)
-{
- memset(iter, 0, sizeof(*iter));
- iter->start_block = -1;
- iter->block = -1;
+ return sg_virt(iter->block);
}
static struct msc_iter *msc_iter_install(struct msc *msc)
@@ -312,7 +455,6 @@ static struct msc_iter *msc_iter_install(struct msc *msc)
goto unlock;
}
- msc_iter_init(iter);
iter->msc = msc;
list_add_tail(&iter->entry, &msc->iter_list);
@@ -333,10 +475,10 @@ static void msc_iter_remove(struct msc_iter *iter, struct msc *msc)
static void msc_iter_block_start(struct msc_iter *iter)
{
- if (iter->start_block != -1)
+ if (iter->start_block)
return;
- iter->start_block = msc_win_oldest_block(iter->win);
+ iter->start_block = msc_win_oldest_sg(iter->win);
iter->block = iter->start_block;
iter->wrap_count = 0;
@@ -360,7 +502,7 @@ static int msc_iter_win_start(struct msc_iter *iter, struct msc *msc)
return -EINVAL;
iter->win = iter->start_win;
- iter->start_block = -1;
+ iter->start_block = NULL;
msc_iter_block_start(iter);
@@ -370,7 +512,7 @@ static int msc_iter_win_start(struct msc_iter *iter, struct msc *msc)
static int msc_iter_win_advance(struct msc_iter *iter)
{
iter->win = msc_next_window(iter->win);
- iter->start_block = -1;
+ iter->start_block = NULL;
if (iter->win == iter->start_win) {
iter->eof++;
@@ -400,8 +542,10 @@ static int msc_iter_block_advance(struct msc_iter *iter)
return msc_iter_win_advance(iter);
/* block advance */
- if (++iter->block == iter->win->nr_segs)
- iter->block = 0;
+ if (sg_is_last(iter->block))
+ iter->block = msc_win_base_sg(iter->win);
+ else
+ iter->block = sg_next(iter->block);
/* no wrapping, sanity check in case there is no last written block */
if (!iter->wrap_count && iter->block == iter->start_block)
@@ -506,14 +650,15 @@ next_block:
static void msc_buffer_clear_hw_header(struct msc *msc)
{
struct msc_window *win;
+ struct scatterlist *sg;
list_for_each_entry(win, &msc->win_list, entry) {
unsigned int blk;
size_t hw_sz = sizeof(struct msc_block_desc) -
offsetof(struct msc_block_desc, hw_tag);
- for (blk = 0; blk < win->nr_segs; blk++) {
- struct msc_block_desc *bdesc = msc_win_block(win, blk);
+ for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
+ struct msc_block_desc *bdesc = sg_virt(sg);
memset(&bdesc->hw_tag, 0, hw_sz);
}
@@ -527,6 +672,9 @@ static int intel_th_msu_init(struct msc *msc)
if (!msc->do_irq)
return 0;
+ if (!msc->mbuf)
+ return 0;
+
mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL);
mintctl |= msc->index ? M1BLIE : M0BLIE;
iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL);
@@ -554,6 +702,49 @@ static void intel_th_msu_deinit(struct msc *msc)
iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL);
}
+static int msc_win_set_lockout(struct msc_window *win,
+ enum lockout_state expect,
+ enum lockout_state new)
+{
+ enum lockout_state old;
+ unsigned long flags;
+ int ret = 0;
+
+ if (!win->msc->mbuf)
+ return 0;
+
+ spin_lock_irqsave(&win->lo_lock, flags);
+ old = win->lockout;
+
+ if (old != expect) {
+ ret = -EINVAL;
+ dev_warn_ratelimited(msc_dev(win->msc),
+ "expected lockout state %d, got %d\n",
+ expect, old);
+ goto unlock;
+ }
+
+ win->lockout = new;
+
+ if (old == expect && new == WIN_LOCKED)
+ atomic_inc(&win->msc->user_count);
+ else if (old == expect && old == WIN_LOCKED)
+ atomic_dec(&win->msc->user_count);
+
+unlock:
+ spin_unlock_irqrestore(&win->lo_lock, flags);
+
+ if (ret) {
+ if (expect == WIN_READY && old == WIN_LOCKED)
+ return -EBUSY;
+
+ /* from intel_th_msc_window_unlock(), don't warn if not locked */
+ if (expect == WIN_LOCKED && old == new)
+ return 0;
+ }
+
+ return ret;
+}
/**
* msc_configure() - set up MSC hardware
* @msc: the MSC device to configure
@@ -571,8 +762,15 @@ static int msc_configure(struct msc *msc)
if (msc->mode > MSC_MODE_MULTI)
return -ENOTSUPP;
- if (msc->mode == MSC_MODE_MULTI)
+ if (msc->mode == MSC_MODE_MULTI) {
+ if (msc_win_set_lockout(msc->cur_win, WIN_READY, WIN_INUSE))
+ return -EBUSY;
+
msc_buffer_clear_hw_header(msc);
+ }
+
+ msc->orig_addr = ioread32(msc->reg_base + REG_MSU_MSC0BAR);
+ msc->orig_sz = ioread32(msc->reg_base + REG_MSU_MSC0SIZE);
reg = msc->base_addr >> PAGE_SHIFT;
iowrite32(reg, msc->reg_base + REG_MSU_MSC0BAR);
@@ -594,10 +792,14 @@ static int msc_configure(struct msc *msc)
iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL);
+ intel_th_msu_init(msc);
+
msc->thdev->output.multiblock = msc->mode == MSC_MODE_MULTI;
intel_th_trace_enable(msc->thdev);
msc->enabled = 1;
+ if (msc->mbuf && msc->mbuf->activate)
+ msc->mbuf->activate(msc->mbuf_priv);
return 0;
}
@@ -611,10 +813,17 @@ static int msc_configure(struct msc *msc)
*/
static void msc_disable(struct msc *msc)
{
+ struct msc_window *win = msc->cur_win;
u32 reg;
lockdep_assert_held(&msc->buf_mutex);
+ if (msc->mode == MSC_MODE_MULTI)
+ msc_win_set_lockout(win, WIN_INUSE, WIN_LOCKED);
+
+ if (msc->mbuf && msc->mbuf->deactivate)
+ msc->mbuf->deactivate(msc->mbuf_priv);
+ intel_th_msu_deinit(msc);
intel_th_trace_disable(msc->thdev);
if (msc->mode == MSC_MODE_SINGLE) {
@@ -630,16 +839,25 @@ static void msc_disable(struct msc *msc)
reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL);
reg &= ~MSC_EN;
iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL);
+
+ if (msc->mbuf && msc->mbuf->ready)
+ msc->mbuf->ready(msc->mbuf_priv, win->sgt,
+ msc_win_total_sz(win));
+
msc->enabled = 0;
- iowrite32(0, msc->reg_base + REG_MSU_MSC0BAR);
- iowrite32(0, msc->reg_base + REG_MSU_MSC0SIZE);
+ iowrite32(msc->orig_addr, msc->reg_base + REG_MSU_MSC0BAR);
+ iowrite32(msc->orig_sz, msc->reg_base + REG_MSU_MSC0SIZE);
dev_dbg(msc_dev(msc), "MSCnNWSA: %08x\n",
ioread32(msc->reg_base + REG_MSU_MSC0NWSA));
reg = ioread32(msc->reg_base + REG_MSU_MSC0STS);
dev_dbg(msc_dev(msc), "MSCnSTS: %08x\n", reg);
+
+ reg = ioread32(msc->reg_base + REG_MSU_MSUSTS);
+ reg &= msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST;
+ iowrite32(reg, msc->reg_base + REG_MSU_MSUSTS);
}
static int intel_th_msc_activate(struct intel_th_device *thdev)
@@ -791,10 +1009,9 @@ static int __msc_buffer_win_alloc(struct msc_window *win,
return nr_segs;
err_nomem:
- for (i--; i >= 0; i--)
+ for_each_sg(win->sgt->sgl, sg_ptr, i, ret)
dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE,
- msc_win_block(win, i),
- msc_win_baddr(win, i));
+ sg_virt(sg_ptr), sg_dma_address(sg_ptr));
sg_free_table(win->sgt);
@@ -804,20 +1021,26 @@ err_nomem:
#ifdef CONFIG_X86
static void msc_buffer_set_uc(struct msc_window *win, unsigned int nr_segs)
{
+ struct scatterlist *sg_ptr;
int i;
- for (i = 0; i < nr_segs; i++)
+ for_each_sg(win->sgt->sgl, sg_ptr, nr_segs, i) {
/* Set the page as uncached */
- set_memory_uc((unsigned long)msc_win_block(win, i), 1);
+ set_memory_uc((unsigned long)sg_virt(sg_ptr),
+ PFN_DOWN(sg_ptr->length));
+ }
}
static void msc_buffer_set_wb(struct msc_window *win)
{
+ struct scatterlist *sg_ptr;
int i;
- for (i = 0; i < win->nr_segs; i++)
+ for_each_sg(win->sgt->sgl, sg_ptr, win->nr_segs, i) {
/* Reset the page to write-back */
- set_memory_wb((unsigned long)msc_win_block(win, i), 1);
+ set_memory_wb((unsigned long)sg_virt(sg_ptr),
+ PFN_DOWN(sg_ptr->length));
+ }
}
#else /* !X86 */
static inline void
@@ -843,19 +1066,14 @@ static int msc_buffer_win_alloc(struct msc *msc, unsigned int nr_blocks)
if (!nr_blocks)
return 0;
- /*
- * This limitation hold as long as we need random access to the
- * block. When that changes, this can go away.
- */
- if (nr_blocks > SG_MAX_SINGLE_ALLOC)
- return -EINVAL;
-
win = kzalloc(sizeof(*win), GFP_KERNEL);
if (!win)
return -ENOMEM;
win->msc = msc;
win->sgt = &win->_sgt;
+ win->lockout = WIN_READY;
+ spin_lock_init(&win->lo_lock);
if (!list_empty(&msc->win_list)) {
struct msc_window *prev = list_last_entry(&msc->win_list,
@@ -865,8 +1083,13 @@ static int msc_buffer_win_alloc(struct msc *msc, unsigned int nr_blocks)
win->pgoff = prev->pgoff + prev->nr_blocks;
}
- ret = __msc_buffer_win_alloc(win, nr_blocks);
- if (ret < 0)
+ if (msc->mbuf && msc->mbuf->alloc_window)
+ ret = msc->mbuf->alloc_window(msc->mbuf_priv, &win->sgt,
+ nr_blocks << PAGE_SHIFT);
+ else
+ ret = __msc_buffer_win_alloc(win, nr_blocks);
+
+ if (ret <= 0)
goto err_nomem;
msc_buffer_set_uc(win, ret);
@@ -875,8 +1098,8 @@ static int msc_buffer_win_alloc(struct msc *msc, unsigned int nr_blocks)
win->nr_blocks = nr_blocks;
if (list_empty(&msc->win_list)) {
- msc->base = msc_win_block(win, 0);
- msc->base_addr = msc_win_baddr(win, 0);
+ msc->base = msc_win_base(win);
+ msc->base_addr = msc_win_base_dma(win);
msc->cur_win = win;
}
@@ -893,14 +1116,15 @@ err_nomem:
static void __msc_buffer_win_free(struct msc *msc, struct msc_window *win)
{
+ struct scatterlist *sg;
int i;
- for (i = 0; i < win->nr_segs; i++) {
- struct page *page = sg_page(&win->sgt->sgl[i]);
+ for_each_sg(win->sgt->sgl, sg, win->nr_segs, i) {
+ struct page *page = sg_page(sg);
page->mapping = NULL;
dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE,
- msc_win_block(win, i), msc_win_baddr(win, i));
+ sg_virt(sg), sg_dma_address(sg));
}
sg_free_table(win->sgt);
}
@@ -925,7 +1149,10 @@ static void msc_buffer_win_free(struct msc *msc, struct msc_window *win)
msc_buffer_set_wb(win);
- __msc_buffer_win_free(msc, win);
+ if (msc->mbuf && msc->mbuf->free_window)
+ msc->mbuf->free_window(msc->mbuf_priv, win->sgt);
+ else
+ __msc_buffer_win_free(msc, win);
kfree(win);
}
@@ -943,6 +1170,7 @@ static void msc_buffer_relink(struct msc *msc)
/* call with msc::mutex locked */
list_for_each_entry(win, &msc->win_list, entry) {
+ struct scatterlist *sg;
unsigned int blk;
u32 sw_tag = 0;
@@ -958,12 +1186,12 @@ static void msc_buffer_relink(struct msc *msc)
next_win = list_next_entry(win, entry);
}
- for (blk = 0; blk < win->nr_segs; blk++) {
- struct msc_block_desc *bdesc = msc_win_block(win, blk);
+ for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
+ struct msc_block_desc *bdesc = sg_virt(sg);
memset(bdesc, 0, sizeof(*bdesc));
- bdesc->next_win = msc_win_bpfn(next_win, 0);
+ bdesc->next_win = msc_win_base_pfn(next_win);
/*
* Similarly to last window, last block should point
@@ -971,13 +1199,15 @@ static void msc_buffer_relink(struct msc *msc)
*/
if (blk == win->nr_segs - 1) {
sw_tag |= MSC_SW_TAG_LASTBLK;
- bdesc->next_blk = msc_win_bpfn(win, 0);
+ bdesc->next_blk = msc_win_base_pfn(win);
} else {
- bdesc->next_blk = msc_win_bpfn(win, blk + 1);
+ dma_addr_t addr = sg_dma_address(sg_next(sg));
+
+ bdesc->next_blk = PFN_DOWN(addr);
}
bdesc->sw_tag = sw_tag;
- bdesc->block_sz = msc_win_actual_bsz(win, blk) / 64;
+ bdesc->block_sz = sg->length / 64;
}
}
@@ -1136,6 +1366,7 @@ static int msc_buffer_free_unless_used(struct msc *msc)
static struct page *msc_buffer_get_page(struct msc *msc, unsigned long pgoff)
{
struct msc_window *win;
+ struct scatterlist *sg;
unsigned int blk;
if (msc->mode == MSC_MODE_SINGLE)
@@ -1150,9 +1381,9 @@ static struct page *msc_buffer_get_page(struct msc *msc, unsigned long pgoff)
found:
pgoff -= win->pgoff;
- for (blk = 0; blk < win->nr_segs; blk++) {
- struct page *page = sg_page(&win->sgt->sgl[blk]);
- size_t pgsz = PFN_DOWN(msc_win_actual_bsz(win, blk));
+ for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
+ struct page *page = sg_page(sg);
+ size_t pgsz = PFN_DOWN(sg->length);
if (pgoff < pgsz)
return page + pgoff;
@@ -1456,24 +1687,83 @@ static void msc_win_switch(struct msc *msc)
else
msc->cur_win = list_next_entry(msc->cur_win, entry);
- msc->base = msc_win_block(msc->cur_win, 0);
- msc->base_addr = msc_win_baddr(msc->cur_win, 0);
+ msc->base = msc_win_base(msc->cur_win);
+ msc->base_addr = msc_win_base_dma(msc->cur_win);
intel_th_trace_switch(msc->thdev);
}
+/**
+ * intel_th_msc_window_unlock - put the window back in rotation
+ * @dev: MSC device to which this relates
+ * @sgt: buffer's sg_table for the window, does nothing if NULL
+ */
+void intel_th_msc_window_unlock(struct device *dev, struct sg_table *sgt)
+{
+ struct msc *msc = dev_get_drvdata(dev);
+ struct msc_window *win;
+
+ if (!sgt)
+ return;
+
+ win = msc_find_window(msc, sgt, false);
+ if (!win)
+ return;
+
+ msc_win_set_lockout(win, WIN_LOCKED, WIN_READY);
+}
+EXPORT_SYMBOL_GPL(intel_th_msc_window_unlock);
+
+static void msc_work(struct work_struct *work)
+{
+ struct msc *msc = container_of(work, struct msc, work);
+
+ intel_th_msc_deactivate(msc->thdev);
+}
+
static irqreturn_t intel_th_msc_interrupt(struct intel_th_device *thdev)
{
struct msc *msc = dev_get_drvdata(&thdev->dev);
u32 msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS);
u32 mask = msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST;
+ struct msc_window *win, *next_win;
+
+ if (!msc->do_irq || !msc->mbuf)
+ return IRQ_NONE;
+
+ msusts &= mask;
+
+ if (!msusts)
+ return msc->enabled ? IRQ_HANDLED : IRQ_NONE;
+
+ iowrite32(msusts, msc->msu_base + REG_MSU_MSUSTS);
- if (!(msusts & mask)) {
- if (msc->enabled)
- return IRQ_HANDLED;
+ if (!msc->enabled)
return IRQ_NONE;
+
+ /* grab the window before we do the switch */
+ win = msc->cur_win;
+ if (!win)
+ return IRQ_HANDLED;
+ next_win = msc_next_window(win);
+ if (!next_win)
+ return IRQ_HANDLED;
+
+ /* next window: if READY, proceed, if LOCKED, stop the trace */
+ if (msc_win_set_lockout(next_win, WIN_READY, WIN_INUSE)) {
+ schedule_work(&msc->work);
+ return IRQ_HANDLED;
}
+ /* current window: INUSE -> LOCKED */
+ msc_win_set_lockout(win, WIN_INUSE, WIN_LOCKED);
+
+ msc_win_switch(msc);
+
+ if (msc->mbuf && msc->mbuf->ready)
+ msc->mbuf->ready(msc->mbuf_priv, win->sgt,
+ msc_win_total_sz(win));
+
return IRQ_HANDLED;
}
@@ -1511,21 +1801,43 @@ wrap_store(struct device *dev, struct device_attribute *attr, const char *buf,
static DEVICE_ATTR_RW(wrap);
+static void msc_buffer_unassign(struct msc *msc)
+{
+ lockdep_assert_held(&msc->buf_mutex);
+
+ if (!msc->mbuf)
+ return;
+
+ msc->mbuf->unassign(msc->mbuf_priv);
+ msu_buffer_put(msc->mbuf);
+ msc->mbuf_priv = NULL;
+ msc->mbuf = NULL;
+}
+
static ssize_t
mode_show(struct device *dev, struct device_attribute *attr, char *buf)
{
struct msc *msc = dev_get_drvdata(dev);
+ const char *mode = msc_mode[msc->mode];
+ ssize_t ret;
+
+ mutex_lock(&msc->buf_mutex);
+ if (msc->mbuf)
+ mode = msc->mbuf->name;
+ ret = scnprintf(buf, PAGE_SIZE, "%s\n", mode);
+ mutex_unlock(&msc->buf_mutex);
- return scnprintf(buf, PAGE_SIZE, "%s\n", msc_mode[msc->mode]);
+ return ret;
}
static ssize_t
mode_store(struct device *dev, struct device_attribute *attr, const char *buf,
size_t size)
{
+ const struct msu_buffer *mbuf = NULL;
struct msc *msc = dev_get_drvdata(dev);
size_t len = size;
- char *cp;
+ char *cp, *mode;
int i, ret;
if (!capable(CAP_SYS_RAWIO))
@@ -1535,17 +1847,59 @@ mode_store(struct device *dev, struct device_attribute *attr, const char *buf,
if (cp)
len = cp - buf;
- for (i = 0; i < ARRAY_SIZE(msc_mode); i++)
- if (!strncmp(msc_mode[i], buf, len))
- goto found;
+ mode = kstrndup(buf, len, GFP_KERNEL);
+ i = match_string(msc_mode, ARRAY_SIZE(msc_mode), mode);
+ if (i >= 0)
+ goto found;
+
+ /* Buffer sinks only work with a usable IRQ */
+ if (!msc->do_irq) {
+ kfree(mode);
+ return -EINVAL;
+ }
+
+ mbuf = msu_buffer_get(mode);
+ kfree(mode);
+ if (mbuf)
+ goto found;
return -EINVAL;
found:
mutex_lock(&msc->buf_mutex);
+ ret = 0;
+
+ /* Same buffer: do nothing */
+ if (mbuf && mbuf == msc->mbuf) {
+ /* put the extra reference we just got */
+ msu_buffer_put(mbuf);
+ goto unlock;
+ }
+
ret = msc_buffer_unlocked_free_unless_used(msc);
- if (!ret)
- msc->mode = i;
+ if (ret)
+ goto unlock;
+
+ if (mbuf) {
+ void *mbuf_priv = mbuf->assign(dev, &i);
+
+ if (!mbuf_priv) {
+ ret = -ENOMEM;
+ goto unlock;
+ }
+
+ msc_buffer_unassign(msc);
+ msc->mbuf_priv = mbuf_priv;
+ msc->mbuf = mbuf;
+ } else {
+ msc_buffer_unassign(msc);
+ }
+
+ msc->mode = i;
+
+unlock:
+ if (ret && mbuf)
+ msu_buffer_put(mbuf);
mutex_unlock(&msc->buf_mutex);
return ret ? ret : size;
@@ -1667,7 +2021,12 @@ win_switch_store(struct device *dev, struct device_attribute *attr,
return -EINVAL;
mutex_lock(&msc->buf_mutex);
- if (msc->mode != MSC_MODE_MULTI)
+ /*
+ * Window switch can only happen in the "multi" mode.
+ * If a external buffer is engaged, they have the full
+ * control over window switching.
+ */
+ if (msc->mode != MSC_MODE_MULTI || msc->mbuf)
ret = -ENOTSUPP;
else
msc_win_switch(msc);
@@ -1720,10 +2079,7 @@ static int intel_th_msc_probe(struct intel_th_device *thdev)
msc->reg_base = base + msc->index * 0x100;
msc->msu_base = base;
- err = intel_th_msu_init(msc);
- if (err)
- return err;
-
+ INIT_WORK(&msc->work, msc_work);
err = intel_th_msc_init(msc);
if (err)
return err;
@@ -1739,7 +2095,6 @@ static void intel_th_msc_remove(struct intel_th_device *thdev)
int ret;
intel_th_msc_deactivate(thdev);
- intel_th_msu_deinit(msc);
/*
* Buffers should not be used at this point except if the
diff --git a/drivers/hwtracing/intel_th/msu.h b/drivers/hwtracing/intel_th/msu.h
index 13d9b141daaa..e771f509bd02 100644
--- a/drivers/hwtracing/intel_th/msu.h
+++ b/drivers/hwtracing/intel_th/msu.h
@@ -44,14 +44,6 @@ enum {
#define M0BLIE BIT(16)
#define M1BLIE BIT(24)
-/* MSC operating modes (MSC_MODE) */
-enum {
- MSC_MODE_SINGLE = 0,
- MSC_MODE_MULTI,
- MSC_MODE_EXI,
- MSC_MODE_DEBUG,
-};
-
/* MSCnSTS bits */
#define MSCSTS_WRAPSTAT BIT(1) /* Wrap occurred */
#define MSCSTS_PLE BIT(2) /* Pipeline Empty */
@@ -93,6 +85,16 @@ static inline unsigned long msc_data_sz(struct msc_block_desc *bdesc)
return bdesc->valid_dw * 4 - MSC_BDESC;
}
+static inline unsigned long msc_total_sz(struct msc_block_desc *bdesc)
+{
+ return bdesc->valid_dw * 4;
+}
+
+static inline unsigned long msc_block_sz(struct msc_block_desc *bdesc)
+{
+ return bdesc->block_sz * 64 - MSC_BDESC;
+}
+
static inline bool msc_block_wrapped(struct msc_block_desc *bdesc)
{
if (bdesc->hw_tag & (MSC_HW_TAG_BLOCKWRAP | MSC_HW_TAG_WINWRAP))
@@ -104,7 +106,7 @@ static inline bool msc_block_wrapped(struct msc_block_desc *bdesc)
static inline bool msc_block_last_written(struct msc_block_desc *bdesc)
{
if ((bdesc->hw_tag & MSC_HW_TAG_ENDBIT) ||
- (msc_data_sz(bdesc) != DATA_IN_PAGE))
+ (msc_data_sz(bdesc) != msc_block_sz(bdesc)))
return true;
return false;
generated by cgit v1.2.3-58-ga151 (git 2.39.0) at 2024-12-25 22:13:01 +0000