summaryrefslogtreecommitdiff
path: root/drivers/irqchip/irq-riscv-imsic-platform.c
blob: 64905e6f52d789fec2c9d5ca49854da7740dc4fd (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
 * Copyright (C) 2022 Ventana Micro Systems Inc.
 */

#define pr_fmt(fmt) "riscv-imsic: " fmt
#include <linux/acpi.h>
#include <linux/bitmap.h>
#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/irq.h>
#include <linux/irqchip.h>
#include <linux/irqdomain.h>
#include <linux/module.h>
#include <linux/msi.h>
#include <linux/pci.h>
#include <linux/platform_device.h>
#include <linux/spinlock.h>
#include <linux/smp.h>

#include "irq-riscv-imsic-state.h"

static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index,
				phys_addr_t *out_msi_pa)
{
	struct imsic_global_config *global;
	struct imsic_local_config *local;

	global = &imsic->global;
	local = per_cpu_ptr(global->local, cpu);

	if (BIT(global->guest_index_bits) <= guest_index)
		return false;

	if (out_msi_pa)
		*out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ);

	return true;
}

static void imsic_irq_mask(struct irq_data *d)
{
	imsic_vector_mask(irq_data_get_irq_chip_data(d));
}

static void imsic_irq_unmask(struct irq_data *d)
{
	imsic_vector_unmask(irq_data_get_irq_chip_data(d));
}

static int imsic_irq_retrigger(struct irq_data *d)
{
	struct imsic_vector *vec = irq_data_get_irq_chip_data(d);
	struct imsic_local_config *local;

	if (WARN_ON(!vec))
		return -ENOENT;

	local = per_cpu_ptr(imsic->global.local, vec->cpu);
	writel_relaxed(vec->local_id, local->msi_va);
	return 0;
}

static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg)
{
	phys_addr_t msi_addr;

	if (WARN_ON(!vec))
		return;

	if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr)))
		return;

	msg->address_hi = upper_32_bits(msi_addr);
	msg->address_lo = lower_32_bits(msi_addr);
	msg->data = vec->local_id;
}

static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg)
{
	imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg);
}

#ifdef CONFIG_SMP
static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec)
{
	struct msi_msg msg = { };

	imsic_irq_compose_vector_msg(vec, &msg);
	irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg);
}

static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
				  bool force)
{
	struct imsic_vector *old_vec, *new_vec;
	struct irq_data *pd = d->parent_data;

	old_vec = irq_data_get_irq_chip_data(pd);
	if (WARN_ON(!old_vec))
		return -ENOENT;

	/* If old vector cpu belongs to the target cpumask then do nothing */
	if (cpumask_test_cpu(old_vec->cpu, mask_val))
		return IRQ_SET_MASK_OK_DONE;

	/* If move is already in-flight then return failure */
	if (imsic_vector_get_move(old_vec))
		return -EBUSY;

	/* Get a new vector on the desired set of CPUs */
	new_vec = imsic_vector_alloc(old_vec->hwirq, mask_val);
	if (!new_vec)
		return -ENOSPC;

	/* Point device to the new vector */
	imsic_msi_update_msg(d, new_vec);

	/* Update irq descriptors with the new vector */
	pd->chip_data = new_vec;

	/* Update effective affinity of parent irq data */
	irq_data_update_effective_affinity(pd, cpumask_of(new_vec->cpu));

	/* Move state of the old vector to the new vector */
	imsic_vector_move(old_vec, new_vec);

	return IRQ_SET_MASK_OK_DONE;
}
#endif

static struct irq_chip imsic_irq_base_chip = {
	.name			= "IMSIC",
	.irq_mask		= imsic_irq_mask,
	.irq_unmask		= imsic_irq_unmask,
	.irq_retrigger		= imsic_irq_retrigger,
	.irq_compose_msi_msg	= imsic_irq_compose_msg,
	.flags			= IRQCHIP_SKIP_SET_WAKE |
				  IRQCHIP_MASK_ON_SUSPEND,
};

static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
				  unsigned int nr_irqs, void *args)
{
	struct imsic_vector *vec;

	/* Multi-MSI is not supported yet. */
	if (nr_irqs > 1)
		return -EOPNOTSUPP;

	vec = imsic_vector_alloc(virq, cpu_online_mask);
	if (!vec)
		return -ENOSPC;

	irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec,
			    handle_simple_irq, NULL, NULL);
	irq_set_noprobe(virq);
	irq_set_affinity(virq, cpu_online_mask);
	irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu));

	return 0;
}

static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
				  unsigned int nr_irqs)
{
	struct irq_data *d = irq_domain_get_irq_data(domain, virq);

	imsic_vector_free(irq_data_get_irq_chip_data(d));
	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
}

static int imsic_irq_domain_select(struct irq_domain *domain, struct irq_fwspec *fwspec,
				   enum irq_domain_bus_token bus_token)
{
	const struct msi_parent_ops *ops = domain->msi_parent_ops;
	u32 busmask = BIT(bus_token);

	if (fwspec->fwnode != domain->fwnode || fwspec->param_count != 0)
		return 0;

	/* Handle pure domain searches */
	if (bus_token == ops->bus_select_token)
		return 1;

	return !!(ops->bus_select_mask & busmask);
}

#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d,
				 struct irq_data *irqd, int ind)
{
	if (!irqd) {
		imsic_vector_debug_show_summary(m, ind);
		return;
	}

	imsic_vector_debug_show(m, irq_data_get_irq_chip_data(irqd), ind);
}
#endif

static const struct irq_domain_ops imsic_base_domain_ops = {
	.alloc		= imsic_irq_domain_alloc,
	.free		= imsic_irq_domain_free,
	.select		= imsic_irq_domain_select,
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
	.debug_show	= imsic_irq_debug_show,
#endif
};

#ifdef CONFIG_RISCV_IMSIC_PCI

static void imsic_pci_mask_irq(struct irq_data *d)
{
	pci_msi_mask_irq(d);
	irq_chip_mask_parent(d);
}

static void imsic_pci_unmask_irq(struct irq_data *d)
{
	irq_chip_unmask_parent(d);
	pci_msi_unmask_irq(d);
}

#define MATCH_PCI_MSI		BIT(DOMAIN_BUS_PCI_MSI)

#else

#define MATCH_PCI_MSI		0

#endif

static bool imsic_init_dev_msi_info(struct device *dev,
				    struct irq_domain *domain,
				    struct irq_domain *real_parent,
				    struct msi_domain_info *info)
{
	const struct msi_parent_ops *pops = real_parent->msi_parent_ops;

	/* MSI parent domain specific settings */
	switch (real_parent->bus_token) {
	case DOMAIN_BUS_NEXUS:
		if (WARN_ON_ONCE(domain != real_parent))
			return false;
#ifdef CONFIG_SMP
		info->chip->irq_set_affinity = imsic_irq_set_affinity;
#endif
		break;
	default:
		WARN_ON_ONCE(1);
		return false;
	}

	/* Is the target supported? */
	switch (info->bus_token) {
#ifdef CONFIG_RISCV_IMSIC_PCI
	case DOMAIN_BUS_PCI_DEVICE_MSI:
	case DOMAIN_BUS_PCI_DEVICE_MSIX:
		info->chip->irq_mask = imsic_pci_mask_irq;
		info->chip->irq_unmask = imsic_pci_unmask_irq;
		break;
#endif
	case DOMAIN_BUS_DEVICE_MSI:
		/*
		 * Per-device MSI should never have any MSI feature bits
		 * set. It's sole purpose is to create a dumb interrupt
		 * chip which has a device specific irq_write_msi_msg()
		 * callback.
		 */
		if (WARN_ON_ONCE(info->flags))
			return false;

		/* Core managed MSI descriptors */
		info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS |
			       MSI_FLAG_FREE_MSI_DESCS;
		break;
	case DOMAIN_BUS_WIRED_TO_MSI:
		break;
	default:
		WARN_ON_ONCE(1);
		return false;
	}

	/* Use hierarchial chip operations re-trigger */
	info->chip->irq_retrigger = irq_chip_retrigger_hierarchy;

	/*
	 * Mask out the domain specific MSI feature flags which are not
	 * supported by the real parent.
	 */
	info->flags &= pops->supported_flags;

	/* Enforce the required flags */
	info->flags |= pops->required_flags;

	return true;
}

#define MATCH_PLATFORM_MSI		BIT(DOMAIN_BUS_PLATFORM_MSI)

static const struct msi_parent_ops imsic_msi_parent_ops = {
	.supported_flags	= MSI_GENERIC_FLAGS_MASK |
				  MSI_FLAG_PCI_MSIX,
	.required_flags		= MSI_FLAG_USE_DEF_DOM_OPS |
				  MSI_FLAG_USE_DEF_CHIP_OPS,
	.bus_select_token	= DOMAIN_BUS_NEXUS,
	.bus_select_mask	= MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
	.init_dev_msi_info	= imsic_init_dev_msi_info,
};

int imsic_irqdomain_init(void)
{
	struct imsic_global_config *global;

	if (!imsic || !imsic->fwnode) {
		pr_err("early driver not probed\n");
		return -ENODEV;
	}

	if (imsic->base_domain) {
		pr_err("%pfwP: irq domain already created\n", imsic->fwnode);
		return -ENODEV;
	}

	/* Create Base IRQ domain */
	imsic->base_domain = irq_domain_create_tree(imsic->fwnode,
						    &imsic_base_domain_ops, imsic);
	if (!imsic->base_domain) {
		pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode);
		return -ENOMEM;
	}
	imsic->base_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT;
	imsic->base_domain->msi_parent_ops = &imsic_msi_parent_ops;

	irq_domain_update_bus_token(imsic->base_domain, DOMAIN_BUS_NEXUS);

	global = &imsic->global;
	pr_info("%pfwP:  hart-index-bits: %d,  guest-index-bits: %d\n",
		imsic->fwnode, global->hart_index_bits, global->guest_index_bits);
	pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n",
		imsic->fwnode, global->group_index_bits, global->group_index_shift);
	pr_info("%pfwP: per-CPU IDs %d at base PPN %pa\n",
		imsic->fwnode, global->nr_ids, &global->base_addr);
	pr_info("%pfwP: total %d interrupts available\n",
		imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1));

	return 0;
}

static int imsic_platform_probe_common(struct fwnode_handle *fwnode)
{
	if (imsic && imsic->fwnode != fwnode) {
		pr_err("%pfwP: fwnode mismatch\n", fwnode);
		return -ENODEV;
	}

	return imsic_irqdomain_init();
}

static int imsic_platform_dt_probe(struct platform_device *pdev)
{
	return imsic_platform_probe_common(pdev->dev.fwnode);
}

#ifdef CONFIG_ACPI

/*
 *  On ACPI based systems, PCI enumeration happens early during boot in
 *  acpi_scan_init(). PCI enumeration expects MSI domain setup before
 *  it calls pci_set_msi_domain(). Hence, unlike in DT where
 *  imsic-platform drive probe happens late during boot, ACPI based
 *  systems need to setup the MSI domain early.
 */
int imsic_platform_acpi_probe(struct fwnode_handle *fwnode)
{
	return imsic_platform_probe_common(fwnode);
}

#endif

static const struct of_device_id imsic_platform_match[] = {
	{ .compatible = "riscv,imsics" },
	{}
};

static struct platform_driver imsic_platform_driver = {
	.driver = {
		.name		= "riscv-imsic",
		.of_match_table	= imsic_platform_match,
	},
	.probe = imsic_platform_dt_probe,
};
builtin_platform_driver(imsic_platform_driver);