summaryrefslogtreecommitdiff
path: root/arch/mips/include/asm/kvm_host.h
blob: 957121a495f0b744d3881f9d59627a89e269bb6c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
/*
* This file is subject to the terms and conditions of the GNU General Public
* License.  See the file "COPYING" in the main directory of this archive
* for more details.
*
* Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
* Authors: Sanjay Lal <sanjayl@kymasys.com>
*/

#ifndef __MIPS_KVM_HOST_H__
#define __MIPS_KVM_HOST_H__

#include <linux/cpumask.h>
#include <linux/mutex.h>
#include <linux/hrtimer.h>
#include <linux/interrupt.h>
#include <linux/types.h>
#include <linux/kvm.h>
#include <linux/kvm_types.h>
#include <linux/threads.h>
#include <linux/spinlock.h>

#include <asm/asm.h>
#include <asm/inst.h>
#include <asm/mipsregs.h>

#include <kvm/iodev.h>

/* MIPS KVM register ids */
#define MIPS_CP0_32(_R, _S)					\
	(KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U32 | (8 * (_R) + (_S)))

#define MIPS_CP0_64(_R, _S)					\
	(KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U64 | (8 * (_R) + (_S)))

#define KVM_REG_MIPS_CP0_INDEX		MIPS_CP0_32(0, 0)
#define KVM_REG_MIPS_CP0_ENTRYLO0	MIPS_CP0_64(2, 0)
#define KVM_REG_MIPS_CP0_ENTRYLO1	MIPS_CP0_64(3, 0)
#define KVM_REG_MIPS_CP0_CONTEXT	MIPS_CP0_64(4, 0)
#define KVM_REG_MIPS_CP0_CONTEXTCONFIG	MIPS_CP0_32(4, 1)
#define KVM_REG_MIPS_CP0_USERLOCAL	MIPS_CP0_64(4, 2)
#define KVM_REG_MIPS_CP0_XCONTEXTCONFIG	MIPS_CP0_64(4, 3)
#define KVM_REG_MIPS_CP0_PAGEMASK	MIPS_CP0_32(5, 0)
#define KVM_REG_MIPS_CP0_PAGEGRAIN	MIPS_CP0_32(5, 1)
#define KVM_REG_MIPS_CP0_SEGCTL0	MIPS_CP0_64(5, 2)
#define KVM_REG_MIPS_CP0_SEGCTL1	MIPS_CP0_64(5, 3)
#define KVM_REG_MIPS_CP0_SEGCTL2	MIPS_CP0_64(5, 4)
#define KVM_REG_MIPS_CP0_PWBASE		MIPS_CP0_64(5, 5)
#define KVM_REG_MIPS_CP0_PWFIELD	MIPS_CP0_64(5, 6)
#define KVM_REG_MIPS_CP0_PWSIZE		MIPS_CP0_64(5, 7)
#define KVM_REG_MIPS_CP0_WIRED		MIPS_CP0_32(6, 0)
#define KVM_REG_MIPS_CP0_PWCTL		MIPS_CP0_32(6, 6)
#define KVM_REG_MIPS_CP0_HWRENA		MIPS_CP0_32(7, 0)
#define KVM_REG_MIPS_CP0_BADVADDR	MIPS_CP0_64(8, 0)
#define KVM_REG_MIPS_CP0_BADINSTR	MIPS_CP0_32(8, 1)
#define KVM_REG_MIPS_CP0_BADINSTRP	MIPS_CP0_32(8, 2)
#define KVM_REG_MIPS_CP0_COUNT		MIPS_CP0_32(9, 0)
#define KVM_REG_MIPS_CP0_ENTRYHI	MIPS_CP0_64(10, 0)
#define KVM_REG_MIPS_CP0_COMPARE	MIPS_CP0_32(11, 0)
#define KVM_REG_MIPS_CP0_STATUS		MIPS_CP0_32(12, 0)
#define KVM_REG_MIPS_CP0_INTCTL		MIPS_CP0_32(12, 1)
#define KVM_REG_MIPS_CP0_CAUSE		MIPS_CP0_32(13, 0)
#define KVM_REG_MIPS_CP0_EPC		MIPS_CP0_64(14, 0)
#define KVM_REG_MIPS_CP0_PRID		MIPS_CP0_32(15, 0)
#define KVM_REG_MIPS_CP0_EBASE		MIPS_CP0_64(15, 1)
#define KVM_REG_MIPS_CP0_CONFIG		MIPS_CP0_32(16, 0)
#define KVM_REG_MIPS_CP0_CONFIG1	MIPS_CP0_32(16, 1)
#define KVM_REG_MIPS_CP0_CONFIG2	MIPS_CP0_32(16, 2)
#define KVM_REG_MIPS_CP0_CONFIG3	MIPS_CP0_32(16, 3)
#define KVM_REG_MIPS_CP0_CONFIG4	MIPS_CP0_32(16, 4)
#define KVM_REG_MIPS_CP0_CONFIG5	MIPS_CP0_32(16, 5)
#define KVM_REG_MIPS_CP0_CONFIG6	MIPS_CP0_32(16, 6)
#define KVM_REG_MIPS_CP0_CONFIG7	MIPS_CP0_32(16, 7)
#define KVM_REG_MIPS_CP0_MAARI		MIPS_CP0_64(17, 2)
#define KVM_REG_MIPS_CP0_XCONTEXT	MIPS_CP0_64(20, 0)
#define KVM_REG_MIPS_CP0_DIAG		MIPS_CP0_32(22, 0)
#define KVM_REG_MIPS_CP0_ERROREPC	MIPS_CP0_64(30, 0)
#define KVM_REG_MIPS_CP0_KSCRATCH1	MIPS_CP0_64(31, 2)
#define KVM_REG_MIPS_CP0_KSCRATCH2	MIPS_CP0_64(31, 3)
#define KVM_REG_MIPS_CP0_KSCRATCH3	MIPS_CP0_64(31, 4)
#define KVM_REG_MIPS_CP0_KSCRATCH4	MIPS_CP0_64(31, 5)
#define KVM_REG_MIPS_CP0_KSCRATCH5	MIPS_CP0_64(31, 6)
#define KVM_REG_MIPS_CP0_KSCRATCH6	MIPS_CP0_64(31, 7)


#define KVM_MAX_VCPUS		16

#define KVM_HALT_POLL_NS_DEFAULT 500000

extern unsigned long GUESTID_MASK;
extern unsigned long GUESTID_FIRST_VERSION;
extern unsigned long GUESTID_VERSION_MASK;

#define KVM_INVALID_ADDR		0xdeadbeef

/*
 * EVA has overlapping user & kernel address spaces, so user VAs may be >
 * PAGE_OFFSET. For this reason we can't use the default KVM_HVA_ERR_BAD of
 * PAGE_OFFSET.
 */

#define KVM_HVA_ERR_BAD			(-1UL)
#define KVM_HVA_ERR_RO_BAD		(-2UL)

static inline bool kvm_is_error_hva(unsigned long addr)
{
	return IS_ERR_VALUE(addr);
}

struct kvm_vm_stat {
	struct kvm_vm_stat_generic generic;
};

struct kvm_vcpu_stat {
	struct kvm_vcpu_stat_generic generic;
	u64 wait_exits;
	u64 cache_exits;
	u64 signal_exits;
	u64 int_exits;
	u64 cop_unusable_exits;
	u64 tlbmod_exits;
	u64 tlbmiss_ld_exits;
	u64 tlbmiss_st_exits;
	u64 addrerr_st_exits;
	u64 addrerr_ld_exits;
	u64 syscall_exits;
	u64 resvd_inst_exits;
	u64 break_inst_exits;
	u64 trap_inst_exits;
	u64 msa_fpe_exits;
	u64 fpe_exits;
	u64 msa_disabled_exits;
	u64 flush_dcache_exits;
	u64 vz_gpsi_exits;
	u64 vz_gsfc_exits;
	u64 vz_hc_exits;
	u64 vz_grr_exits;
	u64 vz_gva_exits;
	u64 vz_ghfc_exits;
	u64 vz_gpa_exits;
	u64 vz_resvd_exits;
#ifdef CONFIG_CPU_LOONGSON64
	u64 vz_cpucfg_exits;
#endif
};

struct kvm_arch_memory_slot {
};

#ifdef CONFIG_CPU_LOONGSON64
struct ipi_state {
	uint32_t status;
	uint32_t en;
	uint32_t set;
	uint32_t clear;
	uint64_t buf[4];
};

struct loongson_kvm_ipi;

struct ipi_io_device {
	int node_id;
	struct loongson_kvm_ipi *ipi;
	struct kvm_io_device device;
};

struct loongson_kvm_ipi {
	spinlock_t lock;
	struct kvm *kvm;
	struct ipi_state ipistate[16];
	struct ipi_io_device dev_ipi[4];
};
#endif

struct kvm_arch {
	/* Guest physical mm */
	struct mm_struct gpa_mm;
	/* Mask of CPUs needing GPA ASID flush */
	cpumask_t asid_flush_mask;
#ifdef CONFIG_CPU_LOONGSON64
	struct loongson_kvm_ipi ipi;
#endif
};

#define N_MIPS_COPROC_REGS	32
#define N_MIPS_COPROC_SEL	8

struct mips_coproc {
	unsigned long reg[N_MIPS_COPROC_REGS][N_MIPS_COPROC_SEL];
#ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
	unsigned long stat[N_MIPS_COPROC_REGS][N_MIPS_COPROC_SEL];
#endif
};

/*
 * Coprocessor 0 register names
 */
#define MIPS_CP0_TLB_INDEX	0
#define MIPS_CP0_TLB_RANDOM	1
#define MIPS_CP0_TLB_LOW	2
#define MIPS_CP0_TLB_LO0	2
#define MIPS_CP0_TLB_LO1	3
#define MIPS_CP0_TLB_CONTEXT	4
#define MIPS_CP0_TLB_PG_MASK	5
#define MIPS_CP0_TLB_WIRED	6
#define MIPS_CP0_HWRENA		7
#define MIPS_CP0_BAD_VADDR	8
#define MIPS_CP0_COUNT		9
#define MIPS_CP0_TLB_HI		10
#define MIPS_CP0_COMPARE	11
#define MIPS_CP0_STATUS		12
#define MIPS_CP0_CAUSE		13
#define MIPS_CP0_EXC_PC		14
#define MIPS_CP0_PRID		15
#define MIPS_CP0_CONFIG		16
#define MIPS_CP0_LLADDR		17
#define MIPS_CP0_WATCH_LO	18
#define MIPS_CP0_WATCH_HI	19
#define MIPS_CP0_TLB_XCONTEXT	20
#define MIPS_CP0_DIAG		22
#define MIPS_CP0_ECC		26
#define MIPS_CP0_CACHE_ERR	27
#define MIPS_CP0_TAG_LO		28
#define MIPS_CP0_TAG_HI		29
#define MIPS_CP0_ERROR_PC	30
#define MIPS_CP0_DEBUG		23
#define MIPS_CP0_DEPC		24
#define MIPS_CP0_PERFCNT	25
#define MIPS_CP0_ERRCTL		26
#define MIPS_CP0_DATA_LO	28
#define MIPS_CP0_DATA_HI	29
#define MIPS_CP0_DESAVE		31

#define MIPS_CP0_CONFIG_SEL	0
#define MIPS_CP0_CONFIG1_SEL	1
#define MIPS_CP0_CONFIG2_SEL	2
#define MIPS_CP0_CONFIG3_SEL	3
#define MIPS_CP0_CONFIG4_SEL	4
#define MIPS_CP0_CONFIG5_SEL	5

#define MIPS_CP0_GUESTCTL2	10
#define MIPS_CP0_GUESTCTL2_SEL	5
#define MIPS_CP0_GTOFFSET	12
#define MIPS_CP0_GTOFFSET_SEL	7

/* Resume Flags */
#define RESUME_FLAG_DR		(1<<0)	/* Reload guest nonvolatile state? */
#define RESUME_FLAG_HOST	(1<<1)	/* Resume host? */

#define RESUME_GUEST		0
#define RESUME_GUEST_DR		RESUME_FLAG_DR
#define RESUME_HOST		RESUME_FLAG_HOST

enum emulation_result {
	EMULATE_DONE,		/* no further processing */
	EMULATE_DO_MMIO,	/* kvm_run filled with MMIO request */
	EMULATE_FAIL,		/* can't emulate this instruction */
	EMULATE_WAIT,		/* WAIT instruction */
	EMULATE_PRIV_FAIL,
	EMULATE_EXCEPT,		/* A guest exception has been generated */
	EMULATE_HYPERCALL,	/* HYPCALL instruction */
};

#if defined(CONFIG_64BIT)
#define VPN2_MASK		GENMASK(cpu_vmbits - 1, 13)
#else
#define VPN2_MASK		0xffffe000
#endif
#define KVM_ENTRYHI_ASID	cpu_asid_mask(&boot_cpu_data)
#define TLB_IS_GLOBAL(x)	((x).tlb_lo[0] & (x).tlb_lo[1] & ENTRYLO_G)
#define TLB_VPN2(x)		((x).tlb_hi & VPN2_MASK)
#define TLB_ASID(x)		((x).tlb_hi & KVM_ENTRYHI_ASID)
#define TLB_LO_IDX(x, va)	(((va) >> PAGE_SHIFT) & 1)
#define TLB_IS_VALID(x, va)	((x).tlb_lo[TLB_LO_IDX(x, va)] & ENTRYLO_V)
#define TLB_IS_DIRTY(x, va)	((x).tlb_lo[TLB_LO_IDX(x, va)] & ENTRYLO_D)
#define TLB_HI_VPN2_HIT(x, y)	((TLB_VPN2(x) & ~(x).tlb_mask) ==	\
				 ((y) & VPN2_MASK & ~(x).tlb_mask))
#define TLB_HI_ASID_HIT(x, y)	(TLB_IS_GLOBAL(x) ||			\
				 TLB_ASID(x) == ((y) & KVM_ENTRYHI_ASID))

struct kvm_mips_tlb {
	long tlb_mask;
	long tlb_hi;
	long tlb_lo[2];
};

#define KVM_MIPS_AUX_FPU	0x1
#define KVM_MIPS_AUX_MSA	0x2

struct kvm_vcpu_arch {
	void *guest_ebase;
	int (*vcpu_run)(struct kvm_vcpu *vcpu);

	/* Host registers preserved across guest mode execution */
	unsigned long host_stack;
	unsigned long host_gp;
	unsigned long host_pgd;
	unsigned long host_entryhi;

	/* Host CP0 registers used when handling exits from guest */
	unsigned long host_cp0_badvaddr;
	unsigned long host_cp0_epc;
	u32 host_cp0_cause;
	u32 host_cp0_guestctl0;
	u32 host_cp0_badinstr;
	u32 host_cp0_badinstrp;

	/* GPRS */
	unsigned long gprs[32];
	unsigned long hi;
	unsigned long lo;
	unsigned long pc;

	/* FPU State */
	struct mips_fpu_struct fpu;
	/* Which auxiliary state is loaded (KVM_MIPS_AUX_*) */
	unsigned int aux_inuse;

	/* COP0 State */
	struct mips_coproc *cop0;

	/* Resume PC after MMIO completion */
	unsigned long io_pc;
	/* GPR used as IO source/target */
	u32 io_gpr;

	struct hrtimer comparecount_timer;
	/* Count timer control KVM register */
	u32 count_ctl;
	/* Count bias from the raw time */
	u32 count_bias;
	/* Frequency of timer in Hz */
	u32 count_hz;
	/* Dynamic nanosecond bias (multiple of count_period) to avoid overflow */
	s64 count_dyn_bias;
	/* Resume time */
	ktime_t count_resume;
	/* Period of timer tick in ns */
	u64 count_period;

	/* Bitmask of exceptions that are pending */
	unsigned long pending_exceptions;

	/* Bitmask of pending exceptions to be cleared */
	unsigned long pending_exceptions_clr;

	/* Cache some mmu pages needed inside spinlock regions */
	struct kvm_mmu_memory_cache mmu_page_cache;

	/* vcpu's vzguestid is different on each host cpu in an smp system */
	u32 vzguestid[NR_CPUS];

	/* wired guest TLB entries */
	struct kvm_mips_tlb *wired_tlb;
	unsigned int wired_tlb_limit;
	unsigned int wired_tlb_used;

	/* emulated guest MAAR registers */
	unsigned long maar[6];

	/* Last CPU the VCPU state was loaded on */
	int last_sched_cpu;
	/* Last CPU the VCPU actually executed guest code on */
	int last_exec_cpu;

	/* WAIT executed */
	int wait;

	u8 fpu_enabled;
	u8 msa_enabled;
};

static inline void _kvm_atomic_set_c0_guest_reg(unsigned long *reg,
						unsigned long val)
{
	unsigned long temp;
	do {
		__asm__ __volatile__(
		"	.set	push				\n"
		"	.set	"MIPS_ISA_ARCH_LEVEL"		\n"
		"	"__stringify(LONG_LL)	" %0, %1	\n"
		"	or	%0, %2				\n"
		"	"__stringify(LONG_SC)	" %0, %1	\n"
		"	.set	pop				\n"
		: "=&r" (temp), "+m" (*reg)
		: "r" (val));
	} while (unlikely(!temp));
}

static inline void _kvm_atomic_clear_c0_guest_reg(unsigned long *reg,
						  unsigned long val)
{
	unsigned long temp;
	do {
		__asm__ __volatile__(
		"	.set	push				\n"
		"	.set	"MIPS_ISA_ARCH_LEVEL"		\n"
		"	"__stringify(LONG_LL)	" %0, %1	\n"
		"	and	%0, %2				\n"
		"	"__stringify(LONG_SC)	" %0, %1	\n"
		"	.set	pop				\n"
		: "=&r" (temp), "+m" (*reg)
		: "r" (~val));
	} while (unlikely(!temp));
}

static inline void _kvm_atomic_change_c0_guest_reg(unsigned long *reg,
						   unsigned long change,
						   unsigned long val)
{
	unsigned long temp;
	do {
		__asm__ __volatile__(
		"	.set	push				\n"
		"	.set	"MIPS_ISA_ARCH_LEVEL"		\n"
		"	"__stringify(LONG_LL)	" %0, %1	\n"
		"	and	%0, %2				\n"
		"	or	%0, %3				\n"
		"	"__stringify(LONG_SC)	" %0, %1	\n"
		"	.set	pop				\n"
		: "=&r" (temp), "+m" (*reg)
		: "r" (~change), "r" (val & change));
	} while (unlikely(!temp));
}

/* Guest register types, used in accessor build below */
#define __KVMT32	u32
#define __KVMTl	unsigned long

/*
 * __BUILD_KVM_$ops_SAVED(): kvm_$op_sw_gc0_$reg()
 * These operate on the saved guest C0 state in RAM.
 */

/* Generate saved context simple accessors */
#define __BUILD_KVM_RW_SAVED(name, type, _reg, sel)			\
static inline __KVMT##type kvm_read_sw_gc0_##name(struct mips_coproc *cop0) \
{									\
	return cop0->reg[(_reg)][(sel)];				\
}									\
static inline void kvm_write_sw_gc0_##name(struct mips_coproc *cop0,	\
					   __KVMT##type val)		\
{									\
	cop0->reg[(_reg)][(sel)] = val;					\
}

/* Generate saved context bitwise modifiers */
#define __BUILD_KVM_SET_SAVED(name, type, _reg, sel)			\
static inline void kvm_set_sw_gc0_##name(struct mips_coproc *cop0,	\
					 __KVMT##type val)		\
{									\
	cop0->reg[(_reg)][(sel)] |= val;				\
}									\
static inline void kvm_clear_sw_gc0_##name(struct mips_coproc *cop0,	\
					   __KVMT##type val)		\
{									\
	cop0->reg[(_reg)][(sel)] &= ~val;				\
}									\
static inline void kvm_change_sw_gc0_##name(struct mips_coproc *cop0,	\
					    __KVMT##type mask,		\
					    __KVMT##type val)		\
{									\
	unsigned long _mask = mask;					\
	cop0->reg[(_reg)][(sel)] &= ~_mask;				\
	cop0->reg[(_reg)][(sel)] |= val & _mask;			\
}

/* Generate saved context atomic bitwise modifiers */
#define __BUILD_KVM_ATOMIC_SAVED(name, type, _reg, sel)			\
static inline void kvm_set_sw_gc0_##name(struct mips_coproc *cop0,	\
					 __KVMT##type val)		\
{									\
	_kvm_atomic_set_c0_guest_reg(&cop0->reg[(_reg)][(sel)], val);	\
}									\
static inline void kvm_clear_sw_gc0_##name(struct mips_coproc *cop0,	\
					   __KVMT##type val)		\
{									\
	_kvm_atomic_clear_c0_guest_reg(&cop0->reg[(_reg)][(sel)], val);	\
}									\
static inline void kvm_change_sw_gc0_##name(struct mips_coproc *cop0,	\
					    __KVMT##type mask,		\
					    __KVMT##type val)		\
{									\
	_kvm_atomic_change_c0_guest_reg(&cop0->reg[(_reg)][(sel)], mask, \
					val);				\
}

/*
 * __BUILD_KVM_$ops_VZ(): kvm_$op_vz_gc0_$reg()
 * These operate on the VZ guest C0 context in hardware.
 */

/* Generate VZ guest context simple accessors */
#define __BUILD_KVM_RW_VZ(name, type, _reg, sel)			\
static inline __KVMT##type kvm_read_vz_gc0_##name(struct mips_coproc *cop0) \
{									\
	return read_gc0_##name();					\
}									\
static inline void kvm_write_vz_gc0_##name(struct mips_coproc *cop0,	\
					   __KVMT##type val)		\
{									\
	write_gc0_##name(val);						\
}

/* Generate VZ guest context bitwise modifiers */
#define __BUILD_KVM_SET_VZ(name, type, _reg, sel)			\
static inline void kvm_set_vz_gc0_##name(struct mips_coproc *cop0,	\
					 __KVMT##type val)		\
{									\
	set_gc0_##name(val);						\
}									\
static inline void kvm_clear_vz_gc0_##name(struct mips_coproc *cop0,	\
					   __KVMT##type val)		\
{									\
	clear_gc0_##name(val);						\
}									\
static inline void kvm_change_vz_gc0_##name(struct mips_coproc *cop0,	\
					    __KVMT##type mask,		\
					    __KVMT##type val)		\
{									\
	change_gc0_##name(mask, val);					\
}

/* Generate VZ guest context save/restore to/from saved context */
#define __BUILD_KVM_SAVE_VZ(name, _reg, sel)			\
static inline void kvm_restore_gc0_##name(struct mips_coproc *cop0)	\
{									\
	write_gc0_##name(cop0->reg[(_reg)][(sel)]);			\
}									\
static inline void kvm_save_gc0_##name(struct mips_coproc *cop0)	\
{									\
	cop0->reg[(_reg)][(sel)] = read_gc0_##name();			\
}

/*
 * __BUILD_KVM_$ops_WRAP(): kvm_$op_$name1() -> kvm_$op_$name2()
 * These wrap a set of operations to provide them with a different name.
 */

/* Generate simple accessor wrapper */
#define __BUILD_KVM_RW_WRAP(name1, name2, type)				\
static inline __KVMT##type kvm_read_##name1(struct mips_coproc *cop0)	\
{									\
	return kvm_read_##name2(cop0);					\
}									\
static inline void kvm_write_##name1(struct mips_coproc *cop0,		\
				     __KVMT##type val)			\
{									\
	kvm_write_##name2(cop0, val);					\
}

/* Generate bitwise modifier wrapper */
#define __BUILD_KVM_SET_WRAP(name1, name2, type)			\
static inline void kvm_set_##name1(struct mips_coproc *cop0,		\
				   __KVMT##type val)			\
{									\
	kvm_set_##name2(cop0, val);					\
}									\
static inline void kvm_clear_##name1(struct mips_coproc *cop0,		\
				     __KVMT##type val)			\
{									\
	kvm_clear_##name2(cop0, val);					\
}									\
static inline void kvm_change_##name1(struct mips_coproc *cop0,		\
				      __KVMT##type mask,		\
				      __KVMT##type val)			\
{									\
	kvm_change_##name2(cop0, mask, val);				\
}

/*
 * __BUILD_KVM_$ops_SW(): kvm_$op_c0_guest_$reg() -> kvm_$op_sw_gc0_$reg()
 * These generate accessors operating on the saved context in RAM, and wrap them
 * with the common guest C0 accessors (for use by common emulation code).
 */

#define __BUILD_KVM_RW_SW(name, type, _reg, sel)			\
	__BUILD_KVM_RW_SAVED(name, type, _reg, sel)			\
	__BUILD_KVM_RW_WRAP(c0_guest_##name, sw_gc0_##name, type)

#define __BUILD_KVM_SET_SW(name, type, _reg, sel)			\
	__BUILD_KVM_SET_SAVED(name, type, _reg, sel)			\
	__BUILD_KVM_SET_WRAP(c0_guest_##name, sw_gc0_##name, type)

#define __BUILD_KVM_ATOMIC_SW(name, type, _reg, sel)			\
	__BUILD_KVM_ATOMIC_SAVED(name, type, _reg, sel)			\
	__BUILD_KVM_SET_WRAP(c0_guest_##name, sw_gc0_##name, type)

/*
 * VZ (hardware assisted virtualisation)
 * These macros use the active guest state in VZ mode (hardware registers),
 */

/*
 * __BUILD_KVM_$ops_HW(): kvm_$op_c0_guest_$reg() -> kvm_$op_vz_gc0_$reg()
 * These generate accessors operating on the VZ guest context in hardware, and
 * wrap them with the common guest C0 accessors (for use by common emulation
 * code).
 *
 * Accessors operating on the saved context in RAM are also generated to allow
 * convenient explicit saving and restoring of the state.
 */

#define __BUILD_KVM_RW_HW(name, type, _reg, sel)			\
	__BUILD_KVM_RW_SAVED(name, type, _reg, sel)			\
	__BUILD_KVM_RW_VZ(name, type, _reg, sel)			\
	__BUILD_KVM_RW_WRAP(c0_guest_##name, vz_gc0_##name, type)	\
	__BUILD_KVM_SAVE_VZ(name, _reg, sel)

#define __BUILD_KVM_SET_HW(name, type, _reg, sel)			\
	__BUILD_KVM_SET_SAVED(name, type, _reg, sel)			\
	__BUILD_KVM_SET_VZ(name, type, _reg, sel)			\
	__BUILD_KVM_SET_WRAP(c0_guest_##name, vz_gc0_##name, type)

/*
 * We can't do atomic modifications of COP0 state if hardware can modify it.
 * Races must be handled explicitly.
 */
#define __BUILD_KVM_ATOMIC_HW	__BUILD_KVM_SET_HW

/*
 * Define accessors for CP0 registers that are accessible to the guest. These
 * are primarily used by common emulation code, which may need to access the
 * registers differently depending on the implementation.
 *
 *    fns_hw/sw    name     type    reg num         select
 */
__BUILD_KVM_RW_HW(index,          32, MIPS_CP0_TLB_INDEX,    0)
__BUILD_KVM_RW_HW(entrylo0,       l,  MIPS_CP0_TLB_LO0,      0)
__BUILD_KVM_RW_HW(entrylo1,       l,  MIPS_CP0_TLB_LO1,      0)
__BUILD_KVM_RW_HW(context,        l,  MIPS_CP0_TLB_CONTEXT,  0)
__BUILD_KVM_RW_HW(contextconfig,  32, MIPS_CP0_TLB_CONTEXT,  1)
__BUILD_KVM_RW_HW(userlocal,      l,  MIPS_CP0_TLB_CONTEXT,  2)
__BUILD_KVM_RW_HW(xcontextconfig, l,  MIPS_CP0_TLB_CONTEXT,  3)
__BUILD_KVM_RW_HW(pagemask,       l,  MIPS_CP0_TLB_PG_MASK,  0)
__BUILD_KVM_RW_HW(pagegrain,      32, MIPS_CP0_TLB_PG_MASK,  1)
__BUILD_KVM_RW_HW(segctl0,        l,  MIPS_CP0_TLB_PG_MASK,  2)
__BUILD_KVM_RW_HW(segctl1,        l,  MIPS_CP0_TLB_PG_MASK,  3)
__BUILD_KVM_RW_HW(segctl2,        l,  MIPS_CP0_TLB_PG_MASK,  4)
__BUILD_KVM_RW_HW(pwbase,         l,  MIPS_CP0_TLB_PG_MASK,  5)
__BUILD_KVM_RW_HW(pwfield,        l,  MIPS_CP0_TLB_PG_MASK,  6)
__BUILD_KVM_RW_HW(pwsize,         l,  MIPS_CP0_TLB_PG_MASK,  7)
__BUILD_KVM_RW_HW(wired,          32, MIPS_CP0_TLB_WIRED,    0)
__BUILD_KVM_RW_HW(pwctl,          32, MIPS_CP0_TLB_WIRED,    6)
__BUILD_KVM_RW_HW(hwrena,         32, MIPS_CP0_HWRENA,       0)
__BUILD_KVM_RW_HW(badvaddr,       l,  MIPS_CP0_BAD_VADDR,    0)
__BUILD_KVM_RW_HW(badinstr,       32, MIPS_CP0_BAD_VADDR,    1)
__BUILD_KVM_RW_HW(badinstrp,      32, MIPS_CP0_BAD_VADDR,    2)
__BUILD_KVM_RW_SW(count,          32, MIPS_CP0_COUNT,        0)
__BUILD_KVM_RW_HW(entryhi,        l,  MIPS_CP0_TLB_HI,       0)
__BUILD_KVM_RW_HW(compare,        32, MIPS_CP0_COMPARE,      0)
__BUILD_KVM_RW_HW(status,         32, MIPS_CP0_STATUS,       0)
__BUILD_KVM_RW_HW(intctl,         32, MIPS_CP0_STATUS,       1)
__BUILD_KVM_RW_HW(cause,          32, MIPS_CP0_CAUSE,        0)
__BUILD_KVM_RW_HW(epc,            l,  MIPS_CP0_EXC_PC,       0)
__BUILD_KVM_RW_SW(prid,           32, MIPS_CP0_PRID,         0)
__BUILD_KVM_RW_HW(ebase,          l,  MIPS_CP0_PRID,         1)
__BUILD_KVM_RW_HW(config,         32, MIPS_CP0_CONFIG,       0)
__BUILD_KVM_RW_HW(config1,        32, MIPS_CP0_CONFIG,       1)
__BUILD_KVM_RW_HW(config2,        32, MIPS_CP0_CONFIG,       2)
__BUILD_KVM_RW_HW(config3,        32, MIPS_CP0_CONFIG,       3)
__BUILD_KVM_RW_HW(config4,        32, MIPS_CP0_CONFIG,       4)
__BUILD_KVM_RW_HW(config5,        32, MIPS_CP0_CONFIG,       5)
__BUILD_KVM_RW_HW(config6,        32, MIPS_CP0_CONFIG,       6)
__BUILD_KVM_RW_HW(config7,        32, MIPS_CP0_CONFIG,       7)
__BUILD_KVM_RW_SW(maari,          l,  MIPS_CP0_LLADDR,       2)
__BUILD_KVM_RW_HW(xcontext,       l,  MIPS_CP0_TLB_XCONTEXT, 0)
__BUILD_KVM_RW_HW(errorepc,       l,  MIPS_CP0_ERROR_PC,     0)
__BUILD_KVM_RW_HW(kscratch1,      l,  MIPS_CP0_DESAVE,       2)
__BUILD_KVM_RW_HW(kscratch2,      l,  MIPS_CP0_DESAVE,       3)
__BUILD_KVM_RW_HW(kscratch3,      l,  MIPS_CP0_DESAVE,       4)
__BUILD_KVM_RW_HW(kscratch4,      l,  MIPS_CP0_DESAVE,       5)
__BUILD_KVM_RW_HW(kscratch5,      l,  MIPS_CP0_DESAVE,       6)
__BUILD_KVM_RW_HW(kscratch6,      l,  MIPS_CP0_DESAVE,       7)

/* Bitwise operations (on HW state) */
__BUILD_KVM_SET_HW(status,        32, MIPS_CP0_STATUS,       0)
/* Cause can be modified asynchronously from hardirq hrtimer callback */
__BUILD_KVM_ATOMIC_HW(cause,      32, MIPS_CP0_CAUSE,        0)
__BUILD_KVM_SET_HW(ebase,         l,  MIPS_CP0_PRID,         1)

/* Bitwise operations (on saved state) */
__BUILD_KVM_SET_SAVED(config,     32, MIPS_CP0_CONFIG,       0)
__BUILD_KVM_SET_SAVED(config1,    32, MIPS_CP0_CONFIG,       1)
__BUILD_KVM_SET_SAVED(config2,    32, MIPS_CP0_CONFIG,       2)
__BUILD_KVM_SET_SAVED(config3,    32, MIPS_CP0_CONFIG,       3)
__BUILD_KVM_SET_SAVED(config4,    32, MIPS_CP0_CONFIG,       4)
__BUILD_KVM_SET_SAVED(config5,    32, MIPS_CP0_CONFIG,       5)

/* Helpers */

static inline bool kvm_mips_guest_can_have_fpu(struct kvm_vcpu_arch *vcpu)
{
	return (!__builtin_constant_p(raw_cpu_has_fpu) || raw_cpu_has_fpu) &&
		vcpu->fpu_enabled;
}

static inline bool kvm_mips_guest_has_fpu(struct kvm_vcpu_arch *vcpu)
{
	return kvm_mips_guest_can_have_fpu(vcpu) &&
		kvm_read_c0_guest_config1(vcpu->cop0) & MIPS_CONF1_FP;
}

static inline bool kvm_mips_guest_can_have_msa(struct kvm_vcpu_arch *vcpu)
{
	return (!__builtin_constant_p(cpu_has_msa) || cpu_has_msa) &&
		vcpu->msa_enabled;
}

static inline bool kvm_mips_guest_has_msa(struct kvm_vcpu_arch *vcpu)
{
	return kvm_mips_guest_can_have_msa(vcpu) &&
		kvm_read_c0_guest_config3(vcpu->cop0) & MIPS_CONF3_MSA;
}

struct kvm_mips_callbacks {
	int (*handle_cop_unusable)(struct kvm_vcpu *vcpu);
	int (*handle_tlb_mod)(struct kvm_vcpu *vcpu);
	int (*handle_tlb_ld_miss)(struct kvm_vcpu *vcpu);
	int (*handle_tlb_st_miss)(struct kvm_vcpu *vcpu);
	int (*handle_addr_err_st)(struct kvm_vcpu *vcpu);
	int (*handle_addr_err_ld)(struct kvm_vcpu *vcpu);
	int (*handle_syscall)(struct kvm_vcpu *vcpu);
	int (*handle_res_inst)(struct kvm_vcpu *vcpu);
	int (*handle_break)(struct kvm_vcpu *vcpu);
	int (*handle_trap)(struct kvm_vcpu *vcpu);
	int (*handle_msa_fpe)(struct kvm_vcpu *vcpu);
	int (*handle_fpe)(struct kvm_vcpu *vcpu);
	int (*handle_msa_disabled)(struct kvm_vcpu *vcpu);
	int (*handle_guest_exit)(struct kvm_vcpu *vcpu);
	int (*hardware_enable)(void);
	void (*hardware_disable)(void);
	int (*check_extension)(struct kvm *kvm, long ext);
	int (*vcpu_init)(struct kvm_vcpu *vcpu);
	void (*vcpu_uninit)(struct kvm_vcpu *vcpu);
	int (*vcpu_setup)(struct kvm_vcpu *vcpu);
	void (*prepare_flush_shadow)(struct kvm *kvm);
	gpa_t (*gva_to_gpa)(gva_t gva);
	void (*queue_timer_int)(struct kvm_vcpu *vcpu);
	void (*dequeue_timer_int)(struct kvm_vcpu *vcpu);
	void (*queue_io_int)(struct kvm_vcpu *vcpu,
			     struct kvm_mips_interrupt *irq);
	void (*dequeue_io_int)(struct kvm_vcpu *vcpu,
			       struct kvm_mips_interrupt *irq);
	int (*irq_deliver)(struct kvm_vcpu *vcpu, unsigned int priority,
			   u32 cause);
	int (*irq_clear)(struct kvm_vcpu *vcpu, unsigned int priority,
			 u32 cause);
	unsigned long (*num_regs)(struct kvm_vcpu *vcpu);
	int (*copy_reg_indices)(struct kvm_vcpu *vcpu, u64 __user *indices);
	int (*get_one_reg)(struct kvm_vcpu *vcpu,
			   const struct kvm_one_reg *reg, s64 *v);
	int (*set_one_reg)(struct kvm_vcpu *vcpu,
			   const struct kvm_one_reg *reg, s64 v);
	int (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu);
	int (*vcpu_put)(struct kvm_vcpu *vcpu, int cpu);
	int (*vcpu_run)(struct kvm_vcpu *vcpu);
	void (*vcpu_reenter)(struct kvm_vcpu *vcpu);
};
extern const struct kvm_mips_callbacks * const kvm_mips_callbacks;
int kvm_mips_emulation_init(void);

/* Debug: dump vcpu state */
int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu);

extern int kvm_mips_handle_exit(struct kvm_vcpu *vcpu);

/* Building of entry/exception code */
int kvm_mips_entry_setup(void);
void *kvm_mips_build_vcpu_run(void *addr);
void *kvm_mips_build_tlb_refill_exception(void *addr, void *handler);
void *kvm_mips_build_exception(void *addr, void *handler);
void *kvm_mips_build_exit(void *addr);

/* FPU/MSA context management */
void __kvm_save_fpu(struct kvm_vcpu_arch *vcpu);
void __kvm_restore_fpu(struct kvm_vcpu_arch *vcpu);
void __kvm_restore_fcsr(struct kvm_vcpu_arch *vcpu);
void __kvm_save_msa(struct kvm_vcpu_arch *vcpu);
void __kvm_restore_msa(struct kvm_vcpu_arch *vcpu);
void __kvm_restore_msa_upper(struct kvm_vcpu_arch *vcpu);
void __kvm_restore_msacsr(struct kvm_vcpu_arch *vcpu);
void kvm_own_fpu(struct kvm_vcpu *vcpu);
void kvm_own_msa(struct kvm_vcpu *vcpu);
void kvm_drop_fpu(struct kvm_vcpu *vcpu);
void kvm_lose_fpu(struct kvm_vcpu *vcpu);

/* TLB handling */
int kvm_mips_handle_vz_root_tlb_fault(unsigned long badvaddr,
				      struct kvm_vcpu *vcpu, bool write_fault);

int kvm_vz_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long entryhi);
int kvm_vz_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long gva,
			    unsigned long *gpa);
void kvm_vz_local_flush_roottlb_all_guests(void);
void kvm_vz_local_flush_guesttlb_all(void);
void kvm_vz_save_guesttlb(struct kvm_mips_tlb *buf, unsigned int index,
			  unsigned int count);
void kvm_vz_load_guesttlb(const struct kvm_mips_tlb *buf, unsigned int index,
			  unsigned int count);
#ifdef CONFIG_CPU_LOONGSON64
void kvm_loongson_clear_guest_vtlb(void);
void kvm_loongson_clear_guest_ftlb(void);
#endif

/* MMU handling */

bool kvm_mips_flush_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn);
int kvm_mips_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn);
pgd_t *kvm_pgd_alloc(void);
void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);

#define KVM_ARCH_WANT_MMU_NOTIFIER

/* Emulation */
enum emulation_result update_pc(struct kvm_vcpu *vcpu, u32 cause);
int kvm_get_badinstr(u32 *opc, struct kvm_vcpu *vcpu, u32 *out);
int kvm_get_badinstrp(u32 *opc, struct kvm_vcpu *vcpu, u32 *out);

/**
 * kvm_is_ifetch_fault() - Find whether a TLBL exception is due to ifetch fault.
 * @vcpu:	Virtual CPU.
 *
 * Returns:	Whether the TLBL exception was likely due to an instruction
 *		fetch fault rather than a data load fault.
 */
static inline bool kvm_is_ifetch_fault(struct kvm_vcpu_arch *vcpu)
{
	unsigned long badvaddr = vcpu->host_cp0_badvaddr;
	unsigned long epc = msk_isa16_mode(vcpu->pc);
	u32 cause = vcpu->host_cp0_cause;

	if (epc == badvaddr)
		return true;

	/*
	 * Branches may be 32-bit or 16-bit instructions.
	 * This isn't exact, but we don't really support MIPS16 or microMIPS yet
	 * in KVM anyway.
	 */
	if ((cause & CAUSEF_BD) && badvaddr - epc <= 4)
		return true;

	return false;
}

extern enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu);

u32 kvm_mips_read_count(struct kvm_vcpu *vcpu);
void kvm_mips_write_count(struct kvm_vcpu *vcpu, u32 count);
void kvm_mips_write_compare(struct kvm_vcpu *vcpu, u32 compare, bool ack);
void kvm_mips_init_count(struct kvm_vcpu *vcpu, unsigned long count_hz);
int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl);
int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume);
int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz);
void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu);
void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu);
enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu);

/* fairly internal functions requiring some care to use */
int kvm_mips_count_disabled(struct kvm_vcpu *vcpu);
ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu, u32 *count);
int kvm_mips_restore_hrtimer(struct kvm_vcpu *vcpu, ktime_t before,
			     u32 count, int min_drift);

void kvm_vz_acquire_htimer(struct kvm_vcpu *vcpu);
void kvm_vz_lose_htimer(struct kvm_vcpu *vcpu);

enum emulation_result kvm_mips_emulate_store(union mips_instruction inst,
					     u32 cause,
					     struct kvm_vcpu *vcpu);
enum emulation_result kvm_mips_emulate_load(union mips_instruction inst,
					    u32 cause,
					    struct kvm_vcpu *vcpu);

/* COP0 */
enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu);

/* Hypercalls (hypcall.c) */

enum emulation_result kvm_mips_emul_hypcall(struct kvm_vcpu *vcpu,
					    union mips_instruction inst);
int kvm_mips_handle_hypcall(struct kvm_vcpu *vcpu);

/* Misc */
extern void kvm_mips_dump_stats(struct kvm_vcpu *vcpu);
extern unsigned long kvm_mips_get_ramsize(struct kvm *kvm);
extern int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
			     struct kvm_mips_interrupt *irq);

static inline void kvm_arch_sync_events(struct kvm *kvm) {}
static inline void kvm_arch_free_memslot(struct kvm *kvm,
					 struct kvm_memory_slot *slot) {}
static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {}
static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}

#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB
int kvm_arch_flush_remote_tlb(struct kvm *kvm);

#endif /* __MIPS_KVM_HOST_H__ */