1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
|
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2020 ARM Ltd.
*/
#include <linux/linkage.h>
#include <asm/asm-uaccess.h>
#include <asm/assembler.h>
#include <asm/mte.h>
#include <asm/page.h>
#include <asm/sysreg.h>
.arch armv8.5-a+memtag
/*
* multitag_transfer_size - set \reg to the block size that is accessed by the
* LDGM/STGM instructions.
*/
.macro multitag_transfer_size, reg, tmp
mrs_s \reg, SYS_GMID_EL1
ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_WIDTH
mov \tmp, #4
lsl \reg, \tmp, \reg
.endm
/*
* Clear the tags in a page
* x0 - address of the page to be cleared
*/
SYM_FUNC_START(mte_clear_page_tags)
multitag_transfer_size x1, x2
1: stgm xzr, [x0]
add x0, x0, x1
tst x0, #(PAGE_SIZE - 1)
b.ne 1b
ret
SYM_FUNC_END(mte_clear_page_tags)
/*
* Zero the page and tags at the same time
*
* Parameters:
* x0 - address to the beginning of the page
*/
SYM_FUNC_START(mte_zero_clear_page_tags)
and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag
mrs x1, dczid_el0
tbnz x1, #4, 2f // Branch if DC GZVA is prohibited
and w1, w1, #0xf
mov x2, #4
lsl x1, x2, x1
1: dc gzva, x0
add x0, x0, x1
tst x0, #(PAGE_SIZE - 1)
b.ne 1b
ret
2: stz2g x0, [x0], #(MTE_GRANULE_SIZE * 2)
tst x0, #(PAGE_SIZE - 1)
b.ne 2b
ret
SYM_FUNC_END(mte_zero_clear_page_tags)
/*
* Copy the tags from the source page to the destination one
* x0 - address of the destination page
* x1 - address of the source page
*/
SYM_FUNC_START(mte_copy_page_tags)
mov x2, x0
mov x3, x1
multitag_transfer_size x5, x6
1: ldgm x4, [x3]
stgm x4, [x2]
add x2, x2, x5
add x3, x3, x5
tst x2, #(PAGE_SIZE - 1)
b.ne 1b
ret
SYM_FUNC_END(mte_copy_page_tags)
/*
* Read tags from a user buffer (one tag per byte) and set the corresponding
* tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
* x0 - kernel address (to)
* x1 - user buffer (from)
* x2 - number of tags/bytes (n)
* Returns:
* x0 - number of tags read/set
*/
SYM_FUNC_START(mte_copy_tags_from_user)
mov x3, x1
cbz x2, 2f
1:
USER(2f, ldtrb w4, [x1])
lsl x4, x4, #MTE_TAG_SHIFT
stg x4, [x0], #MTE_GRANULE_SIZE
add x1, x1, #1
subs x2, x2, #1
b.ne 1b
// exception handling and function return
2: sub x0, x1, x3 // update the number of tags set
ret
SYM_FUNC_END(mte_copy_tags_from_user)
/*
* Get the tags from a kernel address range and write the tag values to the
* given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
* x0 - user buffer (to)
* x1 - kernel address (from)
* x2 - number of tags/bytes (n)
* Returns:
* x0 - number of tags read/set
*/
SYM_FUNC_START(mte_copy_tags_to_user)
mov x3, x0
cbz x2, 2f
1:
ldg x4, [x1]
ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
USER(2f, sttrb w4, [x0])
add x0, x0, #1
add x1, x1, #MTE_GRANULE_SIZE
subs x2, x2, #1
b.ne 1b
// exception handling and function return
2: sub x0, x0, x3 // update the number of tags copied
ret
SYM_FUNC_END(mte_copy_tags_to_user)
/*
* Save the tags in a page
* x0 - page address
* x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes
*/
SYM_FUNC_START(mte_save_page_tags)
multitag_transfer_size x7, x5
1:
mov x2, #0
2:
ldgm x5, [x0]
orr x2, x2, x5
add x0, x0, x7
tst x0, #0xFF // 16 tag values fit in a register,
b.ne 2b // which is 16*16=256 bytes
str x2, [x1], #8
tst x0, #(PAGE_SIZE - 1)
b.ne 1b
ret
SYM_FUNC_END(mte_save_page_tags)
/*
* Restore the tags in a page
* x0 - page address
* x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes
*/
SYM_FUNC_START(mte_restore_page_tags)
multitag_transfer_size x7, x5
1:
ldr x2, [x1], #8
2:
stgm x2, [x0]
add x0, x0, x7
tst x0, #0xFF
b.ne 2b
tst x0, #(PAGE_SIZE - 1)
b.ne 1b
ret
SYM_FUNC_END(mte_restore_page_tags)
|
