1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
// SPDX-License-Identifier: GPL-2.0
//! Allocator support.
use core::alloc::AllocError;
use core::alloc::{GlobalAlloc, Layout};
use core::ptr;
use core::ptr::NonNull;
use crate::bindings;
pub(crate) struct KernelAllocator;
impl KernelAllocator {
#[cfg(not(test))]
#[cfg(not(testlib))]
pub(crate) fn allocate_with_flags(
&self,
layout: Layout,
flags: bindings::gfp_t,
) -> Result<NonNull<[u8]>, AllocError> {
// `krealloc()` is used instead of `kmalloc()` because the latter is
// an inline function and cannot be bound to as a result.
let mem = unsafe { bindings::krealloc(ptr::null(), layout.size(), flags) as *mut u8 };
if mem.is_null() {
return Err(AllocError);
}
let mem = unsafe { core::slice::from_raw_parts_mut(mem, bindings::ksize(mem as _)) };
// Safety: checked for non null above
Ok(unsafe { NonNull::new_unchecked(mem) })
}
#[cfg(test)]
#[cfg(testlib)]
pub(crate) fn allocate_with_flags(
&self,
layout: Layout,
_flags: bindings::gfp_t,
) -> Result<NonNull<[u8]>, AllocError> {
self.allocate(layout)
}
}
/// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment.
///
/// # Safety
///
/// - `ptr` can be either null or a pointer which has been allocated by this allocator.
/// - `new_layout` must have a non-zero size.
unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: bindings::gfp_t) -> *mut u8 {
// Customized layouts from `Layout::from_size_align()` can have size < align, so pad first.
let layout = new_layout.pad_to_align();
let mut size = layout.size();
if layout.align() > bindings::BINDINGS_ARCH_SLAB_MINALIGN {
// The alignment requirement exceeds the slab guarantee, thus try to enlarge the size
// to use the "power-of-two" size/alignment guarantee (see comments in `kmalloc()` for
// more information).
//
// Note that `layout.size()` (after padding) is guaranteed to be a multiple of
// `layout.align()`, so `next_power_of_two` gives enough alignment guarantee.
size = size.next_power_of_two();
}
// SAFETY:
// - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the
// function safety requirement.
// - `size` is greater than 0 since it's either a `layout.size()` (which cannot be zero
// according to the function safety requirement) or a result from `next_power_of_two()`.
unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags) as *mut u8 }
}
unsafe impl GlobalAlloc for KernelAllocator {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
// SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety
// requirement.
unsafe { krealloc_aligned(ptr::null_mut(), layout, bindings::GFP_KERNEL) }
}
unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
unsafe {
bindings::kfree(ptr as *const core::ffi::c_void);
}
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
// SAFETY:
// - `new_size`, when rounded up to the nearest multiple of `layout.align()`, will not
// overflow `isize` by the function safety requirement.
// - `layout.align()` is a proper alignment (i.e. not zero and must be a power of two).
let layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
// SAFETY:
// - `ptr` is either null or a pointer allocated by this allocator by the function safety
// requirement.
// - the size of `layout` is not zero because `new_size` is not zero by the function safety
// requirement.
unsafe { krealloc_aligned(ptr, layout, bindings::GFP_KERNEL) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
// SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety
// requirement.
unsafe {
krealloc_aligned(
ptr::null_mut(),
layout,
bindings::GFP_KERNEL | bindings::__GFP_ZERO,
)
}
}
}
#[global_allocator]
pub(crate) static ALLOCATOR: KernelAllocator = KernelAllocator;
// See <https://github.com/rust-lang/rust/pull/86844>.
#[no_mangle]
static __rust_no_alloc_shim_is_unstable: u8 = 0;
|
