1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2012 ARM Ltd.
4  * Author: Catalin Marinas <[email protected]>
5  * Copyright (C) 2017 Linaro Ltd. <[email protected]>
6  * Copyright (C) 2021 SiFive
7  */
8 #include <linux/compiler.h>
9 #include <linux/irqflags.h>
10 #include <linux/percpu.h>
11 #include <linux/preempt.h>
12 #include <linux/types.h>
13 
14 #include <asm/vector.h>
15 #include <asm/switch_to.h>
16 #include <asm/simd.h>
17 
18 static inline void riscv_v_flags_set(u32 flags)
19 {
20 	current->thread.riscv_v_flags = flags;
21 }
22 
23 static inline void riscv_v_start(u32 flags)
24 {
25 	int orig;
26 
27 	orig = riscv_v_flags();
28 	BUG_ON((orig & flags) != 0);
29 	riscv_v_flags_set(orig | flags);
30 }
31 
32 static inline void riscv_v_stop(u32 flags)
33 {
34 	int orig;
35 
36 	orig = riscv_v_flags();
37 	BUG_ON((orig & flags) == 0);
38 	riscv_v_flags_set(orig & ~flags);
39 }
40 
41 /*
42  * Claim ownership of the CPU vector context for use by the calling context.
43  *
44  * The caller may freely manipulate the vector context metadata until
45  * put_cpu_vector_context() is called.
46  */
47 void get_cpu_vector_context(void)
48 {
49 	preempt_disable();
50 
51 	riscv_v_start(RISCV_KERNEL_MODE_V);
52 }
53 
54 /*
55  * Release the CPU vector context.
56  *
57  * Must be called from a context in which get_cpu_vector_context() was
58  * previously called, with no call to put_cpu_vector_context() in the
59  * meantime.
60  */
61 void put_cpu_vector_context(void)
62 {
63 	riscv_v_stop(RISCV_KERNEL_MODE_V);
64 
65 	preempt_enable();
66 }
67 
68 /*
69  * kernel_vector_begin(): obtain the CPU vector registers for use by the calling
70  * context
71  *
72  * Must not be called unless may_use_simd() returns true.
73  * Task context in the vector registers is saved back to memory as necessary.
74  *
75  * A matching call to kernel_vector_end() must be made before returning from the
76  * calling context.
77  *
78  * The caller may freely use the vector registers until kernel_vector_end() is
79  * called.
80  */
81 void kernel_vector_begin(void)
82 {
83 	if (WARN_ON(!has_vector()))
84 		return;
85 
86 	BUG_ON(!may_use_simd());
87 
88 	get_cpu_vector_context();
89 
90 	riscv_v_vstate_save(current, task_pt_regs(current));
91 
92 	riscv_v_enable();
93 }
94 EXPORT_SYMBOL_GPL(kernel_vector_begin);
95 
96 /*
97  * kernel_vector_end(): give the CPU vector registers back to the current task
98  *
99  * Must be called from a context in which kernel_vector_begin() was previously
100  * called, with no call to kernel_vector_end() in the meantime.
101  *
102  * The caller must not use the vector registers after this function is called,
103  * unless kernel_vector_begin() is called again in the meantime.
104  */
105 void kernel_vector_end(void)
106 {
107 	if (WARN_ON(!has_vector()))
108 		return;
109 
110 	riscv_v_vstate_restore(current, task_pt_regs(current));
111 
112 	riscv_v_disable();
113 
114 	put_cpu_vector_context();
115 }
116 EXPORT_SYMBOL_GPL(kernel_vector_end);
117