1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2012 ARM Ltd.
4  * Author: Catalin Marinas <[email protected]>
5  * Copyright (C) 2017 Linaro Ltd. <[email protected]>
6  * Copyright (C) 2021 SiFive
7  */
8 #include <linux/compiler.h>
9 #include <linux/irqflags.h>
10 #include <linux/percpu.h>
11 #include <linux/preempt.h>
12 #include <linux/types.h>
13 
14 #include <asm/vector.h>
15 #include <asm/switch_to.h>
16 #include <asm/simd.h>
17 
18 static inline void riscv_v_flags_set(u32 flags)
19 {
20 	current->thread.riscv_v_flags = flags;
21 }
22 
23 static inline void riscv_v_start(u32 flags)
24 {
25 	int orig;
26 
27 	orig = riscv_v_flags();
28 	BUG_ON((orig & flags) != 0);
29 	riscv_v_flags_set(orig | flags);
30 }
31 
32 static inline void riscv_v_stop(u32 flags)
33 {
34 	int orig;
35 
36 	orig = riscv_v_flags();
37 	BUG_ON((orig & flags) == 0);
38 	riscv_v_flags_set(orig & ~flags);
39 }
40 
41 /*
42  * Claim ownership of the CPU vector context for use by the calling context.
43  *
44  * The caller may freely manipulate the vector context metadata until
45  * put_cpu_vector_context() is called.
46  */
47 void get_cpu_vector_context(void)
48 {
49 	/*
50 	 * disable softirqs so it is impossible for softirqs to nest
51 	 * get_cpu_vector_context() when kernel is actively using Vector.
52 	 */
53 	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
54 		local_bh_disable();
55 	else
56 		preempt_disable();
57 
58 	riscv_v_start(RISCV_KERNEL_MODE_V);
59 }
60 
61 /*
62  * Release the CPU vector context.
63  *
64  * Must be called from a context in which get_cpu_vector_context() was
65  * previously called, with no call to put_cpu_vector_context() in the
66  * meantime.
67  */
68 void put_cpu_vector_context(void)
69 {
70 	riscv_v_stop(RISCV_KERNEL_MODE_V);
71 
72 	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
73 		local_bh_enable();
74 	else
75 		preempt_enable();
76 }
77 
78 /*
79  * kernel_vector_begin(): obtain the CPU vector registers for use by the calling
80  * context
81  *
82  * Must not be called unless may_use_simd() returns true.
83  * Task context in the vector registers is saved back to memory as necessary.
84  *
85  * A matching call to kernel_vector_end() must be made before returning from the
86  * calling context.
87  *
88  * The caller may freely use the vector registers until kernel_vector_end() is
89  * called.
90  */
91 void kernel_vector_begin(void)
92 {
93 	if (WARN_ON(!has_vector()))
94 		return;
95 
96 	BUG_ON(!may_use_simd());
97 
98 	get_cpu_vector_context();
99 
100 	riscv_v_vstate_save(current, task_pt_regs(current));
101 
102 	riscv_v_enable();
103 }
104 EXPORT_SYMBOL_GPL(kernel_vector_begin);
105 
106 /*
107  * kernel_vector_end(): give the CPU vector registers back to the current task
108  *
109  * Must be called from a context in which kernel_vector_begin() was previously
110  * called, with no call to kernel_vector_end() in the meantime.
111  *
112  * The caller must not use the vector registers after this function is called,
113  * unless kernel_vector_begin() is called again in the meantime.
114  */
115 void kernel_vector_end(void)
116 {
117 	if (WARN_ON(!has_vector()))
118 		return;
119 
120 	riscv_v_vstate_set_restore(current, task_pt_regs(current));
121 
122 	riscv_v_disable();
123 
124 	put_cpu_vector_context();
125 }
126 EXPORT_SYMBOL_GPL(kernel_vector_end);
127