1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2013 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <[email protected]>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include "opt_acpi.h"
35
36 #include <sys/param.h>
37 #include <sys/bus.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 #include <sys/memdesc.h>
41 #include <sys/module.h>
42 #include <sys/rman.h>
43 #include <sys/taskqueue.h>
44 #include <sys/time.h>
45 #include <sys/tree.h>
46 #include <sys/vmem.h>
47 #include <vm/vm.h>
48 #include <vm/vm_extern.h>
49 #include <vm/vm_kern.h>
50 #include <vm/vm_page.h>
51 #include <vm/vm_map.h>
52 #include <contrib/dev/acpica/include/acpi.h>
53 #include <contrib/dev/acpica/include/accommon.h>
54 #include <dev/acpica/acpivar.h>
55 #include <dev/pci/pcireg.h>
56 #include <machine/bus.h>
57 #include <machine/cpu.h>
58 #include <x86/include/busdma_impl.h>
59 #include <dev/iommu/busdma_iommu.h>
60 #include <x86/iommu/intel_reg.h>
61 #include <x86/iommu/intel_dmar.h>
62
63 static bool
dmar_qi_seq_processed(const struct dmar_unit * unit,const struct iommu_qi_genseq * pseq)64 dmar_qi_seq_processed(const struct dmar_unit *unit,
65 const struct iommu_qi_genseq *pseq)
66 {
67
68 return (pseq->gen < unit->inv_waitd_gen ||
69 (pseq->gen == unit->inv_waitd_gen &&
70 pseq->seq <= unit->inv_waitd_seq_hw));
71 }
72
73 static int
dmar_enable_qi(struct dmar_unit * unit)74 dmar_enable_qi(struct dmar_unit *unit)
75 {
76 int error;
77
78 DMAR_ASSERT_LOCKED(unit);
79 unit->hw_gcmd |= DMAR_GCMD_QIE;
80 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
81 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
82 != 0));
83 return (error);
84 }
85
86 static int
dmar_disable_qi(struct dmar_unit * unit)87 dmar_disable_qi(struct dmar_unit *unit)
88 {
89 int error;
90
91 DMAR_ASSERT_LOCKED(unit);
92 unit->hw_gcmd &= ~DMAR_GCMD_QIE;
93 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
94 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
95 == 0));
96 return (error);
97 }
98
99 static void
dmar_qi_advance_tail(struct dmar_unit * unit)100 dmar_qi_advance_tail(struct dmar_unit *unit)
101 {
102
103 DMAR_ASSERT_LOCKED(unit);
104 dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
105 }
106
107 static void
dmar_qi_ensure(struct dmar_unit * unit,int descr_count)108 dmar_qi_ensure(struct dmar_unit *unit, int descr_count)
109 {
110 uint32_t head;
111 int bytes;
112
113 DMAR_ASSERT_LOCKED(unit);
114 bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT;
115 for (;;) {
116 if (bytes <= unit->inv_queue_avail)
117 break;
118 /* refill */
119 head = dmar_read4(unit, DMAR_IQH_REG);
120 head &= DMAR_IQH_MASK;
121 unit->inv_queue_avail = head - unit->inv_queue_tail -
122 DMAR_IQ_DESCR_SZ;
123 if (head <= unit->inv_queue_tail)
124 unit->inv_queue_avail += unit->inv_queue_size;
125 if (bytes <= unit->inv_queue_avail)
126 break;
127
128 /*
129 * No space in the queue, do busy wait. Hardware must
130 * make a progress. But first advance the tail to
131 * inform the descriptor streamer about entries we
132 * might have already filled, otherwise they could
133 * clog the whole queue..
134 */
135 dmar_qi_advance_tail(unit);
136 unit->inv_queue_full++;
137 cpu_spinwait();
138 }
139 unit->inv_queue_avail -= bytes;
140 }
141
142 static void
dmar_qi_emit(struct dmar_unit * unit,uint64_t data1,uint64_t data2)143 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
144 {
145
146 DMAR_ASSERT_LOCKED(unit);
147 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1;
148 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
149 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
150 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
151 (uintmax_t)unit->inv_queue_size));
152 unit->inv_queue_tail &= unit->inv_queue_size - 1;
153 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2;
154 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
155 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
156 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
157 (uintmax_t)unit->inv_queue_size));
158 unit->inv_queue_tail &= unit->inv_queue_size - 1;
159 }
160
161 static void
dmar_qi_emit_wait_descr(struct dmar_unit * unit,uint32_t seq,bool intr,bool memw,bool fence)162 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
163 bool memw, bool fence)
164 {
165
166 DMAR_ASSERT_LOCKED(unit);
167 dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID |
168 (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) |
169 (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) |
170 (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) |
171 (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
172 memw ? unit->inv_waitd_seq_hw_phys : 0);
173 }
174
175 static void
dmar_qi_emit_wait_seq(struct dmar_unit * unit,struct iommu_qi_genseq * pseq,bool emit_wait)176 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct iommu_qi_genseq *pseq,
177 bool emit_wait)
178 {
179 struct iommu_qi_genseq gsec;
180 uint32_t seq;
181
182 KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
183 DMAR_ASSERT_LOCKED(unit);
184 if (unit->inv_waitd_seq == 0xffffffff) {
185 gsec.gen = unit->inv_waitd_gen;
186 gsec.seq = unit->inv_waitd_seq;
187 dmar_qi_ensure(unit, 1);
188 dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false);
189 dmar_qi_advance_tail(unit);
190 while (!dmar_qi_seq_processed(unit, &gsec))
191 cpu_spinwait();
192 unit->inv_waitd_gen++;
193 unit->inv_waitd_seq = 1;
194 }
195 seq = unit->inv_waitd_seq++;
196 pseq->gen = unit->inv_waitd_gen;
197 pseq->seq = seq;
198 if (emit_wait) {
199 dmar_qi_ensure(unit, 1);
200 dmar_qi_emit_wait_descr(unit, seq, true, true, false);
201 }
202 }
203
204 static void
dmar_qi_wait_for_seq(struct dmar_unit * unit,const struct iommu_qi_genseq * gseq,bool nowait)205 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct iommu_qi_genseq *gseq,
206 bool nowait)
207 {
208
209 DMAR_ASSERT_LOCKED(unit);
210 unit->inv_seq_waiters++;
211 while (!dmar_qi_seq_processed(unit, gseq)) {
212 if (cold || nowait) {
213 cpu_spinwait();
214 } else {
215 msleep(&unit->inv_seq_waiters, &unit->iommu.lock, 0,
216 "dmarse", hz);
217 }
218 }
219 unit->inv_seq_waiters--;
220 }
221
222 void
dmar_qi_invalidate_locked(struct dmar_domain * domain,iommu_gaddr_t base,iommu_gaddr_t size,struct iommu_qi_genseq * pseq,bool emit_wait)223 dmar_qi_invalidate_locked(struct dmar_domain *domain, iommu_gaddr_t base,
224 iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait)
225 {
226 struct dmar_unit *unit;
227 iommu_gaddr_t isize;
228 int am;
229
230 unit = domain->dmar;
231 DMAR_ASSERT_LOCKED(unit);
232 for (; size > 0; base += isize, size -= isize) {
233 am = calc_am(unit, base, size, &isize);
234 dmar_qi_ensure(unit, 1);
235 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV |
236 DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW |
237 DMAR_IQ_DESCR_IOTLB_DR |
238 DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
239 base | am);
240 }
241 dmar_qi_emit_wait_seq(unit, pseq, emit_wait);
242 dmar_qi_advance_tail(unit);
243 }
244
245 void
dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit * unit)246 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
247 {
248 struct iommu_qi_genseq gseq;
249
250 DMAR_ASSERT_LOCKED(unit);
251 dmar_qi_ensure(unit, 2);
252 dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
253 dmar_qi_emit_wait_seq(unit, &gseq, true);
254 dmar_qi_advance_tail(unit);
255 dmar_qi_wait_for_seq(unit, &gseq, false);
256 }
257
258 void
dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit * unit)259 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
260 {
261 struct iommu_qi_genseq gseq;
262
263 DMAR_ASSERT_LOCKED(unit);
264 dmar_qi_ensure(unit, 2);
265 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
266 DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
267 dmar_qi_emit_wait_seq(unit, &gseq, true);
268 dmar_qi_advance_tail(unit);
269 dmar_qi_wait_for_seq(unit, &gseq, false);
270 }
271
272 void
dmar_qi_invalidate_iec_glob(struct dmar_unit * unit)273 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
274 {
275 struct iommu_qi_genseq gseq;
276
277 DMAR_ASSERT_LOCKED(unit);
278 dmar_qi_ensure(unit, 2);
279 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
280 dmar_qi_emit_wait_seq(unit, &gseq, true);
281 dmar_qi_advance_tail(unit);
282 dmar_qi_wait_for_seq(unit, &gseq, false);
283 }
284
285 void
dmar_qi_invalidate_iec(struct dmar_unit * unit,u_int start,u_int cnt)286 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
287 {
288 struct iommu_qi_genseq gseq;
289 u_int c, l;
290
291 DMAR_ASSERT_LOCKED(unit);
292 KASSERT(start < unit->irte_cnt && start < start + cnt &&
293 start + cnt <= unit->irte_cnt,
294 ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
295 for (; cnt > 0; cnt -= c, start += c) {
296 l = ffs(start | cnt) - 1;
297 c = 1 << l;
298 dmar_qi_ensure(unit, 1);
299 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV |
300 DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) |
301 DMAR_IQ_DESCR_IEC_IM(l), 0);
302 }
303 dmar_qi_ensure(unit, 1);
304 dmar_qi_emit_wait_seq(unit, &gseq, true);
305 dmar_qi_advance_tail(unit);
306
307 /*
308 * The caller of the function, in particular,
309 * dmar_ir_program_irte(), may be called from the context
310 * where the sleeping is forbidden (in fact, the
311 * intr_table_lock mutex may be held, locked from
312 * intr_shuffle_irqs()). Wait for the invalidation completion
313 * using the busy wait.
314 *
315 * The impact on the interrupt input setup code is small, the
316 * expected overhead is comparable with the chipset register
317 * read. It is more harmful for the parallel DMA operations,
318 * since we own the dmar unit lock until whole invalidation
319 * queue is processed, which includes requests possibly issued
320 * before our request.
321 */
322 dmar_qi_wait_for_seq(unit, &gseq, true);
323 }
324
325 int
dmar_qi_intr(void * arg)326 dmar_qi_intr(void *arg)
327 {
328 struct dmar_unit *unit;
329
330 unit = arg;
331 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled",
332 unit->iommu.unit));
333 taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);
334 return (FILTER_HANDLED);
335 }
336
337 static void
dmar_qi_task(void * arg,int pending __unused)338 dmar_qi_task(void *arg, int pending __unused)
339 {
340 struct dmar_unit *unit;
341 struct iommu_map_entry *entry;
342 uint32_t ics;
343
344 unit = arg;
345
346 DMAR_LOCK(unit);
347 for (;;) {
348 entry = TAILQ_FIRST(&unit->tlb_flush_entries);
349 if (entry == NULL)
350 break;
351 if (!dmar_qi_seq_processed(unit, &entry->gseq))
352 break;
353 TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
354 DMAR_UNLOCK(unit);
355 dmar_domain_free_entry(entry, (entry->flags &
356 IOMMU_MAP_ENTRY_QI_NF) == 0);
357 DMAR_LOCK(unit);
358 }
359 ics = dmar_read4(unit, DMAR_ICS_REG);
360 if ((ics & DMAR_ICS_IWC) != 0) {
361 ics = DMAR_ICS_IWC;
362 dmar_write4(unit, DMAR_ICS_REG, ics);
363 }
364 if (unit->inv_seq_waiters > 0)
365 wakeup(&unit->inv_seq_waiters);
366 DMAR_UNLOCK(unit);
367 }
368
369 int
dmar_init_qi(struct dmar_unit * unit)370 dmar_init_qi(struct dmar_unit *unit)
371 {
372 uint64_t iqa;
373 uint32_t ics;
374 int qi_sz;
375
376 if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0)
377 return (0);
378 unit->qi_enabled = 1;
379 TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
380 if (!unit->qi_enabled)
381 return (0);
382
383 TAILQ_INIT(&unit->tlb_flush_entries);
384 TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
385 unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,
386 taskqueue_thread_enqueue, &unit->qi_taskqueue);
387 taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
388 "dmar%d qi taskq", unit->iommu.unit);
389
390 unit->inv_waitd_gen = 0;
391 unit->inv_waitd_seq = 1;
392
393 qi_sz = DMAR_IQA_QS_DEF;
394 TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
395 if (qi_sz > DMAR_IQA_QS_MAX)
396 qi_sz = DMAR_IQA_QS_MAX;
397 unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
398 /* Reserve one descriptor to prevent wraparound. */
399 unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ;
400
401 /* The invalidation queue reads by DMARs are always coherent. */
402 unit->inv_queue = kmem_alloc_contig(unit->inv_queue_size, M_WAITOK |
403 M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
404 unit->inv_waitd_seq_hw_phys = pmap_kextract(
405 (vm_offset_t)&unit->inv_waitd_seq_hw);
406
407 DMAR_LOCK(unit);
408 dmar_write8(unit, DMAR_IQT_REG, 0);
409 iqa = pmap_kextract(unit->inv_queue);
410 iqa |= qi_sz;
411 dmar_write8(unit, DMAR_IQA_REG, iqa);
412 dmar_enable_qi(unit);
413 ics = dmar_read4(unit, DMAR_ICS_REG);
414 if ((ics & DMAR_ICS_IWC) != 0) {
415 ics = DMAR_ICS_IWC;
416 dmar_write4(unit, DMAR_ICS_REG, ics);
417 }
418 dmar_enable_qi_intr(unit);
419 DMAR_UNLOCK(unit);
420
421 return (0);
422 }
423
424 void
dmar_fini_qi(struct dmar_unit * unit)425 dmar_fini_qi(struct dmar_unit *unit)
426 {
427 struct iommu_qi_genseq gseq;
428
429 if (!unit->qi_enabled)
430 return;
431 taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);
432 taskqueue_free(unit->qi_taskqueue);
433 unit->qi_taskqueue = NULL;
434
435 DMAR_LOCK(unit);
436 /* quisce */
437 dmar_qi_ensure(unit, 1);
438 dmar_qi_emit_wait_seq(unit, &gseq, true);
439 dmar_qi_advance_tail(unit);
440 dmar_qi_wait_for_seq(unit, &gseq, false);
441 /* only after the quisce, disable queue */
442 dmar_disable_qi_intr(unit);
443 dmar_disable_qi(unit);
444 KASSERT(unit->inv_seq_waiters == 0,
445 ("dmar%d: waiters on disabled queue", unit->iommu.unit));
446 DMAR_UNLOCK(unit);
447
448 kmem_free(unit->inv_queue, unit->inv_queue_size);
449 unit->inv_queue = 0;
450 unit->inv_queue_size = 0;
451 unit->qi_enabled = 0;
452 }
453
454 void
dmar_enable_qi_intr(struct dmar_unit * unit)455 dmar_enable_qi_intr(struct dmar_unit *unit)
456 {
457 uint32_t iectl;
458
459 DMAR_ASSERT_LOCKED(unit);
460 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
461 unit->iommu.unit));
462 iectl = dmar_read4(unit, DMAR_IECTL_REG);
463 iectl &= ~DMAR_IECTL_IM;
464 dmar_write4(unit, DMAR_IECTL_REG, iectl);
465 }
466
467 void
dmar_disable_qi_intr(struct dmar_unit * unit)468 dmar_disable_qi_intr(struct dmar_unit *unit)
469 {
470 uint32_t iectl;
471
472 DMAR_ASSERT_LOCKED(unit);
473 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
474 unit->iommu.unit));
475 iectl = dmar_read4(unit, DMAR_IECTL_REG);
476 dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);
477 }
478