1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (c) 2003
5 * Bill Paul <[email protected]>. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed by Bill Paul.
18 * 4. Neither the name of the author nor the names of any co-contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
37
38 #include <sys/ctype.h>
39 #include <sys/unistd.h>
40 #include <sys/param.h>
41 #include <sys/types.h>
42 #include <sys/errno.h>
43 #include <sys/systm.h>
44 #include <sys/malloc.h>
45 #include <sys/lock.h>
46 #include <sys/mutex.h>
47
48 #include <sys/callout.h>
49 #include <sys/kdb.h>
50 #include <sys/kernel.h>
51 #include <sys/proc.h>
52 #include <sys/condvar.h>
53 #include <sys/kthread.h>
54 #include <sys/module.h>
55 #include <sys/smp.h>
56 #include <sys/sched.h>
57 #include <sys/sysctl.h>
58
59 #include <machine/atomic.h>
60 #include <machine/bus.h>
61 #include <machine/stdarg.h>
62 #include <machine/resource.h>
63
64 #include <sys/bus.h>
65 #include <sys/rman.h>
66
67 #include <vm/vm.h>
68 #include <vm/vm_param.h>
69 #include <vm/pmap.h>
70 #include <vm/uma.h>
71 #include <vm/vm_kern.h>
72 #include <vm/vm_map.h>
73 #include <vm/vm_extern.h>
74
75 #include <compat/ndis/pe_var.h>
76 #include <compat/ndis/cfg_var.h>
77 #include <compat/ndis/resource_var.h>
78 #include <compat/ndis/ntoskrnl_var.h>
79 #include <compat/ndis/hal_var.h>
80 #include <compat/ndis/ndis_var.h>
81
82 #ifdef NTOSKRNL_DEBUG_TIMERS
83 static int sysctl_show_timers(SYSCTL_HANDLER_ARGS);
84
85 SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers, CTLTYPE_INT | CTLFLAG_RW,
86 NULL, 0, sysctl_show_timers, "I",
87 "Show ntoskrnl timer stats");
88 #endif
89
90 struct kdpc_queue {
91 list_entry kq_disp;
92 struct thread *kq_td;
93 int kq_cpu;
94 int kq_exit;
95 int kq_running;
96 kspin_lock kq_lock;
97 nt_kevent kq_proc;
98 nt_kevent kq_done;
99 };
100
101 typedef struct kdpc_queue kdpc_queue;
102
103 struct wb_ext {
104 struct cv we_cv;
105 struct thread *we_td;
106 };
107
108 typedef struct wb_ext wb_ext;
109
110 #define NTOSKRNL_TIMEOUTS 256
111 #ifdef NTOSKRNL_DEBUG_TIMERS
112 static uint64_t ntoskrnl_timer_fires;
113 static uint64_t ntoskrnl_timer_sets;
114 static uint64_t ntoskrnl_timer_reloads;
115 static uint64_t ntoskrnl_timer_cancels;
116 #endif
117
118 struct callout_entry {
119 struct callout ce_callout;
120 list_entry ce_list;
121 };
122
123 typedef struct callout_entry callout_entry;
124
125 static struct list_entry ntoskrnl_calllist;
126 static struct mtx ntoskrnl_calllock;
127 struct kuser_shared_data kuser_shared_data;
128
129 static struct list_entry ntoskrnl_intlist;
130 static kspin_lock ntoskrnl_intlock;
131
132 static uint8_t RtlEqualUnicodeString(unicode_string *,
133 unicode_string *, uint8_t);
134 static void RtlCopyString(ansi_string *, const ansi_string *);
135 static void RtlCopyUnicodeString(unicode_string *,
136 unicode_string *);
137 static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *,
138 void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *);
139 static irp *IoBuildAsynchronousFsdRequest(uint32_t,
140 device_object *, void *, uint32_t, uint64_t *, io_status_block *);
141 static irp *IoBuildDeviceIoControlRequest(uint32_t,
142 device_object *, void *, uint32_t, void *, uint32_t,
143 uint8_t, nt_kevent *, io_status_block *);
144 static irp *IoAllocateIrp(uint8_t, uint8_t);
145 static void IoReuseIrp(irp *, uint32_t);
146 static void IoFreeIrp(irp *);
147 static void IoInitializeIrp(irp *, uint16_t, uint8_t);
148 static irp *IoMakeAssociatedIrp(irp *, uint8_t);
149 static uint32_t KeWaitForMultipleObjects(uint32_t,
150 nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
151 int64_t *, wait_block *);
152 static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t);
153 static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *);
154 static void ntoskrnl_satisfy_multiple_waits(wait_block *);
155 static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *);
156 static void ntoskrnl_insert_timer(ktimer *, int);
157 static void ntoskrnl_remove_timer(ktimer *);
158 #ifdef NTOSKRNL_DEBUG_TIMERS
159 static void ntoskrnl_show_timers(void);
160 #endif
161 static void ntoskrnl_timercall(void *);
162 static void ntoskrnl_dpc_thread(void *);
163 static void ntoskrnl_destroy_dpc_threads(void);
164 static void ntoskrnl_destroy_workitem_threads(void);
165 static void ntoskrnl_workitem_thread(void *);
166 static void ntoskrnl_workitem(device_object *, void *);
167 static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int);
168 static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int);
169 static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *);
170 static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t);
171 static uint16_t READ_REGISTER_USHORT(uint16_t *);
172 static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t);
173 static uint32_t READ_REGISTER_ULONG(uint32_t *);
174 static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t);
175 static uint8_t READ_REGISTER_UCHAR(uint8_t *);
176 static int64_t _allmul(int64_t, int64_t);
177 static int64_t _alldiv(int64_t, int64_t);
178 static int64_t _allrem(int64_t, int64_t);
179 static int64_t _allshr(int64_t, uint8_t);
180 static int64_t _allshl(int64_t, uint8_t);
181 static uint64_t _aullmul(uint64_t, uint64_t);
182 static uint64_t _aulldiv(uint64_t, uint64_t);
183 static uint64_t _aullrem(uint64_t, uint64_t);
184 static uint64_t _aullshr(uint64_t, uint8_t);
185 static uint64_t _aullshl(uint64_t, uint8_t);
186 static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
187 static void InitializeSListHead(slist_header *);
188 static slist_entry *ntoskrnl_popsl(slist_header *);
189 static void ExFreePoolWithTag(void *, uint32_t);
190 static void ExInitializePagedLookasideList(paged_lookaside_list *,
191 lookaside_alloc_func *, lookaside_free_func *,
192 uint32_t, size_t, uint32_t, uint16_t);
193 static void ExDeletePagedLookasideList(paged_lookaside_list *);
194 static void ExInitializeNPagedLookasideList(npaged_lookaside_list *,
195 lookaside_alloc_func *, lookaside_free_func *,
196 uint32_t, size_t, uint32_t, uint16_t);
197 static void ExDeleteNPagedLookasideList(npaged_lookaside_list *);
198 static slist_entry
199 *ExInterlockedPushEntrySList(slist_header *,
200 slist_entry *, kspin_lock *);
201 static slist_entry
202 *ExInterlockedPopEntrySList(slist_header *, kspin_lock *);
203 static uint32_t InterlockedIncrement(volatile uint32_t *);
204 static uint32_t InterlockedDecrement(volatile uint32_t *);
205 static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t);
206 static void *MmAllocateContiguousMemory(uint32_t, uint64_t);
207 static void *MmAllocateContiguousMemorySpecifyCache(uint32_t,
208 uint64_t, uint64_t, uint64_t, enum nt_caching_type);
209 static void MmFreeContiguousMemory(void *);
210 static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t,
211 enum nt_caching_type);
212 static uint32_t MmSizeOfMdl(void *, size_t);
213 static void *MmMapLockedPages(mdl *, uint8_t);
214 static void *MmMapLockedPagesSpecifyCache(mdl *,
215 uint8_t, uint32_t, void *, uint32_t, uint32_t);
216 static void MmUnmapLockedPages(void *, mdl *);
217 static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **);
218 static void RtlZeroMemory(void *, size_t);
219 static void RtlSecureZeroMemory(void *, size_t);
220 static void RtlFillMemory(void *, size_t, uint8_t);
221 static void RtlMoveMemory(void *, const void *, size_t);
222 static ndis_status RtlCharToInteger(const char *, uint32_t, uint32_t *);
223 static void RtlCopyMemory(void *, const void *, size_t);
224 static size_t RtlCompareMemory(const void *, const void *, size_t);
225 static ndis_status RtlUnicodeStringToInteger(unicode_string *,
226 uint32_t, uint32_t *);
227 static int atoi (const char *);
228 static long atol (const char *);
229 static int rand(void);
230 static void srand(unsigned int);
231 static void KeQuerySystemTime(uint64_t *);
232 static uint32_t KeTickCount(void);
233 static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t);
234 static int32_t IoOpenDeviceRegistryKey(struct device_object *, uint32_t,
235 uint32_t, void **);
236 static void ntoskrnl_thrfunc(void *);
237 static ndis_status PsCreateSystemThread(ndis_handle *,
238 uint32_t, void *, ndis_handle, void *, void *, void *);
239 static ndis_status PsTerminateSystemThread(ndis_status);
240 static ndis_status IoGetDeviceObjectPointer(unicode_string *,
241 uint32_t, void *, device_object *);
242 static ndis_status IoGetDeviceProperty(device_object *, uint32_t,
243 uint32_t, void *, uint32_t *);
244 static void KeInitializeMutex(kmutant *, uint32_t);
245 static uint32_t KeReleaseMutex(kmutant *, uint8_t);
246 static uint32_t KeReadStateMutex(kmutant *);
247 static ndis_status ObReferenceObjectByHandle(ndis_handle,
248 uint32_t, void *, uint8_t, void **, void **);
249 static void ObfDereferenceObject(void *);
250 static uint32_t ZwClose(ndis_handle);
251 static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t,
252 uint32_t, void *);
253 static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...);
254 static uint32_t IoWMIRegistrationControl(device_object *, uint32_t);
255 static void *ntoskrnl_memset(void *, int, size_t);
256 static void *ntoskrnl_memmove(void *, void *, size_t);
257 static void *ntoskrnl_memchr(void *, unsigned char, size_t);
258 static char *ntoskrnl_strstr(char *, char *);
259 static char *ntoskrnl_strncat(char *, char *, size_t);
260 static int ntoskrnl_toupper(int);
261 static int ntoskrnl_tolower(int);
262 static funcptr ntoskrnl_findwrap(funcptr);
263 static uint32_t DbgPrint(char *, ...);
264 static void DbgBreakPoint(void);
265 static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long);
266 static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *);
267 static int32_t KeSetPriorityThread(struct thread *, int32_t);
268 static void dummy(void);
269
270 static struct mtx ntoskrnl_dispatchlock;
271 static struct mtx ntoskrnl_interlock;
272 static kspin_lock ntoskrnl_cancellock;
273 static int ntoskrnl_kth = 0;
274 static struct nt_objref_head ntoskrnl_reflist;
275 static uma_zone_t mdl_zone;
276 static uma_zone_t iw_zone;
277 static struct kdpc_queue *kq_queues;
278 static struct kdpc_queue *wq_queues;
279 static int wq_idx = 0;
280
281 int
ntoskrnl_libinit()282 ntoskrnl_libinit()
283 {
284 image_patch_table *patch;
285 int error;
286 struct proc *p;
287 kdpc_queue *kq;
288 callout_entry *e;
289 int i;
290
291 mtx_init(&ntoskrnl_dispatchlock,
292 "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE);
293 mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
294 KeInitializeSpinLock(&ntoskrnl_cancellock);
295 KeInitializeSpinLock(&ntoskrnl_intlock);
296 TAILQ_INIT(&ntoskrnl_reflist);
297
298 InitializeListHead(&ntoskrnl_calllist);
299 InitializeListHead(&ntoskrnl_intlist);
300 mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
301
302 kq_queues = ExAllocatePoolWithTag(NonPagedPool,
303 #ifdef NTOSKRNL_MULTIPLE_DPCS
304 sizeof(kdpc_queue) * mp_ncpus, 0);
305 #else
306 sizeof(kdpc_queue), 0);
307 #endif
308
309 if (kq_queues == NULL)
310 return (ENOMEM);
311
312 wq_queues = ExAllocatePoolWithTag(NonPagedPool,
313 sizeof(kdpc_queue) * WORKITEM_THREADS, 0);
314
315 if (wq_queues == NULL)
316 return (ENOMEM);
317
318 #ifdef NTOSKRNL_MULTIPLE_DPCS
319 bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus);
320 #else
321 bzero((char *)kq_queues, sizeof(kdpc_queue));
322 #endif
323 bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS);
324
325 /*
326 * Launch the DPC threads.
327 */
328
329 #ifdef NTOSKRNL_MULTIPLE_DPCS
330 for (i = 0; i < mp_ncpus; i++) {
331 #else
332 for (i = 0; i < 1; i++) {
333 #endif
334 kq = kq_queues + i;
335 kq->kq_cpu = i;
336 error = kproc_create(ntoskrnl_dpc_thread, kq, &p,
337 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i);
338 if (error)
339 panic("failed to launch DPC thread");
340 }
341
342 /*
343 * Launch the workitem threads.
344 */
345
346 for (i = 0; i < WORKITEM_THREADS; i++) {
347 kq = wq_queues + i;
348 error = kproc_create(ntoskrnl_workitem_thread, kq, &p,
349 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i);
350 if (error)
351 panic("failed to launch workitem thread");
352 }
353
354 patch = ntoskrnl_functbl;
355 while (patch->ipt_func != NULL) {
356 windrv_wrap((funcptr)patch->ipt_func,
357 (funcptr *)&patch->ipt_wrap,
358 patch->ipt_argcnt, patch->ipt_ftype);
359 patch++;
360 }
361
362 for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) {
363 e = ExAllocatePoolWithTag(NonPagedPool,
364 sizeof(callout_entry), 0);
365 if (e == NULL)
366 panic("failed to allocate timeouts");
367 mtx_lock_spin(&ntoskrnl_calllock);
368 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
369 mtx_unlock_spin(&ntoskrnl_calllock);
370 }
371
372 /*
373 * MDLs are supposed to be variable size (they describe
374 * buffers containing some number of pages, but we don't
375 * know ahead of time how many pages that will be). But
376 * always allocating them off the heap is very slow. As
377 * a compromise, we create an MDL UMA zone big enough to
378 * handle any buffer requiring up to 16 pages, and we
379 * use those for any MDLs for buffers of 16 pages or less
380 * in size. For buffers larger than that (which we assume
381 * will be few and far between, we allocate the MDLs off
382 * the heap.
383 */
384
385 mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE,
386 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
387
388 iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem),
389 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
390
391 return (0);
392 }
393
394 int
395 ntoskrnl_libfini()
396 {
397 image_patch_table *patch;
398 callout_entry *e;
399 list_entry *l;
400
401 patch = ntoskrnl_functbl;
402 while (patch->ipt_func != NULL) {
403 windrv_unwrap(patch->ipt_wrap);
404 patch++;
405 }
406
407 /* Stop the workitem queues. */
408 ntoskrnl_destroy_workitem_threads();
409 /* Stop the DPC queues. */
410 ntoskrnl_destroy_dpc_threads();
411
412 ExFreePool(kq_queues);
413 ExFreePool(wq_queues);
414
415 uma_zdestroy(mdl_zone);
416 uma_zdestroy(iw_zone);
417
418 mtx_lock_spin(&ntoskrnl_calllock);
419 while(!IsListEmpty(&ntoskrnl_calllist)) {
420 l = RemoveHeadList(&ntoskrnl_calllist);
421 e = CONTAINING_RECORD(l, callout_entry, ce_list);
422 mtx_unlock_spin(&ntoskrnl_calllock);
423 ExFreePool(e);
424 mtx_lock_spin(&ntoskrnl_calllock);
425 }
426 mtx_unlock_spin(&ntoskrnl_calllock);
427
428 mtx_destroy(&ntoskrnl_dispatchlock);
429 mtx_destroy(&ntoskrnl_interlock);
430 mtx_destroy(&ntoskrnl_calllock);
431
432 return (0);
433 }
434
435 /*
436 * We need to be able to reference this externally from the wrapper;
437 * GCC only generates a local implementation of memset.
438 */
439 static void *
440 ntoskrnl_memset(buf, ch, size)
441 void *buf;
442 int ch;
443 size_t size;
444 {
445 return (memset(buf, ch, size));
446 }
447
448 static void *
449 ntoskrnl_memmove(dst, src, size)
450 void *src;
451 void *dst;
452 size_t size;
453 {
454 bcopy(src, dst, size);
455 return (dst);
456 }
457
458 static void *
459 ntoskrnl_memchr(void *buf, unsigned char ch, size_t len)
460 {
461 if (len != 0) {
462 unsigned char *p = buf;
463
464 do {
465 if (*p++ == ch)
466 return (p - 1);
467 } while (--len != 0);
468 }
469 return (NULL);
470 }
471
472 static char *
473 ntoskrnl_strstr(s, find)
474 char *s, *find;
475 {
476 char c, sc;
477 size_t len;
478
479 if ((c = *find++) != 0) {
480 len = strlen(find);
481 do {
482 do {
483 if ((sc = *s++) == 0)
484 return (NULL);
485 } while (sc != c);
486 } while (strncmp(s, find, len) != 0);
487 s--;
488 }
489 return ((char *)s);
490 }
491
492 /* Taken from libc */
493 static char *
494 ntoskrnl_strncat(dst, src, n)
495 char *dst;
496 char *src;
497 size_t n;
498 {
499 if (n != 0) {
500 char *d = dst;
501 const char *s = src;
502
503 while (*d != 0)
504 d++;
505 do {
506 if ((*d = *s++) == 0)
507 break;
508 d++;
509 } while (--n != 0);
510 *d = 0;
511 }
512 return (dst);
513 }
514
515 static int
516 ntoskrnl_toupper(c)
517 int c;
518 {
519 return (toupper(c));
520 }
521
522 static int
523 ntoskrnl_tolower(c)
524 int c;
525 {
526 return (tolower(c));
527 }
528
529 static uint8_t
530 RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2,
531 uint8_t caseinsensitive)
532 {
533 int i;
534
535 if (str1->us_len != str2->us_len)
536 return (FALSE);
537
538 for (i = 0; i < str1->us_len; i++) {
539 if (caseinsensitive == TRUE) {
540 if (toupper((char)(str1->us_buf[i] & 0xFF)) !=
541 toupper((char)(str2->us_buf[i] & 0xFF)))
542 return (FALSE);
543 } else {
544 if (str1->us_buf[i] != str2->us_buf[i])
545 return (FALSE);
546 }
547 }
548
549 return (TRUE);
550 }
551
552 static void
553 RtlCopyString(dst, src)
554 ansi_string *dst;
555 const ansi_string *src;
556 {
557 if (src != NULL && src->as_buf != NULL && dst->as_buf != NULL) {
558 dst->as_len = min(src->as_len, dst->as_maxlen);
559 memcpy(dst->as_buf, src->as_buf, dst->as_len);
560 if (dst->as_len < dst->as_maxlen)
561 dst->as_buf[dst->as_len] = 0;
562 } else
563 dst->as_len = 0;
564 }
565
566 static void
567 RtlCopyUnicodeString(dest, src)
568 unicode_string *dest;
569 unicode_string *src;
570 {
571
572 if (dest->us_maxlen >= src->us_len)
573 dest->us_len = src->us_len;
574 else
575 dest->us_len = dest->us_maxlen;
576 memcpy(dest->us_buf, src->us_buf, dest->us_len);
577 }
578
579 static void
580 ntoskrnl_ascii_to_unicode(ascii, unicode, len)
581 char *ascii;
582 uint16_t *unicode;
583 int len;
584 {
585 int i;
586 uint16_t *ustr;
587
588 ustr = unicode;
589 for (i = 0; i < len; i++) {
590 *ustr = (uint16_t)ascii[i];
591 ustr++;
592 }
593 }
594
595 static void
596 ntoskrnl_unicode_to_ascii(unicode, ascii, len)
597 uint16_t *unicode;
598 char *ascii;
599 int len;
600 {
601 int i;
602 uint8_t *astr;
603
604 astr = ascii;
605 for (i = 0; i < len / 2; i++) {
606 *astr = (uint8_t)unicode[i];
607 astr++;
608 }
609 }
610
611 uint32_t
612 RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate)
613 {
614 if (dest == NULL || src == NULL)
615 return (STATUS_INVALID_PARAMETER);
616
617 dest->as_len = src->us_len / 2;
618 if (dest->as_maxlen < dest->as_len)
619 dest->as_len = dest->as_maxlen;
620
621 if (allocate == TRUE) {
622 dest->as_buf = ExAllocatePoolWithTag(NonPagedPool,
623 (src->us_len / 2) + 1, 0);
624 if (dest->as_buf == NULL)
625 return (STATUS_INSUFFICIENT_RESOURCES);
626 dest->as_len = dest->as_maxlen = src->us_len / 2;
627 } else {
628 dest->as_len = src->us_len / 2; /* XXX */
629 if (dest->as_maxlen < dest->as_len)
630 dest->as_len = dest->as_maxlen;
631 }
632
633 ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf,
634 dest->as_len * 2);
635
636 return (STATUS_SUCCESS);
637 }
638
639 uint32_t
640 RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src,
641 uint8_t allocate)
642 {
643 if (dest == NULL || src == NULL)
644 return (STATUS_INVALID_PARAMETER);
645
646 if (allocate == TRUE) {
647 dest->us_buf = ExAllocatePoolWithTag(NonPagedPool,
648 src->as_len * 2, 0);
649 if (dest->us_buf == NULL)
650 return (STATUS_INSUFFICIENT_RESOURCES);
651 dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2;
652 } else {
653 dest->us_len = src->as_len * 2; /* XXX */
654 if (dest->us_maxlen < dest->us_len)
655 dest->us_len = dest->us_maxlen;
656 }
657
658 ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf,
659 dest->us_len / 2);
660
661 return (STATUS_SUCCESS);
662 }
663
664 void *
665 ExAllocatePoolWithTag(pooltype, len, tag)
666 uint32_t pooltype;
667 size_t len;
668 uint32_t tag;
669 {
670 void *buf;
671
672 buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
673 if (buf == NULL)
674 return (NULL);
675
676 return (buf);
677 }
678
679 static void
680 ExFreePoolWithTag(buf, tag)
681 void *buf;
682 uint32_t tag;
683 {
684 ExFreePool(buf);
685 }
686
687 void
688 ExFreePool(buf)
689 void *buf;
690 {
691 free(buf, M_DEVBUF);
692 }
693
694 uint32_t
695 IoAllocateDriverObjectExtension(drv, clid, extlen, ext)
696 driver_object *drv;
697 void *clid;
698 uint32_t extlen;
699 void **ext;
700 {
701 custom_extension *ce;
702
703 ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension)
704 + extlen, 0);
705
706 if (ce == NULL)
707 return (STATUS_INSUFFICIENT_RESOURCES);
708
709 ce->ce_clid = clid;
710 InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list));
711
712 *ext = (void *)(ce + 1);
713
714 return (STATUS_SUCCESS);
715 }
716
717 void *
718 IoGetDriverObjectExtension(drv, clid)
719 driver_object *drv;
720 void *clid;
721 {
722 list_entry *e;
723 custom_extension *ce;
724
725 /*
726 * Sanity check. Our dummy bus drivers don't have
727 * any driver extensions.
728 */
729
730 if (drv->dro_driverext == NULL)
731 return (NULL);
732
733 e = drv->dro_driverext->dre_usrext.nle_flink;
734 while (e != &drv->dro_driverext->dre_usrext) {
735 ce = (custom_extension *)e;
736 if (ce->ce_clid == clid)
737 return ((void *)(ce + 1));
738 e = e->nle_flink;
739 }
740
741 return (NULL);
742 }
743
744
745 uint32_t
746 IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname,
747 uint32_t devtype, uint32_t devchars, uint8_t exclusive,
748 device_object **newdev)
749 {
750 device_object *dev;
751
752 dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0);
753 if (dev == NULL)
754 return (STATUS_INSUFFICIENT_RESOURCES);
755
756 dev->do_type = devtype;
757 dev->do_drvobj = drv;
758 dev->do_currirp = NULL;
759 dev->do_flags = 0;
760
761 if (devextlen) {
762 dev->do_devext = ExAllocatePoolWithTag(NonPagedPool,
763 devextlen, 0);
764
765 if (dev->do_devext == NULL) {
766 ExFreePool(dev);
767 return (STATUS_INSUFFICIENT_RESOURCES);
768 }
769
770 bzero(dev->do_devext, devextlen);
771 } else
772 dev->do_devext = NULL;
773
774 dev->do_size = sizeof(device_object) + devextlen;
775 dev->do_refcnt = 1;
776 dev->do_attacheddev = NULL;
777 dev->do_nextdev = NULL;
778 dev->do_devtype = devtype;
779 dev->do_stacksize = 1;
780 dev->do_alignreq = 1;
781 dev->do_characteristics = devchars;
782 dev->do_iotimer = NULL;
783 KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE);
784
785 /*
786 * Vpd is used for disk/tape devices,
787 * but we don't support those. (Yet.)
788 */
789 dev->do_vpb = NULL;
790
791 dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool,
792 sizeof(devobj_extension), 0);
793
794 if (dev->do_devobj_ext == NULL) {
795 if (dev->do_devext != NULL)
796 ExFreePool(dev->do_devext);
797 ExFreePool(dev);
798 return (STATUS_INSUFFICIENT_RESOURCES);
799 }
800
801 dev->do_devobj_ext->dve_type = 0;
802 dev->do_devobj_ext->dve_size = sizeof(devobj_extension);
803 dev->do_devobj_ext->dve_devobj = dev;
804
805 /*
806 * Attach this device to the driver object's list
807 * of devices. Note: this is not the same as attaching
808 * the device to the device stack. The driver's AddDevice
809 * routine must explicitly call IoAddDeviceToDeviceStack()
810 * to do that.
811 */
812
813 if (drv->dro_devobj == NULL) {
814 drv->dro_devobj = dev;
815 dev->do_nextdev = NULL;
816 } else {
817 dev->do_nextdev = drv->dro_devobj;
818 drv->dro_devobj = dev;
819 }
820
821 *newdev = dev;
822
823 return (STATUS_SUCCESS);
824 }
825
826 void
827 IoDeleteDevice(dev)
828 device_object *dev;
829 {
830 device_object *prev;
831
832 if (dev == NULL)
833 return;
834
835 if (dev->do_devobj_ext != NULL)
836 ExFreePool(dev->do_devobj_ext);
837
838 if (dev->do_devext != NULL)
839 ExFreePool(dev->do_devext);
840
841 /* Unlink the device from the driver's device list. */
842
843 prev = dev->do_drvobj->dro_devobj;
844 if (prev == dev)
845 dev->do_drvobj->dro_devobj = dev->do_nextdev;
846 else {
847 while (prev->do_nextdev != dev)
848 prev = prev->do_nextdev;
849 prev->do_nextdev = dev->do_nextdev;
850 }
851
852 ExFreePool(dev);
853 }
854
855 device_object *
856 IoGetAttachedDevice(dev)
857 device_object *dev;
858 {
859 device_object *d;
860
861 if (dev == NULL)
862 return (NULL);
863
864 d = dev;
865
866 while (d->do_attacheddev != NULL)
867 d = d->do_attacheddev;
868
869 return (d);
870 }
871
872 static irp *
873 IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
874 uint32_t func;
875 device_object *dobj;
876 void *buf;
877 uint32_t len;
878 uint64_t *off;
879 nt_kevent *event;
880 io_status_block *status;
881 {
882 irp *ip;
883
884 ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
885 if (ip == NULL)
886 return (NULL);
887 ip->irp_usrevent = event;
888
889 return (ip);
890 }
891
892 static irp *
893 IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
894 uint32_t func;
895 device_object *dobj;
896 void *buf;
897 uint32_t len;
898 uint64_t *off;
899 io_status_block *status;
900 {
901 irp *ip;
902 io_stack_location *sl;
903
904 ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
905 if (ip == NULL)
906 return (NULL);
907
908 ip->irp_usriostat = status;
909 ip->irp_tail.irp_overlay.irp_thread = NULL;
910
911 sl = IoGetNextIrpStackLocation(ip);
912 sl->isl_major = func;
913 sl->isl_minor = 0;
914 sl->isl_flags = 0;
915 sl->isl_ctl = 0;
916 sl->isl_devobj = dobj;
917 sl->isl_fileobj = NULL;
918 sl->isl_completionfunc = NULL;
919
920 ip->irp_userbuf = buf;
921
922 if (dobj->do_flags & DO_BUFFERED_IO) {
923 ip->irp_assoc.irp_sysbuf =
924 ExAllocatePoolWithTag(NonPagedPool, len, 0);
925 if (ip->irp_assoc.irp_sysbuf == NULL) {
926 IoFreeIrp(ip);
927 return (NULL);
928 }
929 bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
930 }
931
932 if (dobj->do_flags & DO_DIRECT_IO) {
933 ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
934 if (ip->irp_mdl == NULL) {
935 if (ip->irp_assoc.irp_sysbuf != NULL)
936 ExFreePool(ip->irp_assoc.irp_sysbuf);
937 IoFreeIrp(ip);
938 return (NULL);
939 }
940 ip->irp_userbuf = NULL;
941 ip->irp_assoc.irp_sysbuf = NULL;
942 }
943
944 if (func == IRP_MJ_READ) {
945 sl->isl_parameters.isl_read.isl_len = len;
946 if (off != NULL)
947 sl->isl_parameters.isl_read.isl_byteoff = *off;
948 else
949 sl->isl_parameters.isl_read.isl_byteoff = 0;
950 }
951
952 if (func == IRP_MJ_WRITE) {
953 sl->isl_parameters.isl_write.isl_len = len;
954 if (off != NULL)
955 sl->isl_parameters.isl_write.isl_byteoff = *off;
956 else
957 sl->isl_parameters.isl_write.isl_byteoff = 0;
958 }
959
960 return (ip);
961 }
962
963 static irp *
964 IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf,
965 uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal,
966 nt_kevent *event, io_status_block *status)
967 {
968 irp *ip;
969 io_stack_location *sl;
970 uint32_t buflen;
971
972 ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
973 if (ip == NULL)
974 return (NULL);
975 ip->irp_usrevent = event;
976 ip->irp_usriostat = status;
977 ip->irp_tail.irp_overlay.irp_thread = NULL;
978
979 sl = IoGetNextIrpStackLocation(ip);
980 sl->isl_major = isinternal == TRUE ?
981 IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
982 sl->isl_minor = 0;
983 sl->isl_flags = 0;
984 sl->isl_ctl = 0;
985 sl->isl_devobj = dobj;
986 sl->isl_fileobj = NULL;
987 sl->isl_completionfunc = NULL;
988 sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
989 sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
990 sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
991
992 switch(IO_METHOD(iocode)) {
993 case METHOD_BUFFERED:
994 if (ilen > olen)
995 buflen = ilen;
996 else
997 buflen = olen;
998 if (buflen) {
999 ip->irp_assoc.irp_sysbuf =
1000 ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
1001 if (ip->irp_assoc.irp_sysbuf == NULL) {
1002 IoFreeIrp(ip);
1003 return (NULL);
1004 }
1005 }
1006 if (ilen && ibuf != NULL) {
1007 bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1008 bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
1009 buflen - ilen);
1010 } else
1011 bzero(ip->irp_assoc.irp_sysbuf, ilen);
1012 ip->irp_userbuf = obuf;
1013 break;
1014 case METHOD_IN_DIRECT:
1015 case METHOD_OUT_DIRECT:
1016 if (ilen && ibuf != NULL) {
1017 ip->irp_assoc.irp_sysbuf =
1018 ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
1019 if (ip->irp_assoc.irp_sysbuf == NULL) {
1020 IoFreeIrp(ip);
1021 return (NULL);
1022 }
1023 bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1024 }
1025 if (olen && obuf != NULL) {
1026 ip->irp_mdl = IoAllocateMdl(obuf, olen,
1027 FALSE, FALSE, ip);
1028 /*
1029 * Normally we would MmProbeAndLockPages()
1030 * here, but we don't have to in our
1031 * imlementation.
1032 */
1033 }
1034 break;
1035 case METHOD_NEITHER:
1036 ip->irp_userbuf = obuf;
1037 sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
1038 break;
1039 default:
1040 break;
1041 }
1042
1043 /*
1044 * Ideally, we should associate this IRP with the calling
1045 * thread here.
1046 */
1047
1048 return (ip);
1049 }
1050
1051 static irp *
1052 IoAllocateIrp(uint8_t stsize, uint8_t chargequota)
1053 {
1054 irp *i;
1055
1056 i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0);
1057 if (i == NULL)
1058 return (NULL);
1059
1060 IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize);
1061
1062 return (i);
1063 }
1064
1065 static irp *
1066 IoMakeAssociatedIrp(irp *ip, uint8_t stsize)
1067 {
1068 irp *associrp;
1069
1070 associrp = IoAllocateIrp(stsize, FALSE);
1071 if (associrp == NULL)
1072 return (NULL);
1073
1074 mtx_lock(&ntoskrnl_dispatchlock);
1075 associrp->irp_flags |= IRP_ASSOCIATED_IRP;
1076 associrp->irp_tail.irp_overlay.irp_thread =
1077 ip->irp_tail.irp_overlay.irp_thread;
1078 associrp->irp_assoc.irp_master = ip;
1079 mtx_unlock(&ntoskrnl_dispatchlock);
1080
1081 return (associrp);
1082 }
1083
1084 static void
1085 IoFreeIrp(ip)
1086 irp *ip;
1087 {
1088 ExFreePool(ip);
1089 }
1090
1091 static void
1092 IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize)
1093 {
1094 bzero((char *)io, IoSizeOfIrp(ssize));
1095 io->irp_size = psize;
1096 io->irp_stackcnt = ssize;
1097 io->irp_currentstackloc = ssize;
1098 InitializeListHead(&io->irp_thlist);
1099 io->irp_tail.irp_overlay.irp_csl =
1100 (io_stack_location *)(io + 1) + ssize;
1101 }
1102
1103 static void
1104 IoReuseIrp(ip, status)
1105 irp *ip;
1106 uint32_t status;
1107 {
1108 uint8_t allocflags;
1109
1110 allocflags = ip->irp_allocflags;
1111 IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt);
1112 ip->irp_iostat.isb_status = status;
1113 ip->irp_allocflags = allocflags;
1114 }
1115
1116 void
1117 IoAcquireCancelSpinLock(uint8_t *irql)
1118 {
1119 KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
1120 }
1121
1122 void
1123 IoReleaseCancelSpinLock(uint8_t irql)
1124 {
1125 KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
1126 }
1127
1128 uint8_t
1129 IoCancelIrp(irp *ip)
1130 {
1131 cancel_func cfunc;
1132 uint8_t cancelirql;
1133
1134 IoAcquireCancelSpinLock(&cancelirql);
1135 cfunc = IoSetCancelRoutine(ip, NULL);
1136 ip->irp_cancel = TRUE;
1137 if (cfunc == NULL) {
1138 IoReleaseCancelSpinLock(cancelirql);
1139 return (FALSE);
1140 }
1141 ip->irp_cancelirql = cancelirql;
1142 MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
1143 return (uint8_t)IoSetCancelValue(ip, TRUE);
1144 }
1145
1146 uint32_t
1147 IofCallDriver(dobj, ip)
1148 device_object *dobj;
1149 irp *ip;
1150 {
1151 driver_object *drvobj;
1152 io_stack_location *sl;
1153 uint32_t status;
1154 driver_dispatch disp;
1155
1156 drvobj = dobj->do_drvobj;
1157
1158 if (ip->irp_currentstackloc <= 0)
1159 panic("IoCallDriver(): out of stack locations");
1160
1161 IoSetNextIrpStackLocation(ip);
1162 sl = IoGetCurrentIrpStackLocation(ip);
1163
1164 sl->isl_devobj = dobj;
1165
1166 disp = drvobj->dro_dispatch[sl->isl_major];
1167 status = MSCALL2(disp, dobj, ip);
1168
1169 return (status);
1170 }
1171
1172 void
1173 IofCompleteRequest(irp *ip, uint8_t prioboost)
1174 {
1175 uint32_t status;
1176 device_object *dobj;
1177 io_stack_location *sl;
1178 completion_func cf;
1179
1180 KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING,
1181 ("incorrect IRP(%p) status (STATUS_PENDING)", ip));
1182
1183 sl = IoGetCurrentIrpStackLocation(ip);
1184 IoSkipCurrentIrpStackLocation(ip);
1185
1186 do {
1187 if (sl->isl_ctl & SL_PENDING_RETURNED)
1188 ip->irp_pendingreturned = TRUE;
1189
1190 if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1))
1191 dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj;
1192 else
1193 dobj = NULL;
1194
1195 if (sl->isl_completionfunc != NULL &&
1196 ((ip->irp_iostat.isb_status == STATUS_SUCCESS &&
1197 sl->isl_ctl & SL_INVOKE_ON_SUCCESS) ||
1198 (ip->irp_iostat.isb_status != STATUS_SUCCESS &&
1199 sl->isl_ctl & SL_INVOKE_ON_ERROR) ||
1200 (ip->irp_cancel == TRUE &&
1201 sl->isl_ctl & SL_INVOKE_ON_CANCEL))) {
1202 cf = sl->isl_completionfunc;
1203 status = MSCALL3(cf, dobj, ip, sl->isl_completionctx);
1204 if (status == STATUS_MORE_PROCESSING_REQUIRED)
1205 return;
1206 } else {
1207 if ((ip->irp_currentstackloc <= ip->irp_stackcnt) &&
1208 (ip->irp_pendingreturned == TRUE))
1209 IoMarkIrpPending(ip);
1210 }
1211
1212 /* move to the next. */
1213 IoSkipCurrentIrpStackLocation(ip);
1214 sl++;
1215 } while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
1216
1217 if (ip->irp_usriostat != NULL)
1218 *ip->irp_usriostat = ip->irp_iostat;
1219 if (ip->irp_usrevent != NULL)
1220 KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
1221
1222 /* Handle any associated IRPs. */
1223
1224 if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
1225 uint32_t masterirpcnt;
1226 irp *masterirp;
1227 mdl *m;
1228
1229 masterirp = ip->irp_assoc.irp_master;
1230 masterirpcnt =
1231 InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt);
1232
1233 while ((m = ip->irp_mdl) != NULL) {
1234 ip->irp_mdl = m->mdl_next;
1235 IoFreeMdl(m);
1236 }
1237 IoFreeIrp(ip);
1238 if (masterirpcnt == 0)
1239 IoCompleteRequest(masterirp, IO_NO_INCREMENT);
1240 return;
1241 }
1242
1243 /* With any luck, these conditions will never arise. */
1244
1245 if (ip->irp_flags & IRP_PAGING_IO) {
1246 if (ip->irp_mdl != NULL)
1247 IoFreeMdl(ip->irp_mdl);
1248 IoFreeIrp(ip);
1249 }
1250 }
1251
1252 void
1253 ntoskrnl_intr(arg)
1254 void *arg;
1255 {
1256 kinterrupt *iobj;
1257 uint8_t irql;
1258 uint8_t claimed;
1259 list_entry *l;
1260
1261 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1262 l = ntoskrnl_intlist.nle_flink;
1263 while (l != &ntoskrnl_intlist) {
1264 iobj = CONTAINING_RECORD(l, kinterrupt, ki_list);
1265 claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx);
1266 if (claimed == TRUE)
1267 break;
1268 l = l->nle_flink;
1269 }
1270 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1271 }
1272
1273 uint8_t
1274 KeAcquireInterruptSpinLock(iobj)
1275 kinterrupt *iobj;
1276 {
1277 uint8_t irql;
1278 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1279 return (irql);
1280 }
1281
1282 void
1283 KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql)
1284 {
1285 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1286 }
1287
1288 uint8_t
1289 KeSynchronizeExecution(iobj, syncfunc, syncctx)
1290 kinterrupt *iobj;
1291 void *syncfunc;
1292 void *syncctx;
1293 {
1294 uint8_t irql;
1295
1296 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1297 MSCALL1(syncfunc, syncctx);
1298 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1299
1300 return (TRUE);
1301 }
1302
1303 /*
1304 * IoConnectInterrupt() is passed only the interrupt vector and
1305 * irql that a device wants to use, but no device-specific tag
1306 * of any kind. This conflicts rather badly with FreeBSD's
1307 * bus_setup_intr(), which needs the device_t for the device
1308 * requesting interrupt delivery. In order to bypass this
1309 * inconsistency, we implement a second level of interrupt
1310 * dispatching on top of bus_setup_intr(). All devices use
1311 * ntoskrnl_intr() as their ISR, and any device requesting
1312 * interrupts will be registered with ntoskrnl_intr()'s interrupt
1313 * dispatch list. When an interrupt arrives, we walk the list
1314 * and invoke all the registered ISRs. This effectively makes all
1315 * interrupts shared, but it's the only way to duplicate the
1316 * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly.
1317 */
1318
1319 uint32_t
1320 IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx,
1321 kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql,
1322 uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat)
1323 {
1324 uint8_t curirql;
1325
1326 *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0);
1327 if (*iobj == NULL)
1328 return (STATUS_INSUFFICIENT_RESOURCES);
1329
1330 (*iobj)->ki_svcfunc = svcfunc;
1331 (*iobj)->ki_svcctx = svcctx;
1332
1333 if (lock == NULL) {
1334 KeInitializeSpinLock(&(*iobj)->ki_lock_priv);
1335 (*iobj)->ki_lock = &(*iobj)->ki_lock_priv;
1336 } else
1337 (*iobj)->ki_lock = lock;
1338
1339 KeAcquireSpinLock(&ntoskrnl_intlock, &curirql);
1340 InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list));
1341 KeReleaseSpinLock(&ntoskrnl_intlock, curirql);
1342
1343 return (STATUS_SUCCESS);
1344 }
1345
1346 void
1347 IoDisconnectInterrupt(iobj)
1348 kinterrupt *iobj;
1349 {
1350 uint8_t irql;
1351
1352 if (iobj == NULL)
1353 return;
1354
1355 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1356 RemoveEntryList((&iobj->ki_list));
1357 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1358
1359 ExFreePool(iobj);
1360 }
1361
1362 device_object *
1363 IoAttachDeviceToDeviceStack(src, dst)
1364 device_object *src;
1365 device_object *dst;
1366 {
1367 device_object *attached;
1368
1369 mtx_lock(&ntoskrnl_dispatchlock);
1370 attached = IoGetAttachedDevice(dst);
1371 attached->do_attacheddev = src;
1372 src->do_attacheddev = NULL;
1373 src->do_stacksize = attached->do_stacksize + 1;
1374 mtx_unlock(&ntoskrnl_dispatchlock);
1375
1376 return (attached);
1377 }
1378
1379 void
1380 IoDetachDevice(topdev)
1381 device_object *topdev;
1382 {
1383 device_object *tail;
1384
1385 mtx_lock(&ntoskrnl_dispatchlock);
1386
1387 /* First, break the chain. */
1388 tail = topdev->do_attacheddev;
1389 if (tail == NULL) {
1390 mtx_unlock(&ntoskrnl_dispatchlock);
1391 return;
1392 }
1393 topdev->do_attacheddev = tail->do_attacheddev;
1394 topdev->do_refcnt--;
1395
1396 /* Now reduce the stacksize count for the takm_il objects. */
1397
1398 tail = topdev->do_attacheddev;
1399 while (tail != NULL) {
1400 tail->do_stacksize--;
1401 tail = tail->do_attacheddev;
1402 }
1403
1404 mtx_unlock(&ntoskrnl_dispatchlock);
1405 }
1406
1407 /*
1408 * For the most part, an object is considered signalled if
1409 * dh_sigstate == TRUE. The exception is for mutant objects
1410 * (mutexes), where the logic works like this:
1411 *
1412 * - If the thread already owns the object and sigstate is
1413 * less than or equal to 0, then the object is considered
1414 * signalled (recursive acquisition).
1415 * - If dh_sigstate == 1, the object is also considered
1416 * signalled.
1417 */
1418
1419 static int
1420 ntoskrnl_is_signalled(obj, td)
1421 nt_dispatch_header *obj;
1422 struct thread *td;
1423 {
1424 kmutant *km;
1425
1426 if (obj->dh_type == DISP_TYPE_MUTANT) {
1427 km = (kmutant *)obj;
1428 if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) ||
1429 obj->dh_sigstate == 1)
1430 return (TRUE);
1431 return (FALSE);
1432 }
1433
1434 if (obj->dh_sigstate > 0)
1435 return (TRUE);
1436 return (FALSE);
1437 }
1438
1439 static void
1440 ntoskrnl_satisfy_wait(obj, td)
1441 nt_dispatch_header *obj;
1442 struct thread *td;
1443 {
1444 kmutant *km;
1445
1446 switch (obj->dh_type) {
1447 case DISP_TYPE_MUTANT:
1448 km = (struct kmutant *)obj;
1449 obj->dh_sigstate--;
1450 /*
1451 * If sigstate reaches 0, the mutex is now
1452 * non-signalled (the new thread owns it).
1453 */
1454 if (obj->dh_sigstate == 0) {
1455 km->km_ownerthread = td;
1456 if (km->km_abandoned == TRUE)
1457 km->km_abandoned = FALSE;
1458 }
1459 break;
1460 /* Synchronization objects get reset to unsignalled. */
1461 case DISP_TYPE_SYNCHRONIZATION_EVENT:
1462 case DISP_TYPE_SYNCHRONIZATION_TIMER:
1463 obj->dh_sigstate = 0;
1464 break;
1465 case DISP_TYPE_SEMAPHORE:
1466 obj->dh_sigstate--;
1467 break;
1468 default:
1469 break;
1470 }
1471 }
1472
1473 static void
1474 ntoskrnl_satisfy_multiple_waits(wb)
1475 wait_block *wb;
1476 {
1477 wait_block *cur;
1478 struct thread *td;
1479
1480 cur = wb;
1481 td = wb->wb_kthread;
1482
1483 do {
1484 ntoskrnl_satisfy_wait(wb->wb_object, td);
1485 cur->wb_awakened = TRUE;
1486 cur = cur->wb_next;
1487 } while (cur != wb);
1488 }
1489
1490 /* Always called with dispatcher lock held. */
1491 static void
1492 ntoskrnl_waittest(obj, increment)
1493 nt_dispatch_header *obj;
1494 uint32_t increment;
1495 {
1496 wait_block *w, *next;
1497 list_entry *e;
1498 struct thread *td;
1499 wb_ext *we;
1500 int satisfied;
1501
1502 /*
1503 * Once an object has been signalled, we walk its list of
1504 * wait blocks. If a wait block can be awakened, then satisfy
1505 * waits as necessary and wake the thread.
1506 *
1507 * The rules work like this:
1508 *
1509 * If a wait block is marked as WAITTYPE_ANY, then
1510 * we can satisfy the wait conditions on the current
1511 * object and wake the thread right away. Satisfying
1512 * the wait also has the effect of breaking us out
1513 * of the search loop.
1514 *
1515 * If the object is marked as WAITTYLE_ALL, then the
1516 * wait block will be part of a circularly linked
1517 * list of wait blocks belonging to a waiting thread
1518 * that's sleeping in KeWaitForMultipleObjects(). In
1519 * order to wake the thread, all the objects in the
1520 * wait list must be in the signalled state. If they
1521 * are, we then satisfy all of them and wake the
1522 * thread.
1523 *
1524 */
1525
1526 e = obj->dh_waitlisthead.nle_flink;
1527
1528 while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) {
1529 w = CONTAINING_RECORD(e, wait_block, wb_waitlist);
1530 we = w->wb_ext;
1531 td = we->we_td;
1532 satisfied = FALSE;
1533 if (w->wb_waittype == WAITTYPE_ANY) {
1534 /*
1535 * Thread can be awakened if
1536 * any wait is satisfied.
1537 */
1538 ntoskrnl_satisfy_wait(obj, td);
1539 satisfied = TRUE;
1540 w->wb_awakened = TRUE;
1541 } else {
1542 /*
1543 * Thread can only be woken up
1544 * if all waits are satisfied.
1545 * If the thread is waiting on multiple
1546 * objects, they should all be linked
1547 * through the wb_next pointers in the
1548 * wait blocks.
1549 */
1550 satisfied = TRUE;
1551 next = w->wb_next;
1552 while (next != w) {
1553 if (ntoskrnl_is_signalled(obj, td) == FALSE) {
1554 satisfied = FALSE;
1555 break;
1556 }
1557 next = next->wb_next;
1558 }
1559 ntoskrnl_satisfy_multiple_waits(w);
1560 }
1561
1562 if (satisfied == TRUE)
1563 cv_broadcastpri(&we->we_cv,
1564 (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
1565 w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
1566
1567 e = e->nle_flink;
1568 }
1569 }
1570
1571 /*
1572 * Return the number of 100 nanosecond intervals since
1573 * January 1, 1601. (?!?!)
1574 */
1575 void
1576 ntoskrnl_time(tval)
1577 uint64_t *tval;
1578 {
1579 struct timespec ts;
1580
1581 nanotime(&ts);
1582 *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
1583 11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */
1584 }
1585
1586 static void
1587 KeQuerySystemTime(current_time)
1588 uint64_t *current_time;
1589 {
1590 ntoskrnl_time(current_time);
1591 }
1592
1593 static uint32_t
1594 KeTickCount(void)
1595 {
1596 struct timeval tv;
1597 getmicrouptime(&tv);
1598 return tvtohz(&tv);
1599 }
1600
1601
1602 /*
1603 * KeWaitForSingleObject() is a tricky beast, because it can be used
1604 * with several different object types: semaphores, timers, events,
1605 * mutexes and threads. Semaphores don't appear very often, but the
1606 * other object types are quite common. KeWaitForSingleObject() is
1607 * what's normally used to acquire a mutex, and it can be used to
1608 * wait for a thread termination.
1609 *
1610 * The Windows NDIS API is implemented in terms of Windows kernel
1611 * primitives, and some of the object manipulation is duplicated in
1612 * NDIS. For example, NDIS has timers and events, which are actually
1613 * Windows kevents and ktimers. Now, you're supposed to only use the
1614 * NDIS variants of these objects within the confines of the NDIS API,
1615 * but there are some naughty developers out there who will use
1616 * KeWaitForSingleObject() on NDIS timer and event objects, so we
1617 * have to support that as well. Conseqently, our NDIS timer and event
1618 * code has to be closely tied into our ntoskrnl timer and event code,
1619 * just as it is in Windows.
1620 *
1621 * KeWaitForSingleObject() may do different things for different kinds
1622 * of objects:
1623 *
1624 * - For events, we check if the event has been signalled. If the
1625 * event is already in the signalled state, we just return immediately,
1626 * otherwise we wait for it to be set to the signalled state by someone
1627 * else calling KeSetEvent(). Events can be either synchronization or
1628 * notification events.
1629 *
1630 * - For timers, if the timer has already fired and the timer is in
1631 * the signalled state, we just return, otherwise we wait on the
1632 * timer. Unlike an event, timers get signalled automatically when
1633 * they expire rather than someone having to trip them manually.
1634 * Timers initialized with KeInitializeTimer() are always notification
1635 * events: KeInitializeTimerEx() lets you initialize a timer as
1636 * either a notification or synchronization event.
1637 *
1638 * - For mutexes, we try to acquire the mutex and if we can't, we wait
1639 * on the mutex until it's available and then grab it. When a mutex is
1640 * released, it enters the signalled state, which wakes up one of the
1641 * threads waiting to acquire it. Mutexes are always synchronization
1642 * events.
1643 *
1644 * - For threads, the only thing we do is wait until the thread object
1645 * enters a signalled state, which occurs when the thread terminates.
1646 * Threads are always notification events.
1647 *
1648 * A notification event wakes up all threads waiting on an object. A
1649 * synchronization event wakes up just one. Also, a synchronization event
1650 * is auto-clearing, which means we automatically set the event back to
1651 * the non-signalled state once the wakeup is done.
1652 */
1653
1654 uint32_t
1655 KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode,
1656 uint8_t alertable, int64_t *duetime)
1657 {
1658 wait_block w;
1659 struct thread *td = curthread;
1660 struct timeval tv;
1661 int error = 0;
1662 uint64_t curtime;
1663 wb_ext we;
1664 nt_dispatch_header *obj;
1665
1666 obj = arg;
1667
1668 if (obj == NULL)
1669 return (STATUS_INVALID_PARAMETER);
1670
1671 mtx_lock(&ntoskrnl_dispatchlock);
1672
1673 cv_init(&we.we_cv, "KeWFS");
1674 we.we_td = td;
1675
1676 /*
1677 * Check to see if this object is already signalled,
1678 * and just return without waiting if it is.
1679 */
1680 if (ntoskrnl_is_signalled(obj, td) == TRUE) {
1681 /* Sanity check the signal state value. */
1682 if (obj->dh_sigstate != INT32_MIN) {
1683 ntoskrnl_satisfy_wait(obj, curthread);
1684 mtx_unlock(&ntoskrnl_dispatchlock);
1685 return (STATUS_SUCCESS);
1686 } else {
1687 /*
1688 * There's a limit to how many times we can
1689 * recursively acquire a mutant. If we hit
1690 * the limit, something is very wrong.
1691 */
1692 if (obj->dh_type == DISP_TYPE_MUTANT) {
1693 mtx_unlock(&ntoskrnl_dispatchlock);
1694 panic("mutant limit exceeded");
1695 }
1696 }
1697 }
1698
1699 bzero((char *)&w, sizeof(wait_block));
1700 w.wb_object = obj;
1701 w.wb_ext = &we;
1702 w.wb_waittype = WAITTYPE_ANY;
1703 w.wb_next = &w;
1704 w.wb_waitkey = 0;
1705 w.wb_awakened = FALSE;
1706 w.wb_oldpri = td->td_priority;
1707
1708 InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist));
1709
1710 /*
1711 * The timeout value is specified in 100 nanosecond units
1712 * and can be a positive or negative number. If it's positive,
1713 * then the duetime is absolute, and we need to convert it
1714 * to an absolute offset relative to now in order to use it.
1715 * If it's negative, then the duetime is relative and we
1716 * just have to convert the units.
1717 */
1718
1719 if (duetime != NULL) {
1720 if (*duetime < 0) {
1721 tv.tv_sec = - (*duetime) / 10000000;
1722 tv.tv_usec = (- (*duetime) / 10) -
1723 (tv.tv_sec * 1000000);
1724 } else {
1725 ntoskrnl_time(&curtime);
1726 if (*duetime < curtime)
1727 tv.tv_sec = tv.tv_usec = 0;
1728 else {
1729 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1730 tv.tv_usec = ((*duetime) - curtime) / 10 -
1731 (tv.tv_sec * 1000000);
1732 }
1733 }
1734 }
1735
1736 if (duetime == NULL)
1737 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1738 else
1739 error = cv_timedwait(&we.we_cv,
1740 &ntoskrnl_dispatchlock, tvtohz(&tv));
1741
1742 RemoveEntryList(&w.wb_waitlist);
1743
1744 cv_destroy(&we.we_cv);
1745
1746 /* We timed out. Leave the object alone and return status. */
1747
1748 if (error == EWOULDBLOCK) {
1749 mtx_unlock(&ntoskrnl_dispatchlock);
1750 return (STATUS_TIMEOUT);
1751 }
1752
1753 mtx_unlock(&ntoskrnl_dispatchlock);
1754
1755 return (STATUS_SUCCESS);
1756 /*
1757 return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason,
1758 mode, alertable, duetime, &w));
1759 */
1760 }
1761
1762 static uint32_t
1763 KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype,
1764 uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime,
1765 wait_block *wb_array)
1766 {
1767 struct thread *td = curthread;
1768 wait_block *whead, *w;
1769 wait_block _wb_array[MAX_WAIT_OBJECTS];
1770 nt_dispatch_header *cur;
1771 struct timeval tv;
1772 int i, wcnt = 0, error = 0;
1773 uint64_t curtime;
1774 struct timespec t1, t2;
1775 uint32_t status = STATUS_SUCCESS;
1776 wb_ext we;
1777
1778 if (cnt > MAX_WAIT_OBJECTS)
1779 return (STATUS_INVALID_PARAMETER);
1780 if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
1781 return (STATUS_INVALID_PARAMETER);
1782
1783 mtx_lock(&ntoskrnl_dispatchlock);
1784
1785 cv_init(&we.we_cv, "KeWFM");
1786 we.we_td = td;
1787
1788 if (wb_array == NULL)
1789 whead = _wb_array;
1790 else
1791 whead = wb_array;
1792
1793 bzero((char *)whead, sizeof(wait_block) * cnt);
1794
1795 /* First pass: see if we can satisfy any waits immediately. */
1796
1797 wcnt = 0;
1798 w = whead;
1799
1800 for (i = 0; i < cnt; i++) {
1801 InsertTailList((&obj[i]->dh_waitlisthead),
1802 (&w->wb_waitlist));
1803 w->wb_ext = &we;
1804 w->wb_object = obj[i];
1805 w->wb_waittype = wtype;
1806 w->wb_waitkey = i;
1807 w->wb_awakened = FALSE;
1808 w->wb_oldpri = td->td_priority;
1809 w->wb_next = w + 1;
1810 w++;
1811 wcnt++;
1812 if (ntoskrnl_is_signalled(obj[i], td)) {
1813 /*
1814 * There's a limit to how many times
1815 * we can recursively acquire a mutant.
1816 * If we hit the limit, something
1817 * is very wrong.
1818 */
1819 if (obj[i]->dh_sigstate == INT32_MIN &&
1820 obj[i]->dh_type == DISP_TYPE_MUTANT) {
1821 mtx_unlock(&ntoskrnl_dispatchlock);
1822 panic("mutant limit exceeded");
1823 }
1824
1825 /*
1826 * If this is a WAITTYPE_ANY wait, then
1827 * satisfy the waited object and exit
1828 * right now.
1829 */
1830
1831 if (wtype == WAITTYPE_ANY) {
1832 ntoskrnl_satisfy_wait(obj[i], td);
1833 status = STATUS_WAIT_0 + i;
1834 goto wait_done;
1835 } else {
1836 w--;
1837 wcnt--;
1838 w->wb_object = NULL;
1839 RemoveEntryList(&w->wb_waitlist);
1840 }
1841 }
1842 }
1843
1844 /*
1845 * If this is a WAITTYPE_ALL wait and all objects are
1846 * already signalled, satisfy the waits and exit now.
1847 */
1848
1849 if (wtype == WAITTYPE_ALL && wcnt == 0) {
1850 for (i = 0; i < cnt; i++)
1851 ntoskrnl_satisfy_wait(obj[i], td);
1852 status = STATUS_SUCCESS;
1853 goto wait_done;
1854 }
1855
1856 /*
1857 * Create a circular waitblock list. The waitcount
1858 * must always be non-zero when we get here.
1859 */
1860
1861 (w - 1)->wb_next = whead;
1862
1863 /* Wait on any objects that aren't yet signalled. */
1864
1865 /* Calculate timeout, if any. */
1866
1867 if (duetime != NULL) {
1868 if (*duetime < 0) {
1869 tv.tv_sec = - (*duetime) / 10000000;
1870 tv.tv_usec = (- (*duetime) / 10) -
1871 (tv.tv_sec * 1000000);
1872 } else {
1873 ntoskrnl_time(&curtime);
1874 if (*duetime < curtime)
1875 tv.tv_sec = tv.tv_usec = 0;
1876 else {
1877 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1878 tv.tv_usec = ((*duetime) - curtime) / 10 -
1879 (tv.tv_sec * 1000000);
1880 }
1881 }
1882 }
1883
1884 while (wcnt) {
1885 nanotime(&t1);
1886
1887 if (duetime == NULL)
1888 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1889 else
1890 error = cv_timedwait(&we.we_cv,
1891 &ntoskrnl_dispatchlock, tvtohz(&tv));
1892
1893 /* Wait with timeout expired. */
1894
1895 if (error) {
1896 status = STATUS_TIMEOUT;
1897 goto wait_done;
1898 }
1899
1900 nanotime(&t2);
1901
1902 /* See what's been signalled. */
1903
1904 w = whead;
1905 do {
1906 cur = w->wb_object;
1907 if (ntoskrnl_is_signalled(cur, td) == TRUE ||
1908 w->wb_awakened == TRUE) {
1909 /* Sanity check the signal state value. */
1910 if (cur->dh_sigstate == INT32_MIN &&
1911 cur->dh_type == DISP_TYPE_MUTANT) {
1912 mtx_unlock(&ntoskrnl_dispatchlock);
1913 panic("mutant limit exceeded");
1914 }
1915 wcnt--;
1916 if (wtype == WAITTYPE_ANY) {
1917 status = w->wb_waitkey &
1918 STATUS_WAIT_0;
1919 goto wait_done;
1920 }
1921 }
1922 w = w->wb_next;
1923 } while (w != whead);
1924
1925 /*
1926 * If all objects have been signalled, or if this
1927 * is a WAITTYPE_ANY wait and we were woke up by
1928 * someone, we can bail.
1929 */
1930
1931 if (wcnt == 0) {
1932 status = STATUS_SUCCESS;
1933 goto wait_done;
1934 }
1935
1936 /*
1937 * If this is WAITTYPE_ALL wait, and there's still
1938 * objects that haven't been signalled, deduct the
1939 * time that's elapsed so far from the timeout and
1940 * wait again (or continue waiting indefinitely if
1941 * there's no timeout).
1942 */
1943
1944 if (duetime != NULL) {
1945 tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
1946 tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
1947 }
1948 }
1949
1950
1951 wait_done:
1952
1953 cv_destroy(&we.we_cv);
1954
1955 for (i = 0; i < cnt; i++) {
1956 if (whead[i].wb_object != NULL)
1957 RemoveEntryList(&whead[i].wb_waitlist);
1958
1959 }
1960 mtx_unlock(&ntoskrnl_dispatchlock);
1961
1962 return (status);
1963 }
1964
1965 static void
1966 WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val)
1967 {
1968 bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1969 }
1970
1971 static uint16_t
1972 READ_REGISTER_USHORT(reg)
1973 uint16_t *reg;
1974 {
1975 return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1976 }
1977
1978 static void
1979 WRITE_REGISTER_ULONG(reg, val)
1980 uint32_t *reg;
1981 uint32_t val;
1982 {
1983 bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1984 }
1985
1986 static uint32_t
1987 READ_REGISTER_ULONG(reg)
1988 uint32_t *reg;
1989 {
1990 return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1991 }
1992
1993 static uint8_t
1994 READ_REGISTER_UCHAR(uint8_t *reg)
1995 {
1996 return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1997 }
1998
1999 static void
2000 WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val)
2001 {
2002 bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
2003 }
2004
2005 static int64_t
2006 _allmul(a, b)
2007 int64_t a;
2008 int64_t b;
2009 {
2010 return (a * b);
2011 }
2012
2013 static int64_t
2014 _alldiv(a, b)
2015 int64_t a;
2016 int64_t b;
2017 {
2018 return (a / b);
2019 }
2020
2021 static int64_t
2022 _allrem(a, b)
2023 int64_t a;
2024 int64_t b;
2025 {
2026 return (a % b);
2027 }
2028
2029 static uint64_t
2030 _aullmul(a, b)
2031 uint64_t a;
2032 uint64_t b;
2033 {
2034 return (a * b);
2035 }
2036
2037 static uint64_t
2038 _aulldiv(a, b)
2039 uint64_t a;
2040 uint64_t b;
2041 {
2042 return (a / b);
2043 }
2044
2045 static uint64_t
2046 _aullrem(a, b)
2047 uint64_t a;
2048 uint64_t b;
2049 {
2050 return (a % b);
2051 }
2052
2053 static int64_t
2054 _allshl(int64_t a, uint8_t b)
2055 {
2056 return (a << b);
2057 }
2058
2059 static uint64_t
2060 _aullshl(uint64_t a, uint8_t b)
2061 {
2062 return (a << b);
2063 }
2064
2065 static int64_t
2066 _allshr(int64_t a, uint8_t b)
2067 {
2068 return (a >> b);
2069 }
2070
2071 static uint64_t
2072 _aullshr(uint64_t a, uint8_t b)
2073 {
2074 return (a >> b);
2075 }
2076
2077 static slist_entry *
2078 ntoskrnl_pushsl(head, entry)
2079 slist_header *head;
2080 slist_entry *entry;
2081 {
2082 slist_entry *oldhead;
2083
2084 oldhead = head->slh_list.slh_next;
2085 entry->sl_next = head->slh_list.slh_next;
2086 head->slh_list.slh_next = entry;
2087 head->slh_list.slh_depth++;
2088 head->slh_list.slh_seq++;
2089
2090 return (oldhead);
2091 }
2092
2093 static void
2094 InitializeSListHead(head)
2095 slist_header *head;
2096 {
2097 memset(head, 0, sizeof(*head));
2098 }
2099
2100 static slist_entry *
2101 ntoskrnl_popsl(head)
2102 slist_header *head;
2103 {
2104 slist_entry *first;
2105
2106 first = head->slh_list.slh_next;
2107 if (first != NULL) {
2108 head->slh_list.slh_next = first->sl_next;
2109 head->slh_list.slh_depth--;
2110 head->slh_list.slh_seq++;
2111 }
2112
2113 return (first);
2114 }
2115
2116 /*
2117 * We need this to make lookaside lists work for amd64.
2118 * We pass a pointer to ExAllocatePoolWithTag() the lookaside
2119 * list structure. For amd64 to work right, this has to be a
2120 * pointer to the wrapped version of the routine, not the
2121 * original. Letting the Windows driver invoke the original
2122 * function directly will result in a convention calling
2123 * mismatch and a pretty crash. On x86, this effectively
2124 * becomes a no-op since ipt_func and ipt_wrap are the same.
2125 */
2126
2127 static funcptr
2128 ntoskrnl_findwrap(func)
2129 funcptr func;
2130 {
2131 image_patch_table *patch;
2132
2133 patch = ntoskrnl_functbl;
2134 while (patch->ipt_func != NULL) {
2135 if ((funcptr)patch->ipt_func == func)
2136 return ((funcptr)patch->ipt_wrap);
2137 patch++;
2138 }
2139
2140 return (NULL);
2141 }
2142
2143 static void
2144 ExInitializePagedLookasideList(paged_lookaside_list *lookaside,
2145 lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2146 uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2147 {
2148 bzero((char *)lookaside, sizeof(paged_lookaside_list));
2149
2150 if (size < sizeof(slist_entry))
2151 lookaside->nll_l.gl_size = sizeof(slist_entry);
2152 else
2153 lookaside->nll_l.gl_size = size;
2154 lookaside->nll_l.gl_tag = tag;
2155 if (allocfunc == NULL)
2156 lookaside->nll_l.gl_allocfunc =
2157 ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2158 else
2159 lookaside->nll_l.gl_allocfunc = allocfunc;
2160
2161 if (freefunc == NULL)
2162 lookaside->nll_l.gl_freefunc =
2163 ntoskrnl_findwrap((funcptr)ExFreePool);
2164 else
2165 lookaside->nll_l.gl_freefunc = freefunc;
2166
2167 #ifdef __i386__
2168 KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2169 #endif
2170
2171 lookaside->nll_l.gl_type = NonPagedPool;
2172 lookaside->nll_l.gl_depth = depth;
2173 lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2174 }
2175
2176 static void
2177 ExDeletePagedLookasideList(lookaside)
2178 paged_lookaside_list *lookaside;
2179 {
2180 void *buf;
2181 void (*freefunc)(void *);
2182
2183 freefunc = lookaside->nll_l.gl_freefunc;
2184 while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2185 MSCALL1(freefunc, buf);
2186 }
2187
2188 static void
2189 ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside,
2190 lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2191 uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2192 {
2193 bzero((char *)lookaside, sizeof(npaged_lookaside_list));
2194
2195 if (size < sizeof(slist_entry))
2196 lookaside->nll_l.gl_size = sizeof(slist_entry);
2197 else
2198 lookaside->nll_l.gl_size = size;
2199 lookaside->nll_l.gl_tag = tag;
2200 if (allocfunc == NULL)
2201 lookaside->nll_l.gl_allocfunc =
2202 ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2203 else
2204 lookaside->nll_l.gl_allocfunc = allocfunc;
2205
2206 if (freefunc == NULL)
2207 lookaside->nll_l.gl_freefunc =
2208 ntoskrnl_findwrap((funcptr)ExFreePool);
2209 else
2210 lookaside->nll_l.gl_freefunc = freefunc;
2211
2212 #ifdef __i386__
2213 KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2214 #endif
2215
2216 lookaside->nll_l.gl_type = NonPagedPool;
2217 lookaside->nll_l.gl_depth = depth;
2218 lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2219 }
2220
2221 static void
2222 ExDeleteNPagedLookasideList(lookaside)
2223 npaged_lookaside_list *lookaside;
2224 {
2225 void *buf;
2226 void (*freefunc)(void *);
2227
2228 freefunc = lookaside->nll_l.gl_freefunc;
2229 while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2230 MSCALL1(freefunc, buf);
2231 }
2232
2233 slist_entry *
2234 InterlockedPushEntrySList(head, entry)
2235 slist_header *head;
2236 slist_entry *entry;
2237 {
2238 slist_entry *oldhead;
2239
2240 mtx_lock_spin(&ntoskrnl_interlock);
2241 oldhead = ntoskrnl_pushsl(head, entry);
2242 mtx_unlock_spin(&ntoskrnl_interlock);
2243
2244 return (oldhead);
2245 }
2246
2247 slist_entry *
2248 InterlockedPopEntrySList(head)
2249 slist_header *head;
2250 {
2251 slist_entry *first;
2252
2253 mtx_lock_spin(&ntoskrnl_interlock);
2254 first = ntoskrnl_popsl(head);
2255 mtx_unlock_spin(&ntoskrnl_interlock);
2256
2257 return (first);
2258 }
2259
2260 static slist_entry *
2261 ExInterlockedPushEntrySList(head, entry, lock)
2262 slist_header *head;
2263 slist_entry *entry;
2264 kspin_lock *lock;
2265 {
2266 return (InterlockedPushEntrySList(head, entry));
2267 }
2268
2269 static slist_entry *
2270 ExInterlockedPopEntrySList(head, lock)
2271 slist_header *head;
2272 kspin_lock *lock;
2273 {
2274 return (InterlockedPopEntrySList(head));
2275 }
2276
2277 uint16_t
2278 ExQueryDepthSList(head)
2279 slist_header *head;
2280 {
2281 uint16_t depth;
2282
2283 mtx_lock_spin(&ntoskrnl_interlock);
2284 depth = head->slh_list.slh_depth;
2285 mtx_unlock_spin(&ntoskrnl_interlock);
2286
2287 return (depth);
2288 }
2289
2290 void
2291 KeInitializeSpinLock(lock)
2292 kspin_lock *lock;
2293 {
2294 *lock = 0;
2295 }
2296
2297 #ifdef __i386__
2298 void
2299 KefAcquireSpinLockAtDpcLevel(lock)
2300 kspin_lock *lock;
2301 {
2302 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2303 int i = 0;
2304 #endif
2305
2306 while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) {
2307 /* sit and spin */;
2308 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2309 i++;
2310 if (i > 200000000)
2311 panic("DEADLOCK!");
2312 #endif
2313 }
2314 }
2315
2316 void
2317 KefReleaseSpinLockFromDpcLevel(lock)
2318 kspin_lock *lock;
2319 {
2320 atomic_store_rel_int((volatile u_int *)lock, 0);
2321 }
2322
2323 uint8_t
2324 KeAcquireSpinLockRaiseToDpc(kspin_lock *lock)
2325 {
2326 uint8_t oldirql;
2327
2328 if (KeGetCurrentIrql() > DISPATCH_LEVEL)
2329 panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
2330
2331 KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
2332 KeAcquireSpinLockAtDpcLevel(lock);
2333
2334 return (oldirql);
2335 }
2336 #else
2337 void
2338 KeAcquireSpinLockAtDpcLevel(kspin_lock *lock)
2339 {
2340 while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0)
2341 /* sit and spin */;
2342 }
2343
2344 void
2345 KeReleaseSpinLockFromDpcLevel(kspin_lock *lock)
2346 {
2347 atomic_store_rel_int((volatile u_int *)lock, 0);
2348 }
2349 #endif /* __i386__ */
2350
2351 uintptr_t
2352 InterlockedExchange(dst, val)
2353 volatile uint32_t *dst;
2354 uintptr_t val;
2355 {
2356 uintptr_t r;
2357
2358 mtx_lock_spin(&ntoskrnl_interlock);
2359 r = *dst;
2360 *dst = val;
2361 mtx_unlock_spin(&ntoskrnl_interlock);
2362
2363 return (r);
2364 }
2365
2366 static uint32_t
2367 InterlockedIncrement(addend)
2368 volatile uint32_t *addend;
2369 {
2370 atomic_add_long((volatile u_long *)addend, 1);
2371 return (*addend);
2372 }
2373
2374 static uint32_t
2375 InterlockedDecrement(addend)
2376 volatile uint32_t *addend;
2377 {
2378 atomic_subtract_long((volatile u_long *)addend, 1);
2379 return (*addend);
2380 }
2381
2382 static void
2383 ExInterlockedAddLargeStatistic(addend, inc)
2384 uint64_t *addend;
2385 uint32_t inc;
2386 {
2387 mtx_lock_spin(&ntoskrnl_interlock);
2388 *addend += inc;
2389 mtx_unlock_spin(&ntoskrnl_interlock);
2390 };
2391
2392 mdl *
2393 IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf,
2394 uint8_t chargequota, irp *iopkt)
2395 {
2396 mdl *m;
2397 int zone = 0;
2398
2399 if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE)
2400 m = ExAllocatePoolWithTag(NonPagedPool,
2401 MmSizeOfMdl(vaddr, len), 0);
2402 else {
2403 m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO);
2404 zone++;
2405 }
2406
2407 if (m == NULL)
2408 return (NULL);
2409
2410 MmInitializeMdl(m, vaddr, len);
2411
2412 /*
2413 * MmInitializMdl() clears the flags field, so we
2414 * have to set this here. If the MDL came from the
2415 * MDL UMA zone, tag it so we can release it to
2416 * the right place later.
2417 */
2418 if (zone)
2419 m->mdl_flags = MDL_ZONE_ALLOCED;
2420
2421 if (iopkt != NULL) {
2422 if (secondarybuf == TRUE) {
2423 mdl *last;
2424 last = iopkt->irp_mdl;
2425 while (last->mdl_next != NULL)
2426 last = last->mdl_next;
2427 last->mdl_next = m;
2428 } else {
2429 if (iopkt->irp_mdl != NULL)
2430 panic("leaking an MDL in IoAllocateMdl()");
2431 iopkt->irp_mdl = m;
2432 }
2433 }
2434
2435 return (m);
2436 }
2437
2438 void
2439 IoFreeMdl(m)
2440 mdl *m;
2441 {
2442 if (m == NULL)
2443 return;
2444
2445 if (m->mdl_flags & MDL_ZONE_ALLOCED)
2446 uma_zfree(mdl_zone, m);
2447 else
2448 ExFreePool(m);
2449 }
2450
2451 static void *
2452 MmAllocateContiguousMemory(size, highest)
2453 uint32_t size;
2454 uint64_t highest;
2455 {
2456 void *addr;
2457 size_t pagelength = roundup(size, PAGE_SIZE);
2458
2459 addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0);
2460
2461 return (addr);
2462 }
2463
2464 static void *
2465 MmAllocateContiguousMemorySpecifyCache(size, lowest, highest,
2466 boundary, cachetype)
2467 uint32_t size;
2468 uint64_t lowest;
2469 uint64_t highest;
2470 uint64_t boundary;
2471 enum nt_caching_type cachetype;
2472 {
2473 vm_memattr_t memattr;
2474 void *ret;
2475
2476 switch (cachetype) {
2477 case MmNonCached:
2478 memattr = VM_MEMATTR_UNCACHEABLE;
2479 break;
2480 case MmWriteCombined:
2481 memattr = VM_MEMATTR_WRITE_COMBINING;
2482 break;
2483 case MmNonCachedUnordered:
2484 memattr = VM_MEMATTR_UNCACHEABLE;
2485 break;
2486 case MmCached:
2487 case MmHardwareCoherentCached:
2488 case MmUSWCCached:
2489 default:
2490 memattr = VM_MEMATTR_DEFAULT;
2491 break;
2492 }
2493
2494 ret = (void *)kmem_alloc_contig(size, M_ZERO | M_NOWAIT, lowest,
2495 highest, PAGE_SIZE, boundary, memattr);
2496 if (ret != NULL)
2497 malloc_type_allocated(M_DEVBUF, round_page(size));
2498 return (ret);
2499 }
2500
2501 static void
2502 MmFreeContiguousMemory(base)
2503 void *base;
2504 {
2505 ExFreePool(base);
2506 }
2507
2508 static void
2509 MmFreeContiguousMemorySpecifyCache(base, size, cachetype)
2510 void *base;
2511 uint32_t size;
2512 enum nt_caching_type cachetype;
2513 {
2514 contigfree(base, size, M_DEVBUF);
2515 }
2516
2517 static uint32_t
2518 MmSizeOfMdl(vaddr, len)
2519 void *vaddr;
2520 size_t len;
2521 {
2522 uint32_t l;
2523
2524 l = sizeof(struct mdl) +
2525 (sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len));
2526
2527 return (l);
2528 }
2529
2530 /*
2531 * The Microsoft documentation says this routine fills in the
2532 * page array of an MDL with the _physical_ page addresses that
2533 * comprise the buffer, but we don't really want to do that here.
2534 * Instead, we just fill in the page array with the kernel virtual
2535 * addresses of the buffers.
2536 */
2537 void
2538 MmBuildMdlForNonPagedPool(m)
2539 mdl *m;
2540 {
2541 vm_offset_t *mdl_pages;
2542 int pagecnt, i;
2543
2544 pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount);
2545
2546 if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *))
2547 panic("not enough pages in MDL to describe buffer");
2548
2549 mdl_pages = MmGetMdlPfnArray(m);
2550
2551 for (i = 0; i < pagecnt; i++)
2552 *mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE);
2553
2554 m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL;
2555 m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m);
2556 }
2557
2558 static void *
2559 MmMapLockedPages(mdl *buf, uint8_t accessmode)
2560 {
2561 buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA;
2562 return (MmGetMdlVirtualAddress(buf));
2563 }
2564
2565 static void *
2566 MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype,
2567 void *vaddr, uint32_t bugcheck, uint32_t prio)
2568 {
2569 return (MmMapLockedPages(buf, accessmode));
2570 }
2571
2572 static void
2573 MmUnmapLockedPages(vaddr, buf)
2574 void *vaddr;
2575 mdl *buf;
2576 {
2577 buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA;
2578 }
2579
2580 /*
2581 * This function has a problem in that it will break if you
2582 * compile this module without PAE and try to use it on a PAE
2583 * kernel. Unfortunately, there's no way around this at the
2584 * moment. It's slightly less broken that using pmap_kextract().
2585 * You'd think the virtual memory subsystem would help us out
2586 * here, but it doesn't.
2587 */
2588
2589 static uint64_t
2590 MmGetPhysicalAddress(void *base)
2591 {
2592 return (pmap_extract(kernel_map->pmap, (vm_offset_t)base));
2593 }
2594
2595 void *
2596 MmGetSystemRoutineAddress(ustr)
2597 unicode_string *ustr;
2598 {
2599 ansi_string astr;
2600
2601 if (RtlUnicodeStringToAnsiString(&astr, ustr, TRUE))
2602 return (NULL);
2603 return (ndis_get_routine_address(ntoskrnl_functbl, astr.as_buf));
2604 }
2605
2606 uint8_t
2607 MmIsAddressValid(vaddr)
2608 void *vaddr;
2609 {
2610 if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr))
2611 return (TRUE);
2612
2613 return (FALSE);
2614 }
2615
2616 void *
2617 MmMapIoSpace(paddr, len, cachetype)
2618 uint64_t paddr;
2619 uint32_t len;
2620 uint32_t cachetype;
2621 {
2622 devclass_t nexus_class;
2623 device_t *nexus_devs, devp;
2624 int nexus_count = 0;
2625 device_t matching_dev = NULL;
2626 struct resource *res;
2627 int i;
2628 vm_offset_t v;
2629
2630 /* There will always be at least one nexus. */
2631
2632 nexus_class = devclass_find("nexus");
2633 devclass_get_devices(nexus_class, &nexus_devs, &nexus_count);
2634
2635 for (i = 0; i < nexus_count; i++) {
2636 devp = nexus_devs[i];
2637 matching_dev = ntoskrnl_finddev(devp, paddr, &res);
2638 if (matching_dev)
2639 break;
2640 }
2641
2642 free(nexus_devs, M_TEMP);
2643
2644 if (matching_dev == NULL)
2645 return (NULL);
2646
2647 v = (vm_offset_t)rman_get_virtual(res);
2648 if (paddr > rman_get_start(res))
2649 v += paddr - rman_get_start(res);
2650
2651 return ((void *)v);
2652 }
2653
2654 void
2655 MmUnmapIoSpace(vaddr, len)
2656 void *vaddr;
2657 size_t len;
2658 {
2659 }
2660
2661
2662 static device_t
2663 ntoskrnl_finddev(dev, paddr, res)
2664 device_t dev;
2665 uint64_t paddr;
2666 struct resource **res;
2667 {
2668 device_t *children = NULL;
2669 device_t matching_dev;
2670 int childcnt;
2671 struct resource *r;
2672 struct resource_list *rl;
2673 struct resource_list_entry *rle;
2674 uint32_t flags;
2675 int i;
2676
2677 /* We only want devices that have been successfully probed. */
2678
2679 if (device_is_alive(dev) == FALSE)
2680 return (NULL);
2681
2682 rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
2683 if (rl != NULL) {
2684 STAILQ_FOREACH(rle, rl, link) {
2685 r = rle->res;
2686
2687 if (r == NULL)
2688 continue;
2689
2690 flags = rman_get_flags(r);
2691
2692 if (rle->type == SYS_RES_MEMORY &&
2693 paddr >= rman_get_start(r) &&
2694 paddr <= rman_get_end(r)) {
2695 if (!(flags & RF_ACTIVE))
2696 bus_activate_resource(dev,
2697 SYS_RES_MEMORY, 0, r);
2698 *res = r;
2699 return (dev);
2700 }
2701 }
2702 }
2703
2704 /*
2705 * If this device has children, do another
2706 * level of recursion to inspect them.
2707 */
2708
2709 device_get_children(dev, &children, &childcnt);
2710
2711 for (i = 0; i < childcnt; i++) {
2712 matching_dev = ntoskrnl_finddev(children[i], paddr, res);
2713 if (matching_dev != NULL) {
2714 free(children, M_TEMP);
2715 return (matching_dev);
2716 }
2717 }
2718
2719
2720 /* Won't somebody please think of the children! */
2721
2722 if (children != NULL)
2723 free(children, M_TEMP);
2724
2725 return (NULL);
2726 }
2727
2728 /*
2729 * Workitems are unlike DPCs, in that they run in a user-mode thread
2730 * context rather than at DISPATCH_LEVEL in kernel context. In our
2731 * case we run them in kernel context anyway.
2732 */
2733 static void
2734 ntoskrnl_workitem_thread(arg)
2735 void *arg;
2736 {
2737 kdpc_queue *kq;
2738 list_entry *l;
2739 io_workitem *iw;
2740 uint8_t irql;
2741
2742 kq = arg;
2743
2744 InitializeListHead(&kq->kq_disp);
2745 kq->kq_td = curthread;
2746 kq->kq_exit = 0;
2747 KeInitializeSpinLock(&kq->kq_lock);
2748 KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
2749
2750 while (1) {
2751 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
2752
2753 KeAcquireSpinLock(&kq->kq_lock, &irql);
2754
2755 if (kq->kq_exit) {
2756 kq->kq_exit = 0;
2757 KeReleaseSpinLock(&kq->kq_lock, irql);
2758 break;
2759 }
2760
2761 while (!IsListEmpty(&kq->kq_disp)) {
2762 l = RemoveHeadList(&kq->kq_disp);
2763 iw = CONTAINING_RECORD(l,
2764 io_workitem, iw_listentry);
2765 InitializeListHead((&iw->iw_listentry));
2766 if (iw->iw_func == NULL)
2767 continue;
2768 KeReleaseSpinLock(&kq->kq_lock, irql);
2769 MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx);
2770 KeAcquireSpinLock(&kq->kq_lock, &irql);
2771 }
2772
2773 KeReleaseSpinLock(&kq->kq_lock, irql);
2774 }
2775
2776 kproc_exit(0);
2777 return; /* notreached */
2778 }
2779
2780 static ndis_status
2781 RtlCharToInteger(src, base, val)
2782 const char *src;
2783 uint32_t base;
2784 uint32_t *val;
2785 {
2786 int negative = 0;
2787 uint32_t res;
2788
2789 if (!src || !val)
2790 return (STATUS_ACCESS_VIOLATION);
2791 while (*src != '\0' && *src <= ' ')
2792 src++;
2793 if (*src == '+')
2794 src++;
2795 else if (*src == '-') {
2796 src++;
2797 negative = 1;
2798 }
2799 if (base == 0) {
2800 base = 10;
2801 if (*src == '0') {
2802 src++;
2803 if (*src == 'b') {
2804 base = 2;
2805 src++;
2806 } else if (*src == 'o') {
2807 base = 8;
2808 src++;
2809 } else if (*src == 'x') {
2810 base = 16;
2811 src++;
2812 }
2813 }
2814 } else if (!(base == 2 || base == 8 || base == 10 || base == 16))
2815 return (STATUS_INVALID_PARAMETER);
2816
2817 for (res = 0; *src; src++) {
2818 int v;
2819 if (isdigit(*src))
2820 v = *src - '0';
2821 else if (isxdigit(*src))
2822 v = tolower(*src) - 'a' + 10;
2823 else
2824 v = base;
2825 if (v >= base)
2826 return (STATUS_INVALID_PARAMETER);
2827 res = res * base + v;
2828 }
2829 *val = negative ? -res : res;
2830 return (STATUS_SUCCESS);
2831 }
2832
2833 static void
2834 ntoskrnl_destroy_workitem_threads(void)
2835 {
2836 kdpc_queue *kq;
2837 int i;
2838
2839 for (i = 0; i < WORKITEM_THREADS; i++) {
2840 kq = wq_queues + i;
2841 kq->kq_exit = 1;
2842 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2843 while (kq->kq_exit)
2844 tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10);
2845 }
2846 }
2847
2848 io_workitem *
2849 IoAllocateWorkItem(dobj)
2850 device_object *dobj;
2851 {
2852 io_workitem *iw;
2853
2854 iw = uma_zalloc(iw_zone, M_NOWAIT);
2855 if (iw == NULL)
2856 return (NULL);
2857
2858 InitializeListHead(&iw->iw_listentry);
2859 iw->iw_dobj = dobj;
2860
2861 mtx_lock(&ntoskrnl_dispatchlock);
2862 iw->iw_idx = wq_idx;
2863 WORKIDX_INC(wq_idx);
2864 mtx_unlock(&ntoskrnl_dispatchlock);
2865
2866 return (iw);
2867 }
2868
2869 void
2870 IoFreeWorkItem(iw)
2871 io_workitem *iw;
2872 {
2873 uma_zfree(iw_zone, iw);
2874 }
2875
2876 void
2877 IoQueueWorkItem(iw, iw_func, qtype, ctx)
2878 io_workitem *iw;
2879 io_workitem_func iw_func;
2880 uint32_t qtype;
2881 void *ctx;
2882 {
2883 kdpc_queue *kq;
2884 list_entry *l;
2885 io_workitem *cur;
2886 uint8_t irql;
2887
2888 kq = wq_queues + iw->iw_idx;
2889
2890 KeAcquireSpinLock(&kq->kq_lock, &irql);
2891
2892 /*
2893 * Traverse the list and make sure this workitem hasn't
2894 * already been inserted. Queuing the same workitem
2895 * twice will hose the list but good.
2896 */
2897
2898 l = kq->kq_disp.nle_flink;
2899 while (l != &kq->kq_disp) {
2900 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2901 if (cur == iw) {
2902 /* Already queued -- do nothing. */
2903 KeReleaseSpinLock(&kq->kq_lock, irql);
2904 return;
2905 }
2906 l = l->nle_flink;
2907 }
2908
2909 iw->iw_func = iw_func;
2910 iw->iw_ctx = ctx;
2911
2912 InsertTailList((&kq->kq_disp), (&iw->iw_listentry));
2913 KeReleaseSpinLock(&kq->kq_lock, irql);
2914
2915 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2916 }
2917
2918 static void
2919 ntoskrnl_workitem(dobj, arg)
2920 device_object *dobj;
2921 void *arg;
2922 {
2923 io_workitem *iw;
2924 work_queue_item *w;
2925 work_item_func f;
2926
2927 iw = arg;
2928 w = (work_queue_item *)dobj;
2929 f = (work_item_func)w->wqi_func;
2930 uma_zfree(iw_zone, iw);
2931 MSCALL2(f, w, w->wqi_ctx);
2932 }
2933
2934 /*
2935 * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft
2936 * warns that it's unsafe and to use IoQueueWorkItem() instead. The
2937 * problem with ExQueueWorkItem() is that it can't guard against
2938 * the condition where a driver submits a job to the work queue and
2939 * is then unloaded before the job is able to run. IoQueueWorkItem()
2940 * acquires a reference to the device's device_object via the
2941 * object manager and retains it until after the job has completed,
2942 * which prevents the driver from being unloaded before the job
2943 * runs. (We don't currently support this behavior, though hopefully
2944 * that will change once the object manager API is fleshed out a bit.)
2945 *
2946 * Having said all that, the ExQueueWorkItem() API remains, because
2947 * there are still other parts of Windows that use it, including
2948 * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem().
2949 * We fake up the ExQueueWorkItem() API on top of our implementation
2950 * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively
2951 * for ExQueueWorkItem() jobs, and we pass a pointer to the work
2952 * queue item (provided by the caller) in to IoAllocateWorkItem()
2953 * instead of the device_object. We need to save this pointer so
2954 * we can apply a sanity check: as with the DPC queue and other
2955 * workitem queues, we can't allow the same work queue item to
2956 * be queued twice. If it's already pending, we silently return
2957 */
2958
2959 void
2960 ExQueueWorkItem(w, qtype)
2961 work_queue_item *w;
2962 uint32_t qtype;
2963 {
2964 io_workitem *iw;
2965 io_workitem_func iwf;
2966 kdpc_queue *kq;
2967 list_entry *l;
2968 io_workitem *cur;
2969 uint8_t irql;
2970
2971
2972 /*
2973 * We need to do a special sanity test to make sure
2974 * the ExQueueWorkItem() API isn't used to queue
2975 * the same workitem twice. Rather than checking the
2976 * io_workitem pointer itself, we test the attached
2977 * device object, which is really a pointer to the
2978 * legacy work queue item structure.
2979 */
2980
2981 kq = wq_queues + WORKITEM_LEGACY_THREAD;
2982 KeAcquireSpinLock(&kq->kq_lock, &irql);
2983 l = kq->kq_disp.nle_flink;
2984 while (l != &kq->kq_disp) {
2985 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2986 if (cur->iw_dobj == (device_object *)w) {
2987 /* Already queued -- do nothing. */
2988 KeReleaseSpinLock(&kq->kq_lock, irql);
2989 return;
2990 }
2991 l = l->nle_flink;
2992 }
2993 KeReleaseSpinLock(&kq->kq_lock, irql);
2994
2995 iw = IoAllocateWorkItem((device_object *)w);
2996 if (iw == NULL)
2997 return;
2998
2999 iw->iw_idx = WORKITEM_LEGACY_THREAD;
3000 iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem);
3001 IoQueueWorkItem(iw, iwf, qtype, iw);
3002 }
3003
3004 static void
3005 RtlZeroMemory(dst, len)
3006 void *dst;
3007 size_t len;
3008 {
3009 bzero(dst, len);
3010 }
3011
3012 static void
3013 RtlSecureZeroMemory(dst, len)
3014 void *dst;
3015 size_t len;
3016 {
3017 memset(dst, 0, len);
3018 }
3019
3020 static void
3021 RtlFillMemory(void *dst, size_t len, uint8_t c)
3022 {
3023 memset(dst, c, len);
3024 }
3025
3026 static void
3027 RtlMoveMemory(dst, src, len)
3028 void *dst;
3029 const void *src;
3030 size_t len;
3031 {
3032 memmove(dst, src, len);
3033 }
3034
3035 static void
3036 RtlCopyMemory(dst, src, len)
3037 void *dst;
3038 const void *src;
3039 size_t len;
3040 {
3041 bcopy(src, dst, len);
3042 }
3043
3044 static size_t
3045 RtlCompareMemory(s1, s2, len)
3046 const void *s1;
3047 const void *s2;
3048 size_t len;
3049 {
3050 size_t i;
3051 uint8_t *m1, *m2;
3052
3053 m1 = __DECONST(char *, s1);
3054 m2 = __DECONST(char *, s2);
3055
3056 for (i = 0; i < len && m1[i] == m2[i]; i++);
3057 return (i);
3058 }
3059
3060 void
3061 RtlInitAnsiString(dst, src)
3062 ansi_string *dst;
3063 char *src;
3064 {
3065 ansi_string *a;
3066
3067 a = dst;
3068 if (a == NULL)
3069 return;
3070 if (src == NULL) {
3071 a->as_len = a->as_maxlen = 0;
3072 a->as_buf = NULL;
3073 } else {
3074 a->as_buf = src;
3075 a->as_len = a->as_maxlen = strlen(src);
3076 }
3077 }
3078
3079 void
3080 RtlInitUnicodeString(dst, src)
3081 unicode_string *dst;
3082 uint16_t *src;
3083 {
3084 unicode_string *u;
3085 int i;
3086
3087 u = dst;
3088 if (u == NULL)
3089 return;
3090 if (src == NULL) {
3091 u->us_len = u->us_maxlen = 0;
3092 u->us_buf = NULL;
3093 } else {
3094 i = 0;
3095 while(src[i] != 0)
3096 i++;
3097 u->us_buf = src;
3098 u->us_len = u->us_maxlen = i * 2;
3099 }
3100 }
3101
3102 ndis_status
3103 RtlUnicodeStringToInteger(ustr, base, val)
3104 unicode_string *ustr;
3105 uint32_t base;
3106 uint32_t *val;
3107 {
3108 uint16_t *uchr;
3109 int len, neg = 0;
3110 char abuf[64];
3111 char *astr;
3112
3113 uchr = ustr->us_buf;
3114 len = ustr->us_len;
3115 bzero(abuf, sizeof(abuf));
3116
3117 if ((char)((*uchr) & 0xFF) == '-') {
3118 neg = 1;
3119 uchr++;
3120 len -= 2;
3121 } else if ((char)((*uchr) & 0xFF) == '+') {
3122 neg = 0;
3123 uchr++;
3124 len -= 2;
3125 }
3126
3127 if (base == 0) {
3128 if ((char)((*uchr) & 0xFF) == 'b') {
3129 base = 2;
3130 uchr++;
3131 len -= 2;
3132 } else if ((char)((*uchr) & 0xFF) == 'o') {
3133 base = 8;
3134 uchr++;
3135 len -= 2;
3136 } else if ((char)((*uchr) & 0xFF) == 'x') {
3137 base = 16;
3138 uchr++;
3139 len -= 2;
3140 } else
3141 base = 10;
3142 }
3143
3144 astr = abuf;
3145 if (neg) {
3146 strcpy(astr, "-");
3147 astr++;
3148 }
3149
3150 ntoskrnl_unicode_to_ascii(uchr, astr, len);
3151 *val = strtoul(abuf, NULL, base);
3152
3153 return (STATUS_SUCCESS);
3154 }
3155
3156 void
3157 RtlFreeUnicodeString(ustr)
3158 unicode_string *ustr;
3159 {
3160 if (ustr->us_buf == NULL)
3161 return;
3162 ExFreePool(ustr->us_buf);
3163 ustr->us_buf = NULL;
3164 }
3165
3166 void
3167 RtlFreeAnsiString(astr)
3168 ansi_string *astr;
3169 {
3170 if (astr->as_buf == NULL)
3171 return;
3172 ExFreePool(astr->as_buf);
3173 astr->as_buf = NULL;
3174 }
3175
3176 static int
3177 atoi(str)
3178 const char *str;
3179 {
3180 return (int)strtol(str, (char **)NULL, 10);
3181 }
3182
3183 static long
3184 atol(str)
3185 const char *str;
3186 {
3187 return strtol(str, (char **)NULL, 10);
3188 }
3189
3190 static int
3191 rand(void)
3192 {
3193
3194 return (random());
3195 }
3196
3197 static void
3198 srand(unsigned int seed)
3199 {
3200
3201 srandom(seed);
3202 }
3203
3204 static uint8_t
3205 IoIsWdmVersionAvailable(uint8_t major, uint8_t minor)
3206 {
3207 if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
3208 return (TRUE);
3209 return (FALSE);
3210 }
3211
3212 static int32_t
3213 IoOpenDeviceRegistryKey(struct device_object *devobj, uint32_t type,
3214 uint32_t mask, void **key)
3215 {
3216 return (NDIS_STATUS_INVALID_DEVICE_REQUEST);
3217 }
3218
3219 static ndis_status
3220 IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj)
3221 unicode_string *name;
3222 uint32_t reqaccess;
3223 void *fileobj;
3224 device_object *devobj;
3225 {
3226 return (STATUS_SUCCESS);
3227 }
3228
3229 static ndis_status
3230 IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen)
3231 device_object *devobj;
3232 uint32_t regprop;
3233 uint32_t buflen;
3234 void *prop;
3235 uint32_t *reslen;
3236 {
3237 driver_object *drv;
3238 uint16_t **name;
3239
3240 drv = devobj->do_drvobj;
3241
3242 switch (regprop) {
3243 case DEVPROP_DRIVER_KEYNAME:
3244 name = prop;
3245 *name = drv->dro_drivername.us_buf;
3246 *reslen = drv->dro_drivername.us_len;
3247 break;
3248 default:
3249 return (STATUS_INVALID_PARAMETER_2);
3250 break;
3251 }
3252
3253 return (STATUS_SUCCESS);
3254 }
3255
3256 static void
3257 KeInitializeMutex(kmutex, level)
3258 kmutant *kmutex;
3259 uint32_t level;
3260 {
3261 InitializeListHead((&kmutex->km_header.dh_waitlisthead));
3262 kmutex->km_abandoned = FALSE;
3263 kmutex->km_apcdisable = 1;
3264 kmutex->km_header.dh_sigstate = 1;
3265 kmutex->km_header.dh_type = DISP_TYPE_MUTANT;
3266 kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t);
3267 kmutex->km_ownerthread = NULL;
3268 }
3269
3270 static uint32_t
3271 KeReleaseMutex(kmutant *kmutex, uint8_t kwait)
3272 {
3273 uint32_t prevstate;
3274
3275 mtx_lock(&ntoskrnl_dispatchlock);
3276 prevstate = kmutex->km_header.dh_sigstate;
3277 if (kmutex->km_ownerthread != curthread) {
3278 mtx_unlock(&ntoskrnl_dispatchlock);
3279 return (STATUS_MUTANT_NOT_OWNED);
3280 }
3281
3282 kmutex->km_header.dh_sigstate++;
3283 kmutex->km_abandoned = FALSE;
3284
3285 if (kmutex->km_header.dh_sigstate == 1) {
3286 kmutex->km_ownerthread = NULL;
3287 ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT);
3288 }
3289
3290 mtx_unlock(&ntoskrnl_dispatchlock);
3291
3292 return (prevstate);
3293 }
3294
3295 static uint32_t
3296 KeReadStateMutex(kmutex)
3297 kmutant *kmutex;
3298 {
3299 return (kmutex->km_header.dh_sigstate);
3300 }
3301
3302 void
3303 KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state)
3304 {
3305 InitializeListHead((&kevent->k_header.dh_waitlisthead));
3306 kevent->k_header.dh_sigstate = state;
3307 if (type == EVENT_TYPE_NOTIFY)
3308 kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT;
3309 else
3310 kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT;
3311 kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t);
3312 }
3313
3314 uint32_t
3315 KeResetEvent(kevent)
3316 nt_kevent *kevent;
3317 {
3318 uint32_t prevstate;
3319
3320 mtx_lock(&ntoskrnl_dispatchlock);
3321 prevstate = kevent->k_header.dh_sigstate;
3322 kevent->k_header.dh_sigstate = FALSE;
3323 mtx_unlock(&ntoskrnl_dispatchlock);
3324
3325 return (prevstate);
3326 }
3327
3328 uint32_t
3329 KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
3330 {
3331 uint32_t prevstate;
3332 wait_block *w;
3333 nt_dispatch_header *dh;
3334 struct thread *td;
3335 wb_ext *we;
3336
3337 mtx_lock(&ntoskrnl_dispatchlock);
3338 prevstate = kevent->k_header.dh_sigstate;
3339 dh = &kevent->k_header;
3340
3341 if (IsListEmpty(&dh->dh_waitlisthead))
3342 /*
3343 * If there's nobody in the waitlist, just set
3344 * the state to signalled.
3345 */
3346 dh->dh_sigstate = 1;
3347 else {
3348 /*
3349 * Get the first waiter. If this is a synchronization
3350 * event, just wake up that one thread (don't bother
3351 * setting the state to signalled since we're supposed
3352 * to automatically clear synchronization events anyway).
3353 *
3354 * If it's a notification event, or the first
3355 * waiter is doing a WAITTYPE_ALL wait, go through
3356 * the full wait satisfaction process.
3357 */
3358 w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink,
3359 wait_block, wb_waitlist);
3360 we = w->wb_ext;
3361 td = we->we_td;
3362 if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT ||
3363 w->wb_waittype == WAITTYPE_ALL) {
3364 if (prevstate == 0) {
3365 dh->dh_sigstate = 1;
3366 ntoskrnl_waittest(dh, increment);
3367 }
3368 } else {
3369 w->wb_awakened |= TRUE;
3370 cv_broadcastpri(&we->we_cv,
3371 (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
3372 w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
3373 }
3374 }
3375
3376 mtx_unlock(&ntoskrnl_dispatchlock);
3377
3378 return (prevstate);
3379 }
3380
3381 void
3382 KeClearEvent(kevent)
3383 nt_kevent *kevent;
3384 {
3385 kevent->k_header.dh_sigstate = FALSE;
3386 }
3387
3388 uint32_t
3389 KeReadStateEvent(kevent)
3390 nt_kevent *kevent;
3391 {
3392 return (kevent->k_header.dh_sigstate);
3393 }
3394
3395 /*
3396 * The object manager in Windows is responsible for managing
3397 * references and access to various types of objects, including
3398 * device_objects, events, threads, timers and so on. However,
3399 * there's a difference in the way objects are handled in user
3400 * mode versus kernel mode.
3401 *
3402 * In user mode (i.e. Win32 applications), all objects are
3403 * managed by the object manager. For example, when you create
3404 * a timer or event object, you actually end up with an
3405 * object_header (for the object manager's bookkeeping
3406 * purposes) and an object body (which contains the actual object
3407 * structure, e.g. ktimer, kevent, etc...). This allows Windows
3408 * to manage resource quotas and to enforce access restrictions
3409 * on basically every kind of system object handled by the kernel.
3410 *
3411 * However, in kernel mode, you only end up using the object
3412 * manager some of the time. For example, in a driver, you create
3413 * a timer object by simply allocating the memory for a ktimer
3414 * structure and initializing it with KeInitializeTimer(). Hence,
3415 * the timer has no object_header and no reference counting or
3416 * security/resource checks are done on it. The assumption in
3417 * this case is that if you're running in kernel mode, you know
3418 * what you're doing, and you're already at an elevated privilege
3419 * anyway.
3420 *
3421 * There are some exceptions to this. The two most important ones
3422 * for our purposes are device_objects and threads. We need to use
3423 * the object manager to do reference counting on device_objects,
3424 * and for threads, you can only get a pointer to a thread's
3425 * dispatch header by using ObReferenceObjectByHandle() on the
3426 * handle returned by PsCreateSystemThread().
3427 */
3428
3429 static ndis_status
3430 ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype,
3431 uint8_t accessmode, void **object, void **handleinfo)
3432 {
3433 nt_objref *nr;
3434
3435 nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO);
3436 if (nr == NULL)
3437 return (STATUS_INSUFFICIENT_RESOURCES);
3438
3439 InitializeListHead((&nr->no_dh.dh_waitlisthead));
3440 nr->no_obj = handle;
3441 nr->no_dh.dh_type = DISP_TYPE_THREAD;
3442 nr->no_dh.dh_sigstate = 0;
3443 nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) /
3444 sizeof(uint32_t));
3445 TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
3446 *object = nr;
3447
3448 return (STATUS_SUCCESS);
3449 }
3450
3451 static void
3452 ObfDereferenceObject(object)
3453 void *object;
3454 {
3455 nt_objref *nr;
3456
3457 nr = object;
3458 TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
3459 free(nr, M_DEVBUF);
3460 }
3461
3462 static uint32_t
3463 ZwClose(handle)
3464 ndis_handle handle;
3465 {
3466 return (STATUS_SUCCESS);
3467 }
3468
3469 static uint32_t
3470 WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf)
3471 uint32_t traceclass;
3472 void *traceinfo;
3473 uint32_t infolen;
3474 uint32_t reqlen;
3475 void *buf;
3476 {
3477 return (STATUS_NOT_FOUND);
3478 }
3479
3480 static uint32_t
3481 WmiTraceMessage(uint64_t loghandle, uint32_t messageflags,
3482 void *guid, uint16_t messagenum, ...)
3483 {
3484 return (STATUS_SUCCESS);
3485 }
3486
3487 static uint32_t
3488 IoWMIRegistrationControl(dobj, action)
3489 device_object *dobj;
3490 uint32_t action;
3491 {
3492 return (STATUS_SUCCESS);
3493 }
3494
3495 /*
3496 * This is here just in case the thread returns without calling
3497 * PsTerminateSystemThread().
3498 */
3499 static void
3500 ntoskrnl_thrfunc(arg)
3501 void *arg;
3502 {
3503 thread_context *thrctx;
3504 uint32_t (*tfunc)(void *);
3505 void *tctx;
3506 uint32_t rval;
3507
3508 thrctx = arg;
3509 tfunc = thrctx->tc_thrfunc;
3510 tctx = thrctx->tc_thrctx;
3511 free(thrctx, M_TEMP);
3512
3513 rval = MSCALL1(tfunc, tctx);
3514
3515 PsTerminateSystemThread(rval);
3516 return; /* notreached */
3517 }
3518
3519 static ndis_status
3520 PsCreateSystemThread(handle, reqaccess, objattrs, phandle,
3521 clientid, thrfunc, thrctx)
3522 ndis_handle *handle;
3523 uint32_t reqaccess;
3524 void *objattrs;
3525 ndis_handle phandle;
3526 void *clientid;
3527 void *thrfunc;
3528 void *thrctx;
3529 {
3530 int error;
3531 thread_context *tc;
3532 struct proc *p;
3533
3534 tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT);
3535 if (tc == NULL)
3536 return (STATUS_INSUFFICIENT_RESOURCES);
3537
3538 tc->tc_thrctx = thrctx;
3539 tc->tc_thrfunc = thrfunc;
3540
3541 error = kproc_create(ntoskrnl_thrfunc, tc, &p,
3542 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth);
3543
3544 if (error) {
3545 free(tc, M_TEMP);
3546 return (STATUS_INSUFFICIENT_RESOURCES);
3547 }
3548
3549 *handle = p;
3550 ntoskrnl_kth++;
3551
3552 return (STATUS_SUCCESS);
3553 }
3554
3555 /*
3556 * In Windows, the exit of a thread is an event that you're allowed
3557 * to wait on, assuming you've obtained a reference to the thread using
3558 * ObReferenceObjectByHandle(). Unfortunately, the only way we can
3559 * simulate this behavior is to register each thread we create in a
3560 * reference list, and if someone holds a reference to us, we poke
3561 * them.
3562 */
3563 static ndis_status
3564 PsTerminateSystemThread(status)
3565 ndis_status status;
3566 {
3567 struct nt_objref *nr;
3568
3569 mtx_lock(&ntoskrnl_dispatchlock);
3570 TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
3571 if (nr->no_obj != curthread->td_proc)
3572 continue;
3573 nr->no_dh.dh_sigstate = 1;
3574 ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT);
3575 break;
3576 }
3577 mtx_unlock(&ntoskrnl_dispatchlock);
3578
3579 ntoskrnl_kth--;
3580
3581 kproc_exit(0);
3582 return (0); /* notreached */
3583 }
3584
3585 static uint32_t
3586 DbgPrint(char *fmt, ...)
3587 {
3588 va_list ap;
3589
3590 if (bootverbose) {
3591 va_start(ap, fmt);
3592 vprintf(fmt, ap);
3593 va_end(ap);
3594 }
3595
3596 return (STATUS_SUCCESS);
3597 }
3598
3599 static void
3600 DbgBreakPoint(void)
3601 {
3602
3603 kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint");
3604 }
3605
3606 static void
3607 KeBugCheckEx(code, param1, param2, param3, param4)
3608 uint32_t code;
3609 u_long param1;
3610 u_long param2;
3611 u_long param3;
3612 u_long param4;
3613 {
3614 panic("KeBugCheckEx: STOP 0x%X", code);
3615 }
3616
3617 static void
3618 ntoskrnl_timercall(arg)
3619 void *arg;
3620 {
3621 ktimer *timer;
3622 struct timeval tv;
3623 kdpc *dpc;
3624
3625 mtx_lock(&ntoskrnl_dispatchlock);
3626
3627 timer = arg;
3628
3629 #ifdef NTOSKRNL_DEBUG_TIMERS
3630 ntoskrnl_timer_fires++;
3631 #endif
3632 ntoskrnl_remove_timer(timer);
3633
3634 /*
3635 * This should never happen, but complain
3636 * if it does.
3637 */
3638
3639 if (timer->k_header.dh_inserted == FALSE) {
3640 mtx_unlock(&ntoskrnl_dispatchlock);
3641 printf("NTOS: timer %p fired even though "
3642 "it was canceled\n", timer);
3643 return;
3644 }
3645
3646 /* Mark the timer as no longer being on the timer queue. */
3647
3648 timer->k_header.dh_inserted = FALSE;
3649
3650 /* Now signal the object and satisfy any waits on it. */
3651
3652 timer->k_header.dh_sigstate = 1;
3653 ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT);
3654
3655 /*
3656 * If this is a periodic timer, re-arm it
3657 * so it will fire again. We do this before
3658 * calling any deferred procedure calls because
3659 * it's possible the DPC might cancel the timer,
3660 * in which case it would be wrong for us to
3661 * re-arm it again afterwards.
3662 */
3663
3664 if (timer->k_period) {
3665 tv.tv_sec = 0;
3666 tv.tv_usec = timer->k_period * 1000;
3667 timer->k_header.dh_inserted = TRUE;
3668 ntoskrnl_insert_timer(timer, tvtohz(&tv));
3669 #ifdef NTOSKRNL_DEBUG_TIMERS
3670 ntoskrnl_timer_reloads++;
3671 #endif
3672 }
3673
3674 dpc = timer->k_dpc;
3675
3676 mtx_unlock(&ntoskrnl_dispatchlock);
3677
3678 /* If there's a DPC associated with the timer, queue it up. */
3679
3680 if (dpc != NULL)
3681 KeInsertQueueDpc(dpc, NULL, NULL);
3682 }
3683
3684 #ifdef NTOSKRNL_DEBUG_TIMERS
3685 static int
3686 sysctl_show_timers(SYSCTL_HANDLER_ARGS)
3687 {
3688 int ret;
3689
3690 ret = 0;
3691 ntoskrnl_show_timers();
3692 return (sysctl_handle_int(oidp, &ret, 0, req));
3693 }
3694
3695 static void
3696 ntoskrnl_show_timers()
3697 {
3698 int i = 0;
3699 list_entry *l;
3700
3701 mtx_lock_spin(&ntoskrnl_calllock);
3702 l = ntoskrnl_calllist.nle_flink;
3703 while(l != &ntoskrnl_calllist) {
3704 i++;
3705 l = l->nle_flink;
3706 }
3707 mtx_unlock_spin(&ntoskrnl_calllock);
3708
3709 printf("\n");
3710 printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS);
3711 printf("timer sets: %qu\n", ntoskrnl_timer_sets);
3712 printf("timer reloads: %qu\n", ntoskrnl_timer_reloads);
3713 printf("timer cancels: %qu\n", ntoskrnl_timer_cancels);
3714 printf("timer fires: %qu\n", ntoskrnl_timer_fires);
3715 printf("\n");
3716 }
3717 #endif
3718
3719 /*
3720 * Must be called with dispatcher lock held.
3721 */
3722
3723 static void
3724 ntoskrnl_insert_timer(timer, ticks)
3725 ktimer *timer;
3726 int ticks;
3727 {
3728 callout_entry *e;
3729 list_entry *l;
3730 struct callout *c;
3731
3732 /*
3733 * Try and allocate a timer.
3734 */
3735 mtx_lock_spin(&ntoskrnl_calllock);
3736 if (IsListEmpty(&ntoskrnl_calllist)) {
3737 mtx_unlock_spin(&ntoskrnl_calllock);
3738 #ifdef NTOSKRNL_DEBUG_TIMERS
3739 ntoskrnl_show_timers();
3740 #endif
3741 panic("out of timers!");
3742 }
3743 l = RemoveHeadList(&ntoskrnl_calllist);
3744 mtx_unlock_spin(&ntoskrnl_calllock);
3745
3746 e = CONTAINING_RECORD(l, callout_entry, ce_list);
3747 c = &e->ce_callout;
3748
3749 timer->k_callout = c;
3750
3751 callout_init(c, 1);
3752 callout_reset(c, ticks, ntoskrnl_timercall, timer);
3753 }
3754
3755 static void
3756 ntoskrnl_remove_timer(timer)
3757 ktimer *timer;
3758 {
3759 callout_entry *e;
3760
3761 e = (callout_entry *)timer->k_callout;
3762 callout_stop(timer->k_callout);
3763
3764 mtx_lock_spin(&ntoskrnl_calllock);
3765 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
3766 mtx_unlock_spin(&ntoskrnl_calllock);
3767 }
3768
3769 void
3770 KeInitializeTimer(timer)
3771 ktimer *timer;
3772 {
3773 if (timer == NULL)
3774 return;
3775
3776 KeInitializeTimerEx(timer, EVENT_TYPE_NOTIFY);
3777 }
3778
3779 void
3780 KeInitializeTimerEx(timer, type)
3781 ktimer *timer;
3782 uint32_t type;
3783 {
3784 if (timer == NULL)
3785 return;
3786
3787 bzero((char *)timer, sizeof(ktimer));
3788 InitializeListHead((&timer->k_header.dh_waitlisthead));
3789 timer->k_header.dh_sigstate = FALSE;
3790 timer->k_header.dh_inserted = FALSE;
3791 if (type == EVENT_TYPE_NOTIFY)
3792 timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER;
3793 else
3794 timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER;
3795 timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t);
3796 }
3797
3798 /*
3799 * DPC subsystem. A Windows Defered Procedure Call has the following
3800 * properties:
3801 * - It runs at DISPATCH_LEVEL.
3802 * - It can have one of 3 importance values that control when it
3803 * runs relative to other DPCs in the queue.
3804 * - On SMP systems, it can be set to run on a specific processor.
3805 * In order to satisfy the last property, we create a DPC thread for
3806 * each CPU in the system and bind it to that CPU. Each thread
3807 * maintains three queues with different importance levels, which
3808 * will be processed in order from lowest to highest.
3809 *
3810 * In Windows, interrupt handlers run as DPCs. (Not to be confused
3811 * with ISRs, which run in interrupt context and can preempt DPCs.)
3812 * ISRs are given the highest importance so that they'll take
3813 * precedence over timers and other things.
3814 */
3815
3816 static void
3817 ntoskrnl_dpc_thread(arg)
3818 void *arg;
3819 {
3820 kdpc_queue *kq;
3821 kdpc *d;
3822 list_entry *l;
3823 uint8_t irql;
3824
3825 kq = arg;
3826
3827 InitializeListHead(&kq->kq_disp);
3828 kq->kq_td = curthread;
3829 kq->kq_exit = 0;
3830 kq->kq_running = FALSE;
3831 KeInitializeSpinLock(&kq->kq_lock);
3832 KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
3833 KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE);
3834
3835 /*
3836 * Elevate our priority. DPCs are used to run interrupt
3837 * handlers, and they should trigger as soon as possible
3838 * once scheduled by an ISR.
3839 */
3840
3841 thread_lock(curthread);
3842 #ifdef NTOSKRNL_MULTIPLE_DPCS
3843 sched_bind(curthread, kq->kq_cpu);
3844 #endif
3845 sched_prio(curthread, PRI_MIN_KERN);
3846 thread_unlock(curthread);
3847
3848 while (1) {
3849 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
3850
3851 KeAcquireSpinLock(&kq->kq_lock, &irql);
3852
3853 if (kq->kq_exit) {
3854 kq->kq_exit = 0;
3855 KeReleaseSpinLock(&kq->kq_lock, irql);
3856 break;
3857 }
3858
3859 kq->kq_running = TRUE;
3860
3861 while (!IsListEmpty(&kq->kq_disp)) {
3862 l = RemoveHeadList((&kq->kq_disp));
3863 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3864 InitializeListHead((&d->k_dpclistentry));
3865 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
3866 MSCALL4(d->k_deferedfunc, d, d->k_deferredctx,
3867 d->k_sysarg1, d->k_sysarg2);
3868 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3869 }
3870
3871 kq->kq_running = FALSE;
3872
3873 KeReleaseSpinLock(&kq->kq_lock, irql);
3874
3875 KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE);
3876 }
3877
3878 kproc_exit(0);
3879 return; /* notreached */
3880 }
3881
3882 static void
3883 ntoskrnl_destroy_dpc_threads(void)
3884 {
3885 kdpc_queue *kq;
3886 kdpc dpc;
3887 int i;
3888
3889 kq = kq_queues;
3890 #ifdef NTOSKRNL_MULTIPLE_DPCS
3891 for (i = 0; i < mp_ncpus; i++) {
3892 #else
3893 for (i = 0; i < 1; i++) {
3894 #endif
3895 kq += i;
3896
3897 kq->kq_exit = 1;
3898 KeInitializeDpc(&dpc, NULL, NULL);
3899 KeSetTargetProcessorDpc(&dpc, i);
3900 KeInsertQueueDpc(&dpc, NULL, NULL);
3901 while (kq->kq_exit)
3902 tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10);
3903 }
3904 }
3905
3906 static uint8_t
3907 ntoskrnl_insert_dpc(head, dpc)
3908 list_entry *head;
3909 kdpc *dpc;
3910 {
3911 list_entry *l;
3912 kdpc *d;
3913
3914 l = head->nle_flink;
3915 while (l != head) {
3916 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3917 if (d == dpc)
3918 return (FALSE);
3919 l = l->nle_flink;
3920 }
3921
3922 if (dpc->k_importance == KDPC_IMPORTANCE_LOW)
3923 InsertTailList((head), (&dpc->k_dpclistentry));
3924 else
3925 InsertHeadList((head), (&dpc->k_dpclistentry));
3926
3927 return (TRUE);
3928 }
3929
3930 void
3931 KeInitializeDpc(dpc, dpcfunc, dpcctx)
3932 kdpc *dpc;
3933 void *dpcfunc;
3934 void *dpcctx;
3935 {
3936
3937 if (dpc == NULL)
3938 return;
3939
3940 dpc->k_deferedfunc = dpcfunc;
3941 dpc->k_deferredctx = dpcctx;
3942 dpc->k_num = KDPC_CPU_DEFAULT;
3943 dpc->k_importance = KDPC_IMPORTANCE_MEDIUM;
3944 InitializeListHead((&dpc->k_dpclistentry));
3945 }
3946
3947 uint8_t
3948 KeInsertQueueDpc(dpc, sysarg1, sysarg2)
3949 kdpc *dpc;
3950 void *sysarg1;
3951 void *sysarg2;
3952 {
3953 kdpc_queue *kq;
3954 uint8_t r;
3955 uint8_t irql;
3956
3957 if (dpc == NULL)
3958 return (FALSE);
3959
3960 kq = kq_queues;
3961
3962 #ifdef NTOSKRNL_MULTIPLE_DPCS
3963 KeRaiseIrql(DISPATCH_LEVEL, &irql);
3964
3965 /*
3966 * By default, the DPC is queued to run on the same CPU
3967 * that scheduled it.
3968 */
3969
3970 if (dpc->k_num == KDPC_CPU_DEFAULT)
3971 kq += curthread->td_oncpu;
3972 else
3973 kq += dpc->k_num;
3974 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3975 #else
3976 KeAcquireSpinLock(&kq->kq_lock, &irql);
3977 #endif
3978
3979 r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc);
3980 if (r == TRUE) {
3981 dpc->k_sysarg1 = sysarg1;
3982 dpc->k_sysarg2 = sysarg2;
3983 }
3984 KeReleaseSpinLock(&kq->kq_lock, irql);
3985
3986 if (r == FALSE)
3987 return (r);
3988
3989 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
3990
3991 return (r);
3992 }
3993
3994 uint8_t
3995 KeRemoveQueueDpc(dpc)
3996 kdpc *dpc;
3997 {
3998 kdpc_queue *kq;
3999 uint8_t irql;
4000
4001 if (dpc == NULL)
4002 return (FALSE);
4003
4004 #ifdef NTOSKRNL_MULTIPLE_DPCS
4005 KeRaiseIrql(DISPATCH_LEVEL, &irql);
4006
4007 kq = kq_queues + dpc->k_num;
4008
4009 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
4010 #else
4011 kq = kq_queues;
4012 KeAcquireSpinLock(&kq->kq_lock, &irql);
4013 #endif
4014
4015 if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) {
4016 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
4017 KeLowerIrql(irql);
4018 return (FALSE);
4019 }
4020
4021 RemoveEntryList((&dpc->k_dpclistentry));
4022 InitializeListHead((&dpc->k_dpclistentry));
4023
4024 KeReleaseSpinLock(&kq->kq_lock, irql);
4025
4026 return (TRUE);
4027 }
4028
4029 void
4030 KeSetImportanceDpc(dpc, imp)
4031 kdpc *dpc;
4032 uint32_t imp;
4033 {
4034 if (imp != KDPC_IMPORTANCE_LOW &&
4035 imp != KDPC_IMPORTANCE_MEDIUM &&
4036 imp != KDPC_IMPORTANCE_HIGH)
4037 return;
4038
4039 dpc->k_importance = (uint8_t)imp;
4040 }
4041
4042 void
4043 KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu)
4044 {
4045 if (cpu > mp_ncpus)
4046 return;
4047
4048 dpc->k_num = cpu;
4049 }
4050
4051 void
4052 KeFlushQueuedDpcs(void)
4053 {
4054 kdpc_queue *kq;
4055 int i;
4056
4057 /*
4058 * Poke each DPC queue and wait
4059 * for them to drain.
4060 */
4061
4062 #ifdef NTOSKRNL_MULTIPLE_DPCS
4063 for (i = 0; i < mp_ncpus; i++) {
4064 #else
4065 for (i = 0; i < 1; i++) {
4066 #endif
4067 kq = kq_queues + i;
4068 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
4069 KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL);
4070 }
4071 }
4072
4073 uint32_t
4074 KeGetCurrentProcessorNumber(void)
4075 {
4076 return ((uint32_t)curthread->td_oncpu);
4077 }
4078
4079 uint8_t
4080 KeSetTimerEx(timer, duetime, period, dpc)
4081 ktimer *timer;
4082 int64_t duetime;
4083 uint32_t period;
4084 kdpc *dpc;
4085 {
4086 struct timeval tv;
4087 uint64_t curtime;
4088 uint8_t pending;
4089
4090 if (timer == NULL)
4091 return (FALSE);
4092
4093 mtx_lock(&ntoskrnl_dispatchlock);
4094
4095 if (timer->k_header.dh_inserted == TRUE) {
4096 ntoskrnl_remove_timer(timer);
4097 #ifdef NTOSKRNL_DEBUG_TIMERS
4098 ntoskrnl_timer_cancels++;
4099 #endif
4100 timer->k_header.dh_inserted = FALSE;
4101 pending = TRUE;
4102 } else
4103 pending = FALSE;
4104
4105 timer->k_duetime = duetime;
4106 timer->k_period = period;
4107 timer->k_header.dh_sigstate = FALSE;
4108 timer->k_dpc = dpc;
4109
4110 if (duetime < 0) {
4111 tv.tv_sec = - (duetime) / 10000000;
4112 tv.tv_usec = (- (duetime) / 10) -
4113 (tv.tv_sec * 1000000);
4114 } else {
4115 ntoskrnl_time(&curtime);
4116 if (duetime < curtime)
4117 tv.tv_sec = tv.tv_usec = 0;
4118 else {
4119 tv.tv_sec = ((duetime) - curtime) / 10000000;
4120 tv.tv_usec = ((duetime) - curtime) / 10 -
4121 (tv.tv_sec * 1000000);
4122 }
4123 }
4124
4125 timer->k_header.dh_inserted = TRUE;
4126 ntoskrnl_insert_timer(timer, tvtohz(&tv));
4127 #ifdef NTOSKRNL_DEBUG_TIMERS
4128 ntoskrnl_timer_sets++;
4129 #endif
4130
4131 mtx_unlock(&ntoskrnl_dispatchlock);
4132
4133 return (pending);
4134 }
4135
4136 uint8_t
4137 KeSetTimer(timer, duetime, dpc)
4138 ktimer *timer;
4139 int64_t duetime;
4140 kdpc *dpc;
4141 {
4142 return (KeSetTimerEx(timer, duetime, 0, dpc));
4143 }
4144
4145 /*
4146 * The Windows DDK documentation seems to say that cancelling
4147 * a timer that has a DPC will result in the DPC also being
4148 * cancelled, but this isn't really the case.
4149 */
4150
4151 uint8_t
4152 KeCancelTimer(timer)
4153 ktimer *timer;
4154 {
4155 uint8_t pending;
4156
4157 if (timer == NULL)
4158 return (FALSE);
4159
4160 mtx_lock(&ntoskrnl_dispatchlock);
4161
4162 pending = timer->k_header.dh_inserted;
4163
4164 if (timer->k_header.dh_inserted == TRUE) {
4165 timer->k_header.dh_inserted = FALSE;
4166 ntoskrnl_remove_timer(timer);
4167 #ifdef NTOSKRNL_DEBUG_TIMERS
4168 ntoskrnl_timer_cancels++;
4169 #endif
4170 }
4171
4172 mtx_unlock(&ntoskrnl_dispatchlock);
4173
4174 return (pending);
4175 }
4176
4177 uint8_t
4178 KeReadStateTimer(timer)
4179 ktimer *timer;
4180 {
4181 return (timer->k_header.dh_sigstate);
4182 }
4183
4184 static int32_t
4185 KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval)
4186 {
4187 ktimer timer;
4188
4189 if (wait_mode != 0)
4190 panic("invalid wait_mode %d", wait_mode);
4191
4192 KeInitializeTimer(&timer);
4193 KeSetTimer(&timer, *interval, NULL);
4194 KeWaitForSingleObject(&timer, 0, 0, alertable, NULL);
4195
4196 return STATUS_SUCCESS;
4197 }
4198
4199 static uint64_t
4200 KeQueryInterruptTime(void)
4201 {
4202 int ticks;
4203 struct timeval tv;
4204
4205 getmicrouptime(&tv);
4206
4207 ticks = tvtohz(&tv);
4208
4209 return ticks * howmany(10000000, hz);
4210 }
4211
4212 static struct thread *
4213 KeGetCurrentThread(void)
4214 {
4215
4216 return curthread;
4217 }
4218
4219 static int32_t
4220 KeSetPriorityThread(td, pri)
4221 struct thread *td;
4222 int32_t pri;
4223 {
4224 int32_t old;
4225
4226 if (td == NULL)
4227 return LOW_REALTIME_PRIORITY;
4228
4229 if (td->td_priority <= PRI_MIN_KERN)
4230 old = HIGH_PRIORITY;
4231 else if (td->td_priority >= PRI_MAX_KERN)
4232 old = LOW_PRIORITY;
4233 else
4234 old = LOW_REALTIME_PRIORITY;
4235
4236 thread_lock(td);
4237 if (pri == HIGH_PRIORITY)
4238 sched_prio(td, PRI_MIN_KERN);
4239 if (pri == LOW_REALTIME_PRIORITY)
4240 sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2);
4241 if (pri == LOW_PRIORITY)
4242 sched_prio(td, PRI_MAX_KERN);
4243 thread_unlock(td);
4244
4245 return old;
4246 }
4247
4248 static void
4249 dummy()
4250 {
4251 printf("ntoskrnl dummy called...\n");
4252 }
4253
4254
4255 image_patch_table ntoskrnl_functbl[] = {
4256 IMPORT_SFUNC(RtlZeroMemory, 2),
4257 IMPORT_SFUNC(RtlSecureZeroMemory, 2),
4258 IMPORT_SFUNC(RtlFillMemory, 3),
4259 IMPORT_SFUNC(RtlMoveMemory, 3),
4260 IMPORT_SFUNC(RtlCharToInteger, 3),
4261 IMPORT_SFUNC(RtlCopyMemory, 3),
4262 IMPORT_SFUNC(RtlCopyString, 2),
4263 IMPORT_SFUNC(RtlCompareMemory, 3),
4264 IMPORT_SFUNC(RtlEqualUnicodeString, 3),
4265 IMPORT_SFUNC(RtlCopyUnicodeString, 2),
4266 IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3),
4267 IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3),
4268 IMPORT_SFUNC(RtlInitAnsiString, 2),
4269 IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2),
4270 IMPORT_SFUNC(RtlInitUnicodeString, 2),
4271 IMPORT_SFUNC(RtlFreeAnsiString, 1),
4272 IMPORT_SFUNC(RtlFreeUnicodeString, 1),
4273 IMPORT_SFUNC(RtlUnicodeStringToInteger, 3),
4274 IMPORT_CFUNC(sprintf, 0),
4275 IMPORT_CFUNC(vsprintf, 0),
4276 IMPORT_CFUNC_MAP(_snprintf, snprintf, 0),
4277 IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0),
4278 IMPORT_CFUNC(DbgPrint, 0),
4279 IMPORT_SFUNC(DbgBreakPoint, 0),
4280 IMPORT_SFUNC(KeBugCheckEx, 5),
4281 IMPORT_CFUNC(strncmp, 0),
4282 IMPORT_CFUNC(strcmp, 0),
4283 IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0),
4284 IMPORT_CFUNC(strncpy, 0),
4285 IMPORT_CFUNC(strcpy, 0),
4286 IMPORT_CFUNC(strlen, 0),
4287 IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0),
4288 IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0),
4289 IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0),
4290 IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0),
4291 IMPORT_CFUNC_MAP(strchr, index, 0),
4292 IMPORT_CFUNC_MAP(strrchr, rindex, 0),
4293 IMPORT_CFUNC(memcpy, 0),
4294 IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0),
4295 IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0),
4296 IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0),
4297 IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4),
4298 IMPORT_SFUNC(IoGetDriverObjectExtension, 2),
4299 IMPORT_FFUNC(IofCallDriver, 2),
4300 IMPORT_FFUNC(IofCompleteRequest, 2),
4301 IMPORT_SFUNC(IoAcquireCancelSpinLock, 1),
4302 IMPORT_SFUNC(IoReleaseCancelSpinLock, 1),
4303 IMPORT_SFUNC(IoCancelIrp, 1),
4304 IMPORT_SFUNC(IoConnectInterrupt, 11),
4305 IMPORT_SFUNC(IoDisconnectInterrupt, 1),
4306 IMPORT_SFUNC(IoCreateDevice, 7),
4307 IMPORT_SFUNC(IoDeleteDevice, 1),
4308 IMPORT_SFUNC(IoGetAttachedDevice, 1),
4309 IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2),
4310 IMPORT_SFUNC(IoDetachDevice, 1),
4311 IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7),
4312 IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6),
4313 IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9),
4314 IMPORT_SFUNC(IoAllocateIrp, 2),
4315 IMPORT_SFUNC(IoReuseIrp, 2),
4316 IMPORT_SFUNC(IoMakeAssociatedIrp, 2),
4317 IMPORT_SFUNC(IoFreeIrp, 1),
4318 IMPORT_SFUNC(IoInitializeIrp, 3),
4319 IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1),
4320 IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2),
4321 IMPORT_SFUNC(KeSynchronizeExecution, 3),
4322 IMPORT_SFUNC(KeWaitForSingleObject, 5),
4323 IMPORT_SFUNC(KeWaitForMultipleObjects, 8),
4324 IMPORT_SFUNC(_allmul, 4),
4325 IMPORT_SFUNC(_alldiv, 4),
4326 IMPORT_SFUNC(_allrem, 4),
4327 IMPORT_RFUNC(_allshr, 0),
4328 IMPORT_RFUNC(_allshl, 0),
4329 IMPORT_SFUNC(_aullmul, 4),
4330 IMPORT_SFUNC(_aulldiv, 4),
4331 IMPORT_SFUNC(_aullrem, 4),
4332 IMPORT_RFUNC(_aullshr, 0),
4333 IMPORT_RFUNC(_aullshl, 0),
4334 IMPORT_CFUNC(atoi, 0),
4335 IMPORT_CFUNC(atol, 0),
4336 IMPORT_CFUNC(rand, 0),
4337 IMPORT_CFUNC(srand, 0),
4338 IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2),
4339 IMPORT_SFUNC(READ_REGISTER_USHORT, 1),
4340 IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2),
4341 IMPORT_SFUNC(READ_REGISTER_ULONG, 1),
4342 IMPORT_SFUNC(READ_REGISTER_UCHAR, 1),
4343 IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2),
4344 IMPORT_SFUNC(ExInitializePagedLookasideList, 7),
4345 IMPORT_SFUNC(ExDeletePagedLookasideList, 1),
4346 IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7),
4347 IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1),
4348 IMPORT_FFUNC(InterlockedPopEntrySList, 1),
4349 IMPORT_FFUNC(InitializeSListHead, 1),
4350 IMPORT_FFUNC(InterlockedPushEntrySList, 2),
4351 IMPORT_SFUNC(ExQueryDepthSList, 1),
4352 IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList,
4353 InterlockedPopEntrySList, 1),
4354 IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList,
4355 InterlockedPushEntrySList, 2),
4356 IMPORT_FFUNC(ExInterlockedPopEntrySList, 2),
4357 IMPORT_FFUNC(ExInterlockedPushEntrySList, 3),
4358 IMPORT_SFUNC(ExAllocatePoolWithTag, 3),
4359 IMPORT_SFUNC(ExFreePoolWithTag, 2),
4360 IMPORT_SFUNC(ExFreePool, 1),
4361 #ifdef __i386__
4362 IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1),
4363 IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1),
4364 IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1),
4365 #else
4366 /*
4367 * For AMD64, we can get away with just mapping
4368 * KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock()
4369 * because the calling conventions end up being the same.
4370 * On i386, we have to be careful because KfAcquireSpinLock()
4371 * is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't.
4372 */
4373 IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1),
4374 IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1),
4375 IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1),
4376 #endif
4377 IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1),
4378 IMPORT_FFUNC(InterlockedIncrement, 1),
4379 IMPORT_FFUNC(InterlockedDecrement, 1),
4380 IMPORT_FFUNC(InterlockedExchange, 2),
4381 IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2),
4382 IMPORT_SFUNC(IoAllocateMdl, 5),
4383 IMPORT_SFUNC(IoFreeMdl, 1),
4384 IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1),
4385 IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3),
4386 IMPORT_SFUNC(MmFreeContiguousMemory, 1),
4387 IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3),
4388 IMPORT_SFUNC(MmSizeOfMdl, 1),
4389 IMPORT_SFUNC(MmMapLockedPages, 2),
4390 IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6),
4391 IMPORT_SFUNC(MmUnmapLockedPages, 2),
4392 IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1),
4393 IMPORT_SFUNC(MmGetPhysicalAddress, 1),
4394 IMPORT_SFUNC(MmGetSystemRoutineAddress, 1),
4395 IMPORT_SFUNC(MmIsAddressValid, 1),
4396 IMPORT_SFUNC(MmMapIoSpace, 3 + 1),
4397 IMPORT_SFUNC(MmUnmapIoSpace, 2),
4398 IMPORT_SFUNC(KeInitializeSpinLock, 1),
4399 IMPORT_SFUNC(IoIsWdmVersionAvailable, 2),
4400 IMPORT_SFUNC(IoOpenDeviceRegistryKey, 4),
4401 IMPORT_SFUNC(IoGetDeviceObjectPointer, 4),
4402 IMPORT_SFUNC(IoGetDeviceProperty, 5),
4403 IMPORT_SFUNC(IoAllocateWorkItem, 1),
4404 IMPORT_SFUNC(IoFreeWorkItem, 1),
4405 IMPORT_SFUNC(IoQueueWorkItem, 4),
4406 IMPORT_SFUNC(ExQueueWorkItem, 2),
4407 IMPORT_SFUNC(ntoskrnl_workitem, 2),
4408 IMPORT_SFUNC(KeInitializeMutex, 2),
4409 IMPORT_SFUNC(KeReleaseMutex, 2),
4410 IMPORT_SFUNC(KeReadStateMutex, 1),
4411 IMPORT_SFUNC(KeInitializeEvent, 3),
4412 IMPORT_SFUNC(KeSetEvent, 3),
4413 IMPORT_SFUNC(KeResetEvent, 1),
4414 IMPORT_SFUNC(KeClearEvent, 1),
4415 IMPORT_SFUNC(KeReadStateEvent, 1),
4416 IMPORT_SFUNC(KeInitializeTimer, 1),
4417 IMPORT_SFUNC(KeInitializeTimerEx, 2),
4418 IMPORT_SFUNC(KeSetTimer, 3),
4419 IMPORT_SFUNC(KeSetTimerEx, 4),
4420 IMPORT_SFUNC(KeCancelTimer, 1),
4421 IMPORT_SFUNC(KeReadStateTimer, 1),
4422 IMPORT_SFUNC(KeInitializeDpc, 3),
4423 IMPORT_SFUNC(KeInsertQueueDpc, 3),
4424 IMPORT_SFUNC(KeRemoveQueueDpc, 1),
4425 IMPORT_SFUNC(KeSetImportanceDpc, 2),
4426 IMPORT_SFUNC(KeSetTargetProcessorDpc, 2),
4427 IMPORT_SFUNC(KeFlushQueuedDpcs, 0),
4428 IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1),
4429 IMPORT_SFUNC(ObReferenceObjectByHandle, 6),
4430 IMPORT_FFUNC(ObfDereferenceObject, 1),
4431 IMPORT_SFUNC(ZwClose, 1),
4432 IMPORT_SFUNC(PsCreateSystemThread, 7),
4433 IMPORT_SFUNC(PsTerminateSystemThread, 1),
4434 IMPORT_SFUNC(IoWMIRegistrationControl, 2),
4435 IMPORT_SFUNC(WmiQueryTraceInformation, 5),
4436 IMPORT_CFUNC(WmiTraceMessage, 0),
4437 IMPORT_SFUNC(KeQuerySystemTime, 1),
4438 IMPORT_CFUNC(KeTickCount, 0),
4439 IMPORT_SFUNC(KeDelayExecutionThread, 3),
4440 IMPORT_SFUNC(KeQueryInterruptTime, 0),
4441 IMPORT_SFUNC(KeGetCurrentThread, 0),
4442 IMPORT_SFUNC(KeSetPriorityThread, 2),
4443
4444 /*
4445 * This last entry is a catch-all for any function we haven't
4446 * implemented yet. The PE import list patching routine will
4447 * use it for any function that doesn't have an explicit match
4448 * in this table.
4449 */
4450
4451 { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
4452
4453 /* End of list. */
4454
4455 { NULL, NULL, NULL }
4456 };
4457