1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (c) 2003
5 * Bill Paul <[email protected]>. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed by Bill Paul.
18 * 4. Neither the name of the author nor the names of any co-contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
37
38 #include <sys/ctype.h>
39 #include <sys/unistd.h>
40 #include <sys/param.h>
41 #include <sys/types.h>
42 #include <sys/errno.h>
43 #include <sys/systm.h>
44 #include <sys/malloc.h>
45 #include <sys/lock.h>
46 #include <sys/mutex.h>
47
48 #include <sys/callout.h>
49 #include <sys/kdb.h>
50 #include <sys/kernel.h>
51 #include <sys/proc.h>
52 #include <sys/condvar.h>
53 #include <sys/kthread.h>
54 #include <sys/module.h>
55 #include <sys/smp.h>
56 #include <sys/sched.h>
57 #include <sys/sysctl.h>
58
59 #include <machine/atomic.h>
60 #include <machine/bus.h>
61 #include <machine/stdarg.h>
62 #include <machine/resource.h>
63
64 #include <sys/bus.h>
65 #include <sys/rman.h>
66
67 #include <vm/vm.h>
68 #include <vm/vm_param.h>
69 #include <vm/pmap.h>
70 #include <vm/uma.h>
71 #include <vm/vm_kern.h>
72 #include <vm/vm_map.h>
73 #include <vm/vm_extern.h>
74
75 #include <compat/ndis/pe_var.h>
76 #include <compat/ndis/cfg_var.h>
77 #include <compat/ndis/resource_var.h>
78 #include <compat/ndis/ntoskrnl_var.h>
79 #include <compat/ndis/hal_var.h>
80 #include <compat/ndis/ndis_var.h>
81
82 #ifdef NTOSKRNL_DEBUG_TIMERS
83 static int sysctl_show_timers(SYSCTL_HANDLER_ARGS);
84
85 SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers,
86 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
87 sysctl_show_timers, "I",
88 "Show ntoskrnl timer stats");
89 #endif
90
91 struct kdpc_queue {
92 list_entry kq_disp;
93 struct thread *kq_td;
94 int kq_cpu;
95 int kq_exit;
96 int kq_running;
97 kspin_lock kq_lock;
98 nt_kevent kq_proc;
99 nt_kevent kq_done;
100 };
101
102 typedef struct kdpc_queue kdpc_queue;
103
104 struct wb_ext {
105 struct cv we_cv;
106 struct thread *we_td;
107 };
108
109 typedef struct wb_ext wb_ext;
110
111 #define NTOSKRNL_TIMEOUTS 256
112 #ifdef NTOSKRNL_DEBUG_TIMERS
113 static uint64_t ntoskrnl_timer_fires;
114 static uint64_t ntoskrnl_timer_sets;
115 static uint64_t ntoskrnl_timer_reloads;
116 static uint64_t ntoskrnl_timer_cancels;
117 #endif
118
119 struct callout_entry {
120 struct callout ce_callout;
121 list_entry ce_list;
122 };
123
124 typedef struct callout_entry callout_entry;
125
126 static struct list_entry ntoskrnl_calllist;
127 static struct mtx ntoskrnl_calllock;
128 struct kuser_shared_data kuser_shared_data;
129
130 static struct list_entry ntoskrnl_intlist;
131 static kspin_lock ntoskrnl_intlock;
132
133 static uint8_t RtlEqualUnicodeString(unicode_string *,
134 unicode_string *, uint8_t);
135 static void RtlCopyString(ansi_string *, const ansi_string *);
136 static void RtlCopyUnicodeString(unicode_string *,
137 unicode_string *);
138 static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *,
139 void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *);
140 static irp *IoBuildAsynchronousFsdRequest(uint32_t,
141 device_object *, void *, uint32_t, uint64_t *, io_status_block *);
142 static irp *IoBuildDeviceIoControlRequest(uint32_t,
143 device_object *, void *, uint32_t, void *, uint32_t,
144 uint8_t, nt_kevent *, io_status_block *);
145 static irp *IoAllocateIrp(uint8_t, uint8_t);
146 static void IoReuseIrp(irp *, uint32_t);
147 static void IoFreeIrp(irp *);
148 static void IoInitializeIrp(irp *, uint16_t, uint8_t);
149 static irp *IoMakeAssociatedIrp(irp *, uint8_t);
150 static uint32_t KeWaitForMultipleObjects(uint32_t,
151 nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
152 int64_t *, wait_block *);
153 static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t);
154 static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *);
155 static void ntoskrnl_satisfy_multiple_waits(wait_block *);
156 static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *);
157 static void ntoskrnl_insert_timer(ktimer *, int);
158 static void ntoskrnl_remove_timer(ktimer *);
159 #ifdef NTOSKRNL_DEBUG_TIMERS
160 static void ntoskrnl_show_timers(void);
161 #endif
162 static void ntoskrnl_timercall(void *);
163 static void ntoskrnl_dpc_thread(void *);
164 static void ntoskrnl_destroy_dpc_threads(void);
165 static void ntoskrnl_destroy_workitem_threads(void);
166 static void ntoskrnl_workitem_thread(void *);
167 static void ntoskrnl_workitem(device_object *, void *);
168 static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int);
169 static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int);
170 static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *);
171 static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t);
172 static uint16_t READ_REGISTER_USHORT(uint16_t *);
173 static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t);
174 static uint32_t READ_REGISTER_ULONG(uint32_t *);
175 static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t);
176 static uint8_t READ_REGISTER_UCHAR(uint8_t *);
177 static int64_t _allmul(int64_t, int64_t);
178 static int64_t _alldiv(int64_t, int64_t);
179 static int64_t _allrem(int64_t, int64_t);
180 static int64_t _allshr(int64_t, uint8_t);
181 static int64_t _allshl(int64_t, uint8_t);
182 static uint64_t _aullmul(uint64_t, uint64_t);
183 static uint64_t _aulldiv(uint64_t, uint64_t);
184 static uint64_t _aullrem(uint64_t, uint64_t);
185 static uint64_t _aullshr(uint64_t, uint8_t);
186 static uint64_t _aullshl(uint64_t, uint8_t);
187 static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
188 static void InitializeSListHead(slist_header *);
189 static slist_entry *ntoskrnl_popsl(slist_header *);
190 static void ExFreePoolWithTag(void *, uint32_t);
191 static void ExInitializePagedLookasideList(paged_lookaside_list *,
192 lookaside_alloc_func *, lookaside_free_func *,
193 uint32_t, size_t, uint32_t, uint16_t);
194 static void ExDeletePagedLookasideList(paged_lookaside_list *);
195 static void ExInitializeNPagedLookasideList(npaged_lookaside_list *,
196 lookaside_alloc_func *, lookaside_free_func *,
197 uint32_t, size_t, uint32_t, uint16_t);
198 static void ExDeleteNPagedLookasideList(npaged_lookaside_list *);
199 static slist_entry
200 *ExInterlockedPushEntrySList(slist_header *,
201 slist_entry *, kspin_lock *);
202 static slist_entry
203 *ExInterlockedPopEntrySList(slist_header *, kspin_lock *);
204 static uint32_t InterlockedIncrement(volatile uint32_t *);
205 static uint32_t InterlockedDecrement(volatile uint32_t *);
206 static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t);
207 static void *MmAllocateContiguousMemory(uint32_t, uint64_t);
208 static void *MmAllocateContiguousMemorySpecifyCache(uint32_t,
209 uint64_t, uint64_t, uint64_t, enum nt_caching_type);
210 static void MmFreeContiguousMemory(void *);
211 static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t,
212 enum nt_caching_type);
213 static uint32_t MmSizeOfMdl(void *, size_t);
214 static void *MmMapLockedPages(mdl *, uint8_t);
215 static void *MmMapLockedPagesSpecifyCache(mdl *,
216 uint8_t, uint32_t, void *, uint32_t, uint32_t);
217 static void MmUnmapLockedPages(void *, mdl *);
218 static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **);
219 static void RtlZeroMemory(void *, size_t);
220 static void RtlSecureZeroMemory(void *, size_t);
221 static void RtlFillMemory(void *, size_t, uint8_t);
222 static void RtlMoveMemory(void *, const void *, size_t);
223 static ndis_status RtlCharToInteger(const char *, uint32_t, uint32_t *);
224 static void RtlCopyMemory(void *, const void *, size_t);
225 static size_t RtlCompareMemory(const void *, const void *, size_t);
226 static ndis_status RtlUnicodeStringToInteger(unicode_string *,
227 uint32_t, uint32_t *);
228 static int atoi (const char *);
229 static long atol (const char *);
230 static int rand(void);
231 static void srand(unsigned int);
232 static void KeQuerySystemTime(uint64_t *);
233 static uint32_t KeTickCount(void);
234 static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t);
235 static int32_t IoOpenDeviceRegistryKey(struct device_object *, uint32_t,
236 uint32_t, void **);
237 static void ntoskrnl_thrfunc(void *);
238 static ndis_status PsCreateSystemThread(ndis_handle *,
239 uint32_t, void *, ndis_handle, void *, void *, void *);
240 static ndis_status PsTerminateSystemThread(ndis_status);
241 static ndis_status IoGetDeviceObjectPointer(unicode_string *,
242 uint32_t, void *, device_object *);
243 static ndis_status IoGetDeviceProperty(device_object *, uint32_t,
244 uint32_t, void *, uint32_t *);
245 static void KeInitializeMutex(kmutant *, uint32_t);
246 static uint32_t KeReleaseMutex(kmutant *, uint8_t);
247 static uint32_t KeReadStateMutex(kmutant *);
248 static ndis_status ObReferenceObjectByHandle(ndis_handle,
249 uint32_t, void *, uint8_t, void **, void **);
250 static void ObfDereferenceObject(void *);
251 static uint32_t ZwClose(ndis_handle);
252 static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t,
253 uint32_t, void *);
254 static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...);
255 static uint32_t IoWMIRegistrationControl(device_object *, uint32_t);
256 static void *ntoskrnl_memset(void *, int, size_t);
257 static void *ntoskrnl_memmove(void *, void *, size_t);
258 static void *ntoskrnl_memchr(void *, unsigned char, size_t);
259 static char *ntoskrnl_strstr(char *, char *);
260 static char *ntoskrnl_strncat(char *, char *, size_t);
261 static int ntoskrnl_toupper(int);
262 static int ntoskrnl_tolower(int);
263 static funcptr ntoskrnl_findwrap(funcptr);
264 static uint32_t DbgPrint(char *, ...);
265 static void DbgBreakPoint(void);
266 static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long);
267 static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *);
268 static int32_t KeSetPriorityThread(struct thread *, int32_t);
269 static void dummy(void);
270
271 static struct mtx ntoskrnl_dispatchlock;
272 static struct mtx ntoskrnl_interlock;
273 static kspin_lock ntoskrnl_cancellock;
274 static int ntoskrnl_kth = 0;
275 static struct nt_objref_head ntoskrnl_reflist;
276 static uma_zone_t mdl_zone;
277 static uma_zone_t iw_zone;
278 static struct kdpc_queue *kq_queues;
279 static struct kdpc_queue *wq_queues;
280 static int wq_idx = 0;
281
282 int
ntoskrnl_libinit()283 ntoskrnl_libinit()
284 {
285 image_patch_table *patch;
286 int error;
287 struct proc *p;
288 kdpc_queue *kq;
289 callout_entry *e;
290 int i;
291
292 mtx_init(&ntoskrnl_dispatchlock,
293 "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE);
294 mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
295 KeInitializeSpinLock(&ntoskrnl_cancellock);
296 KeInitializeSpinLock(&ntoskrnl_intlock);
297 TAILQ_INIT(&ntoskrnl_reflist);
298
299 InitializeListHead(&ntoskrnl_calllist);
300 InitializeListHead(&ntoskrnl_intlist);
301 mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
302
303 kq_queues = ExAllocatePoolWithTag(NonPagedPool,
304 #ifdef NTOSKRNL_MULTIPLE_DPCS
305 sizeof(kdpc_queue) * mp_ncpus, 0);
306 #else
307 sizeof(kdpc_queue), 0);
308 #endif
309
310 if (kq_queues == NULL)
311 return (ENOMEM);
312
313 wq_queues = ExAllocatePoolWithTag(NonPagedPool,
314 sizeof(kdpc_queue) * WORKITEM_THREADS, 0);
315
316 if (wq_queues == NULL)
317 return (ENOMEM);
318
319 #ifdef NTOSKRNL_MULTIPLE_DPCS
320 bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus);
321 #else
322 bzero((char *)kq_queues, sizeof(kdpc_queue));
323 #endif
324 bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS);
325
326 /*
327 * Launch the DPC threads.
328 */
329
330 #ifdef NTOSKRNL_MULTIPLE_DPCS
331 for (i = 0; i < mp_ncpus; i++) {
332 #else
333 for (i = 0; i < 1; i++) {
334 #endif
335 kq = kq_queues + i;
336 kq->kq_cpu = i;
337 error = kproc_create(ntoskrnl_dpc_thread, kq, &p,
338 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i);
339 if (error)
340 panic("failed to launch DPC thread");
341 }
342
343 /*
344 * Launch the workitem threads.
345 */
346
347 for (i = 0; i < WORKITEM_THREADS; i++) {
348 kq = wq_queues + i;
349 error = kproc_create(ntoskrnl_workitem_thread, kq, &p,
350 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i);
351 if (error)
352 panic("failed to launch workitem thread");
353 }
354
355 patch = ntoskrnl_functbl;
356 while (patch->ipt_func != NULL) {
357 windrv_wrap((funcptr)patch->ipt_func,
358 (funcptr *)&patch->ipt_wrap,
359 patch->ipt_argcnt, patch->ipt_ftype);
360 patch++;
361 }
362
363 for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) {
364 e = ExAllocatePoolWithTag(NonPagedPool,
365 sizeof(callout_entry), 0);
366 if (e == NULL)
367 panic("failed to allocate timeouts");
368 mtx_lock_spin(&ntoskrnl_calllock);
369 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
370 mtx_unlock_spin(&ntoskrnl_calllock);
371 }
372
373 /*
374 * MDLs are supposed to be variable size (they describe
375 * buffers containing some number of pages, but we don't
376 * know ahead of time how many pages that will be). But
377 * always allocating them off the heap is very slow. As
378 * a compromise, we create an MDL UMA zone big enough to
379 * handle any buffer requiring up to 16 pages, and we
380 * use those for any MDLs for buffers of 16 pages or less
381 * in size. For buffers larger than that (which we assume
382 * will be few and far between, we allocate the MDLs off
383 * the heap.
384 */
385
386 mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE,
387 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
388
389 iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem),
390 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
391
392 return (0);
393 }
394
395 int
396 ntoskrnl_libfini()
397 {
398 image_patch_table *patch;
399 callout_entry *e;
400 list_entry *l;
401
402 patch = ntoskrnl_functbl;
403 while (patch->ipt_func != NULL) {
404 windrv_unwrap(patch->ipt_wrap);
405 patch++;
406 }
407
408 /* Stop the workitem queues. */
409 ntoskrnl_destroy_workitem_threads();
410 /* Stop the DPC queues. */
411 ntoskrnl_destroy_dpc_threads();
412
413 ExFreePool(kq_queues);
414 ExFreePool(wq_queues);
415
416 uma_zdestroy(mdl_zone);
417 uma_zdestroy(iw_zone);
418
419 mtx_lock_spin(&ntoskrnl_calllock);
420 while(!IsListEmpty(&ntoskrnl_calllist)) {
421 l = RemoveHeadList(&ntoskrnl_calllist);
422 e = CONTAINING_RECORD(l, callout_entry, ce_list);
423 mtx_unlock_spin(&ntoskrnl_calllock);
424 ExFreePool(e);
425 mtx_lock_spin(&ntoskrnl_calllock);
426 }
427 mtx_unlock_spin(&ntoskrnl_calllock);
428
429 mtx_destroy(&ntoskrnl_dispatchlock);
430 mtx_destroy(&ntoskrnl_interlock);
431 mtx_destroy(&ntoskrnl_calllock);
432
433 return (0);
434 }
435
436 /*
437 * We need to be able to reference this externally from the wrapper;
438 * GCC only generates a local implementation of memset.
439 */
440 static void *
441 ntoskrnl_memset(buf, ch, size)
442 void *buf;
443 int ch;
444 size_t size;
445 {
446 return (memset(buf, ch, size));
447 }
448
449 static void *
450 ntoskrnl_memmove(dst, src, size)
451 void *src;
452 void *dst;
453 size_t size;
454 {
455 bcopy(src, dst, size);
456 return (dst);
457 }
458
459 static void *
460 ntoskrnl_memchr(void *buf, unsigned char ch, size_t len)
461 {
462 if (len != 0) {
463 unsigned char *p = buf;
464
465 do {
466 if (*p++ == ch)
467 return (p - 1);
468 } while (--len != 0);
469 }
470 return (NULL);
471 }
472
473 static char *
474 ntoskrnl_strstr(s, find)
475 char *s, *find;
476 {
477 char c, sc;
478 size_t len;
479
480 if ((c = *find++) != 0) {
481 len = strlen(find);
482 do {
483 do {
484 if ((sc = *s++) == 0)
485 return (NULL);
486 } while (sc != c);
487 } while (strncmp(s, find, len) != 0);
488 s--;
489 }
490 return ((char *)s);
491 }
492
493 /* Taken from libc */
494 static char *
495 ntoskrnl_strncat(dst, src, n)
496 char *dst;
497 char *src;
498 size_t n;
499 {
500 if (n != 0) {
501 char *d = dst;
502 const char *s = src;
503
504 while (*d != 0)
505 d++;
506 do {
507 if ((*d = *s++) == 0)
508 break;
509 d++;
510 } while (--n != 0);
511 *d = 0;
512 }
513 return (dst);
514 }
515
516 static int
517 ntoskrnl_toupper(c)
518 int c;
519 {
520 return (toupper(c));
521 }
522
523 static int
524 ntoskrnl_tolower(c)
525 int c;
526 {
527 return (tolower(c));
528 }
529
530 static uint8_t
531 RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2,
532 uint8_t caseinsensitive)
533 {
534 int i;
535
536 if (str1->us_len != str2->us_len)
537 return (FALSE);
538
539 for (i = 0; i < str1->us_len; i++) {
540 if (caseinsensitive == TRUE) {
541 if (toupper((char)(str1->us_buf[i] & 0xFF)) !=
542 toupper((char)(str2->us_buf[i] & 0xFF)))
543 return (FALSE);
544 } else {
545 if (str1->us_buf[i] != str2->us_buf[i])
546 return (FALSE);
547 }
548 }
549
550 return (TRUE);
551 }
552
553 static void
554 RtlCopyString(dst, src)
555 ansi_string *dst;
556 const ansi_string *src;
557 {
558 if (src != NULL && src->as_buf != NULL && dst->as_buf != NULL) {
559 dst->as_len = min(src->as_len, dst->as_maxlen);
560 memcpy(dst->as_buf, src->as_buf, dst->as_len);
561 if (dst->as_len < dst->as_maxlen)
562 dst->as_buf[dst->as_len] = 0;
563 } else
564 dst->as_len = 0;
565 }
566
567 static void
568 RtlCopyUnicodeString(dest, src)
569 unicode_string *dest;
570 unicode_string *src;
571 {
572
573 if (dest->us_maxlen >= src->us_len)
574 dest->us_len = src->us_len;
575 else
576 dest->us_len = dest->us_maxlen;
577 memcpy(dest->us_buf, src->us_buf, dest->us_len);
578 }
579
580 static void
581 ntoskrnl_ascii_to_unicode(ascii, unicode, len)
582 char *ascii;
583 uint16_t *unicode;
584 int len;
585 {
586 int i;
587 uint16_t *ustr;
588
589 ustr = unicode;
590 for (i = 0; i < len; i++) {
591 *ustr = (uint16_t)ascii[i];
592 ustr++;
593 }
594 }
595
596 static void
597 ntoskrnl_unicode_to_ascii(unicode, ascii, len)
598 uint16_t *unicode;
599 char *ascii;
600 int len;
601 {
602 int i;
603 uint8_t *astr;
604
605 astr = ascii;
606 for (i = 0; i < len / 2; i++) {
607 *astr = (uint8_t)unicode[i];
608 astr++;
609 }
610 }
611
612 uint32_t
613 RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate)
614 {
615 if (dest == NULL || src == NULL)
616 return (STATUS_INVALID_PARAMETER);
617
618 dest->as_len = src->us_len / 2;
619 if (dest->as_maxlen < dest->as_len)
620 dest->as_len = dest->as_maxlen;
621
622 if (allocate == TRUE) {
623 dest->as_buf = ExAllocatePoolWithTag(NonPagedPool,
624 (src->us_len / 2) + 1, 0);
625 if (dest->as_buf == NULL)
626 return (STATUS_INSUFFICIENT_RESOURCES);
627 dest->as_len = dest->as_maxlen = src->us_len / 2;
628 } else {
629 dest->as_len = src->us_len / 2; /* XXX */
630 if (dest->as_maxlen < dest->as_len)
631 dest->as_len = dest->as_maxlen;
632 }
633
634 ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf,
635 dest->as_len * 2);
636
637 return (STATUS_SUCCESS);
638 }
639
640 uint32_t
641 RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src,
642 uint8_t allocate)
643 {
644 if (dest == NULL || src == NULL)
645 return (STATUS_INVALID_PARAMETER);
646
647 if (allocate == TRUE) {
648 dest->us_buf = ExAllocatePoolWithTag(NonPagedPool,
649 src->as_len * 2, 0);
650 if (dest->us_buf == NULL)
651 return (STATUS_INSUFFICIENT_RESOURCES);
652 dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2;
653 } else {
654 dest->us_len = src->as_len * 2; /* XXX */
655 if (dest->us_maxlen < dest->us_len)
656 dest->us_len = dest->us_maxlen;
657 }
658
659 ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf,
660 dest->us_len / 2);
661
662 return (STATUS_SUCCESS);
663 }
664
665 void *
666 ExAllocatePoolWithTag(pooltype, len, tag)
667 uint32_t pooltype;
668 size_t len;
669 uint32_t tag;
670 {
671 void *buf;
672
673 buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
674 if (buf == NULL)
675 return (NULL);
676
677 return (buf);
678 }
679
680 static void
681 ExFreePoolWithTag(buf, tag)
682 void *buf;
683 uint32_t tag;
684 {
685 ExFreePool(buf);
686 }
687
688 void
689 ExFreePool(buf)
690 void *buf;
691 {
692 free(buf, M_DEVBUF);
693 }
694
695 uint32_t
696 IoAllocateDriverObjectExtension(drv, clid, extlen, ext)
697 driver_object *drv;
698 void *clid;
699 uint32_t extlen;
700 void **ext;
701 {
702 custom_extension *ce;
703
704 ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension)
705 + extlen, 0);
706
707 if (ce == NULL)
708 return (STATUS_INSUFFICIENT_RESOURCES);
709
710 ce->ce_clid = clid;
711 InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list));
712
713 *ext = (void *)(ce + 1);
714
715 return (STATUS_SUCCESS);
716 }
717
718 void *
719 IoGetDriverObjectExtension(drv, clid)
720 driver_object *drv;
721 void *clid;
722 {
723 list_entry *e;
724 custom_extension *ce;
725
726 /*
727 * Sanity check. Our dummy bus drivers don't have
728 * any driver extensions.
729 */
730
731 if (drv->dro_driverext == NULL)
732 return (NULL);
733
734 e = drv->dro_driverext->dre_usrext.nle_flink;
735 while (e != &drv->dro_driverext->dre_usrext) {
736 ce = (custom_extension *)e;
737 if (ce->ce_clid == clid)
738 return ((void *)(ce + 1));
739 e = e->nle_flink;
740 }
741
742 return (NULL);
743 }
744
745 uint32_t
746 IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname,
747 uint32_t devtype, uint32_t devchars, uint8_t exclusive,
748 device_object **newdev)
749 {
750 device_object *dev;
751
752 dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0);
753 if (dev == NULL)
754 return (STATUS_INSUFFICIENT_RESOURCES);
755
756 dev->do_type = devtype;
757 dev->do_drvobj = drv;
758 dev->do_currirp = NULL;
759 dev->do_flags = 0;
760
761 if (devextlen) {
762 dev->do_devext = ExAllocatePoolWithTag(NonPagedPool,
763 devextlen, 0);
764
765 if (dev->do_devext == NULL) {
766 ExFreePool(dev);
767 return (STATUS_INSUFFICIENT_RESOURCES);
768 }
769
770 bzero(dev->do_devext, devextlen);
771 } else
772 dev->do_devext = NULL;
773
774 dev->do_size = sizeof(device_object) + devextlen;
775 dev->do_refcnt = 1;
776 dev->do_attacheddev = NULL;
777 dev->do_nextdev = NULL;
778 dev->do_devtype = devtype;
779 dev->do_stacksize = 1;
780 dev->do_alignreq = 1;
781 dev->do_characteristics = devchars;
782 dev->do_iotimer = NULL;
783 KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE);
784
785 /*
786 * Vpd is used for disk/tape devices,
787 * but we don't support those. (Yet.)
788 */
789 dev->do_vpb = NULL;
790
791 dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool,
792 sizeof(devobj_extension), 0);
793
794 if (dev->do_devobj_ext == NULL) {
795 if (dev->do_devext != NULL)
796 ExFreePool(dev->do_devext);
797 ExFreePool(dev);
798 return (STATUS_INSUFFICIENT_RESOURCES);
799 }
800
801 dev->do_devobj_ext->dve_type = 0;
802 dev->do_devobj_ext->dve_size = sizeof(devobj_extension);
803 dev->do_devobj_ext->dve_devobj = dev;
804
805 /*
806 * Attach this device to the driver object's list
807 * of devices. Note: this is not the same as attaching
808 * the device to the device stack. The driver's AddDevice
809 * routine must explicitly call IoAddDeviceToDeviceStack()
810 * to do that.
811 */
812
813 if (drv->dro_devobj == NULL) {
814 drv->dro_devobj = dev;
815 dev->do_nextdev = NULL;
816 } else {
817 dev->do_nextdev = drv->dro_devobj;
818 drv->dro_devobj = dev;
819 }
820
821 *newdev = dev;
822
823 return (STATUS_SUCCESS);
824 }
825
826 void
827 IoDeleteDevice(dev)
828 device_object *dev;
829 {
830 device_object *prev;
831
832 if (dev == NULL)
833 return;
834
835 if (dev->do_devobj_ext != NULL)
836 ExFreePool(dev->do_devobj_ext);
837
838 if (dev->do_devext != NULL)
839 ExFreePool(dev->do_devext);
840
841 /* Unlink the device from the driver's device list. */
842
843 prev = dev->do_drvobj->dro_devobj;
844 if (prev == dev)
845 dev->do_drvobj->dro_devobj = dev->do_nextdev;
846 else {
847 while (prev->do_nextdev != dev)
848 prev = prev->do_nextdev;
849 prev->do_nextdev = dev->do_nextdev;
850 }
851
852 ExFreePool(dev);
853 }
854
855 device_object *
856 IoGetAttachedDevice(dev)
857 device_object *dev;
858 {
859 device_object *d;
860
861 if (dev == NULL)
862 return (NULL);
863
864 d = dev;
865
866 while (d->do_attacheddev != NULL)
867 d = d->do_attacheddev;
868
869 return (d);
870 }
871
872 static irp *
873 IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
874 uint32_t func;
875 device_object *dobj;
876 void *buf;
877 uint32_t len;
878 uint64_t *off;
879 nt_kevent *event;
880 io_status_block *status;
881 {
882 irp *ip;
883
884 ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
885 if (ip == NULL)
886 return (NULL);
887 ip->irp_usrevent = event;
888
889 return (ip);
890 }
891
892 static irp *
893 IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
894 uint32_t func;
895 device_object *dobj;
896 void *buf;
897 uint32_t len;
898 uint64_t *off;
899 io_status_block *status;
900 {
901 irp *ip;
902 io_stack_location *sl;
903
904 ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
905 if (ip == NULL)
906 return (NULL);
907
908 ip->irp_usriostat = status;
909 ip->irp_tail.irp_overlay.irp_thread = NULL;
910
911 sl = IoGetNextIrpStackLocation(ip);
912 sl->isl_major = func;
913 sl->isl_minor = 0;
914 sl->isl_flags = 0;
915 sl->isl_ctl = 0;
916 sl->isl_devobj = dobj;
917 sl->isl_fileobj = NULL;
918 sl->isl_completionfunc = NULL;
919
920 ip->irp_userbuf = buf;
921
922 if (dobj->do_flags & DO_BUFFERED_IO) {
923 ip->irp_assoc.irp_sysbuf =
924 ExAllocatePoolWithTag(NonPagedPool, len, 0);
925 if (ip->irp_assoc.irp_sysbuf == NULL) {
926 IoFreeIrp(ip);
927 return (NULL);
928 }
929 bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
930 }
931
932 if (dobj->do_flags & DO_DIRECT_IO) {
933 ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
934 if (ip->irp_mdl == NULL) {
935 if (ip->irp_assoc.irp_sysbuf != NULL)
936 ExFreePool(ip->irp_assoc.irp_sysbuf);
937 IoFreeIrp(ip);
938 return (NULL);
939 }
940 ip->irp_userbuf = NULL;
941 ip->irp_assoc.irp_sysbuf = NULL;
942 }
943
944 if (func == IRP_MJ_READ) {
945 sl->isl_parameters.isl_read.isl_len = len;
946 if (off != NULL)
947 sl->isl_parameters.isl_read.isl_byteoff = *off;
948 else
949 sl->isl_parameters.isl_read.isl_byteoff = 0;
950 }
951
952 if (func == IRP_MJ_WRITE) {
953 sl->isl_parameters.isl_write.isl_len = len;
954 if (off != NULL)
955 sl->isl_parameters.isl_write.isl_byteoff = *off;
956 else
957 sl->isl_parameters.isl_write.isl_byteoff = 0;
958 }
959
960 return (ip);
961 }
962
963 static irp *
964 IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf,
965 uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal,
966 nt_kevent *event, io_status_block *status)
967 {
968 irp *ip;
969 io_stack_location *sl;
970 uint32_t buflen;
971
972 ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
973 if (ip == NULL)
974 return (NULL);
975 ip->irp_usrevent = event;
976 ip->irp_usriostat = status;
977 ip->irp_tail.irp_overlay.irp_thread = NULL;
978
979 sl = IoGetNextIrpStackLocation(ip);
980 sl->isl_major = isinternal == TRUE ?
981 IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
982 sl->isl_minor = 0;
983 sl->isl_flags = 0;
984 sl->isl_ctl = 0;
985 sl->isl_devobj = dobj;
986 sl->isl_fileobj = NULL;
987 sl->isl_completionfunc = NULL;
988 sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
989 sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
990 sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
991
992 switch(IO_METHOD(iocode)) {
993 case METHOD_BUFFERED:
994 if (ilen > olen)
995 buflen = ilen;
996 else
997 buflen = olen;
998 if (buflen) {
999 ip->irp_assoc.irp_sysbuf =
1000 ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
1001 if (ip->irp_assoc.irp_sysbuf == NULL) {
1002 IoFreeIrp(ip);
1003 return (NULL);
1004 }
1005 }
1006 if (ilen && ibuf != NULL) {
1007 bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1008 bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
1009 buflen - ilen);
1010 } else
1011 bzero(ip->irp_assoc.irp_sysbuf, ilen);
1012 ip->irp_userbuf = obuf;
1013 break;
1014 case METHOD_IN_DIRECT:
1015 case METHOD_OUT_DIRECT:
1016 if (ilen && ibuf != NULL) {
1017 ip->irp_assoc.irp_sysbuf =
1018 ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
1019 if (ip->irp_assoc.irp_sysbuf == NULL) {
1020 IoFreeIrp(ip);
1021 return (NULL);
1022 }
1023 bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1024 }
1025 if (olen && obuf != NULL) {
1026 ip->irp_mdl = IoAllocateMdl(obuf, olen,
1027 FALSE, FALSE, ip);
1028 /*
1029 * Normally we would MmProbeAndLockPages()
1030 * here, but we don't have to in our
1031 * imlementation.
1032 */
1033 }
1034 break;
1035 case METHOD_NEITHER:
1036 ip->irp_userbuf = obuf;
1037 sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
1038 break;
1039 default:
1040 break;
1041 }
1042
1043 /*
1044 * Ideally, we should associate this IRP with the calling
1045 * thread here.
1046 */
1047
1048 return (ip);
1049 }
1050
1051 static irp *
1052 IoAllocateIrp(uint8_t stsize, uint8_t chargequota)
1053 {
1054 irp *i;
1055
1056 i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0);
1057 if (i == NULL)
1058 return (NULL);
1059
1060 IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize);
1061
1062 return (i);
1063 }
1064
1065 static irp *
1066 IoMakeAssociatedIrp(irp *ip, uint8_t stsize)
1067 {
1068 irp *associrp;
1069
1070 associrp = IoAllocateIrp(stsize, FALSE);
1071 if (associrp == NULL)
1072 return (NULL);
1073
1074 mtx_lock(&ntoskrnl_dispatchlock);
1075 associrp->irp_flags |= IRP_ASSOCIATED_IRP;
1076 associrp->irp_tail.irp_overlay.irp_thread =
1077 ip->irp_tail.irp_overlay.irp_thread;
1078 associrp->irp_assoc.irp_master = ip;
1079 mtx_unlock(&ntoskrnl_dispatchlock);
1080
1081 return (associrp);
1082 }
1083
1084 static void
1085 IoFreeIrp(ip)
1086 irp *ip;
1087 {
1088 ExFreePool(ip);
1089 }
1090
1091 static void
1092 IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize)
1093 {
1094 bzero((char *)io, IoSizeOfIrp(ssize));
1095 io->irp_size = psize;
1096 io->irp_stackcnt = ssize;
1097 io->irp_currentstackloc = ssize;
1098 InitializeListHead(&io->irp_thlist);
1099 io->irp_tail.irp_overlay.irp_csl =
1100 (io_stack_location *)(io + 1) + ssize;
1101 }
1102
1103 static void
1104 IoReuseIrp(ip, status)
1105 irp *ip;
1106 uint32_t status;
1107 {
1108 uint8_t allocflags;
1109
1110 allocflags = ip->irp_allocflags;
1111 IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt);
1112 ip->irp_iostat.isb_status = status;
1113 ip->irp_allocflags = allocflags;
1114 }
1115
1116 void
1117 IoAcquireCancelSpinLock(uint8_t *irql)
1118 {
1119 KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
1120 }
1121
1122 void
1123 IoReleaseCancelSpinLock(uint8_t irql)
1124 {
1125 KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
1126 }
1127
1128 uint8_t
1129 IoCancelIrp(irp *ip)
1130 {
1131 cancel_func cfunc;
1132 uint8_t cancelirql;
1133
1134 IoAcquireCancelSpinLock(&cancelirql);
1135 cfunc = IoSetCancelRoutine(ip, NULL);
1136 ip->irp_cancel = TRUE;
1137 if (cfunc == NULL) {
1138 IoReleaseCancelSpinLock(cancelirql);
1139 return (FALSE);
1140 }
1141 ip->irp_cancelirql = cancelirql;
1142 MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
1143 return (uint8_t)IoSetCancelValue(ip, TRUE);
1144 }
1145
1146 uint32_t
1147 IofCallDriver(dobj, ip)
1148 device_object *dobj;
1149 irp *ip;
1150 {
1151 driver_object *drvobj;
1152 io_stack_location *sl;
1153 uint32_t status;
1154 driver_dispatch disp;
1155
1156 drvobj = dobj->do_drvobj;
1157
1158 if (ip->irp_currentstackloc <= 0)
1159 panic("IoCallDriver(): out of stack locations");
1160
1161 IoSetNextIrpStackLocation(ip);
1162 sl = IoGetCurrentIrpStackLocation(ip);
1163
1164 sl->isl_devobj = dobj;
1165
1166 disp = drvobj->dro_dispatch[sl->isl_major];
1167 status = MSCALL2(disp, dobj, ip);
1168
1169 return (status);
1170 }
1171
1172 void
1173 IofCompleteRequest(irp *ip, uint8_t prioboost)
1174 {
1175 uint32_t status;
1176 device_object *dobj;
1177 io_stack_location *sl;
1178 completion_func cf;
1179
1180 KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING,
1181 ("incorrect IRP(%p) status (STATUS_PENDING)", ip));
1182
1183 sl = IoGetCurrentIrpStackLocation(ip);
1184 IoSkipCurrentIrpStackLocation(ip);
1185
1186 do {
1187 if (sl->isl_ctl & SL_PENDING_RETURNED)
1188 ip->irp_pendingreturned = TRUE;
1189
1190 if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1))
1191 dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj;
1192 else
1193 dobj = NULL;
1194
1195 if (sl->isl_completionfunc != NULL &&
1196 ((ip->irp_iostat.isb_status == STATUS_SUCCESS &&
1197 sl->isl_ctl & SL_INVOKE_ON_SUCCESS) ||
1198 (ip->irp_iostat.isb_status != STATUS_SUCCESS &&
1199 sl->isl_ctl & SL_INVOKE_ON_ERROR) ||
1200 (ip->irp_cancel == TRUE &&
1201 sl->isl_ctl & SL_INVOKE_ON_CANCEL))) {
1202 cf = sl->isl_completionfunc;
1203 status = MSCALL3(cf, dobj, ip, sl->isl_completionctx);
1204 if (status == STATUS_MORE_PROCESSING_REQUIRED)
1205 return;
1206 } else {
1207 if ((ip->irp_currentstackloc <= ip->irp_stackcnt) &&
1208 (ip->irp_pendingreturned == TRUE))
1209 IoMarkIrpPending(ip);
1210 }
1211
1212 /* move to the next. */
1213 IoSkipCurrentIrpStackLocation(ip);
1214 sl++;
1215 } while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
1216
1217 if (ip->irp_usriostat != NULL)
1218 *ip->irp_usriostat = ip->irp_iostat;
1219 if (ip->irp_usrevent != NULL)
1220 KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
1221
1222 /* Handle any associated IRPs. */
1223
1224 if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
1225 uint32_t masterirpcnt;
1226 irp *masterirp;
1227 mdl *m;
1228
1229 masterirp = ip->irp_assoc.irp_master;
1230 masterirpcnt =
1231 InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt);
1232
1233 while ((m = ip->irp_mdl) != NULL) {
1234 ip->irp_mdl = m->mdl_next;
1235 IoFreeMdl(m);
1236 }
1237 IoFreeIrp(ip);
1238 if (masterirpcnt == 0)
1239 IoCompleteRequest(masterirp, IO_NO_INCREMENT);
1240 return;
1241 }
1242
1243 /* With any luck, these conditions will never arise. */
1244
1245 if (ip->irp_flags & IRP_PAGING_IO) {
1246 if (ip->irp_mdl != NULL)
1247 IoFreeMdl(ip->irp_mdl);
1248 IoFreeIrp(ip);
1249 }
1250 }
1251
1252 void
1253 ntoskrnl_intr(arg)
1254 void *arg;
1255 {
1256 kinterrupt *iobj;
1257 uint8_t irql;
1258 uint8_t claimed;
1259 list_entry *l;
1260
1261 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1262 l = ntoskrnl_intlist.nle_flink;
1263 while (l != &ntoskrnl_intlist) {
1264 iobj = CONTAINING_RECORD(l, kinterrupt, ki_list);
1265 claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx);
1266 if (claimed == TRUE)
1267 break;
1268 l = l->nle_flink;
1269 }
1270 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1271 }
1272
1273 uint8_t
1274 KeAcquireInterruptSpinLock(iobj)
1275 kinterrupt *iobj;
1276 {
1277 uint8_t irql;
1278 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1279 return (irql);
1280 }
1281
1282 void
1283 KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql)
1284 {
1285 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1286 }
1287
1288 uint8_t
1289 KeSynchronizeExecution(iobj, syncfunc, syncctx)
1290 kinterrupt *iobj;
1291 void *syncfunc;
1292 void *syncctx;
1293 {
1294 uint8_t irql;
1295
1296 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1297 MSCALL1(syncfunc, syncctx);
1298 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1299
1300 return (TRUE);
1301 }
1302
1303 /*
1304 * IoConnectInterrupt() is passed only the interrupt vector and
1305 * irql that a device wants to use, but no device-specific tag
1306 * of any kind. This conflicts rather badly with FreeBSD's
1307 * bus_setup_intr(), which needs the device_t for the device
1308 * requesting interrupt delivery. In order to bypass this
1309 * inconsistency, we implement a second level of interrupt
1310 * dispatching on top of bus_setup_intr(). All devices use
1311 * ntoskrnl_intr() as their ISR, and any device requesting
1312 * interrupts will be registered with ntoskrnl_intr()'s interrupt
1313 * dispatch list. When an interrupt arrives, we walk the list
1314 * and invoke all the registered ISRs. This effectively makes all
1315 * interrupts shared, but it's the only way to duplicate the
1316 * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly.
1317 */
1318
1319 uint32_t
1320 IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx,
1321 kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql,
1322 uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat)
1323 {
1324 uint8_t curirql;
1325
1326 *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0);
1327 if (*iobj == NULL)
1328 return (STATUS_INSUFFICIENT_RESOURCES);
1329
1330 (*iobj)->ki_svcfunc = svcfunc;
1331 (*iobj)->ki_svcctx = svcctx;
1332
1333 if (lock == NULL) {
1334 KeInitializeSpinLock(&(*iobj)->ki_lock_priv);
1335 (*iobj)->ki_lock = &(*iobj)->ki_lock_priv;
1336 } else
1337 (*iobj)->ki_lock = lock;
1338
1339 KeAcquireSpinLock(&ntoskrnl_intlock, &curirql);
1340 InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list));
1341 KeReleaseSpinLock(&ntoskrnl_intlock, curirql);
1342
1343 return (STATUS_SUCCESS);
1344 }
1345
1346 void
1347 IoDisconnectInterrupt(iobj)
1348 kinterrupt *iobj;
1349 {
1350 uint8_t irql;
1351
1352 if (iobj == NULL)
1353 return;
1354
1355 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1356 RemoveEntryList((&iobj->ki_list));
1357 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1358
1359 ExFreePool(iobj);
1360 }
1361
1362 device_object *
1363 IoAttachDeviceToDeviceStack(src, dst)
1364 device_object *src;
1365 device_object *dst;
1366 {
1367 device_object *attached;
1368
1369 mtx_lock(&ntoskrnl_dispatchlock);
1370 attached = IoGetAttachedDevice(dst);
1371 attached->do_attacheddev = src;
1372 src->do_attacheddev = NULL;
1373 src->do_stacksize = attached->do_stacksize + 1;
1374 mtx_unlock(&ntoskrnl_dispatchlock);
1375
1376 return (attached);
1377 }
1378
1379 void
1380 IoDetachDevice(topdev)
1381 device_object *topdev;
1382 {
1383 device_object *tail;
1384
1385 mtx_lock(&ntoskrnl_dispatchlock);
1386
1387 /* First, break the chain. */
1388 tail = topdev->do_attacheddev;
1389 if (tail == NULL) {
1390 mtx_unlock(&ntoskrnl_dispatchlock);
1391 return;
1392 }
1393 topdev->do_attacheddev = tail->do_attacheddev;
1394 topdev->do_refcnt--;
1395
1396 /* Now reduce the stacksize count for the takm_il objects. */
1397
1398 tail = topdev->do_attacheddev;
1399 while (tail != NULL) {
1400 tail->do_stacksize--;
1401 tail = tail->do_attacheddev;
1402 }
1403
1404 mtx_unlock(&ntoskrnl_dispatchlock);
1405 }
1406
1407 /*
1408 * For the most part, an object is considered signalled if
1409 * dh_sigstate == TRUE. The exception is for mutant objects
1410 * (mutexes), where the logic works like this:
1411 *
1412 * - If the thread already owns the object and sigstate is
1413 * less than or equal to 0, then the object is considered
1414 * signalled (recursive acquisition).
1415 * - If dh_sigstate == 1, the object is also considered
1416 * signalled.
1417 */
1418
1419 static int
1420 ntoskrnl_is_signalled(obj, td)
1421 nt_dispatch_header *obj;
1422 struct thread *td;
1423 {
1424 kmutant *km;
1425
1426 if (obj->dh_type == DISP_TYPE_MUTANT) {
1427 km = (kmutant *)obj;
1428 if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) ||
1429 obj->dh_sigstate == 1)
1430 return (TRUE);
1431 return (FALSE);
1432 }
1433
1434 if (obj->dh_sigstate > 0)
1435 return (TRUE);
1436 return (FALSE);
1437 }
1438
1439 static void
1440 ntoskrnl_satisfy_wait(obj, td)
1441 nt_dispatch_header *obj;
1442 struct thread *td;
1443 {
1444 kmutant *km;
1445
1446 switch (obj->dh_type) {
1447 case DISP_TYPE_MUTANT:
1448 km = (struct kmutant *)obj;
1449 obj->dh_sigstate--;
1450 /*
1451 * If sigstate reaches 0, the mutex is now
1452 * non-signalled (the new thread owns it).
1453 */
1454 if (obj->dh_sigstate == 0) {
1455 km->km_ownerthread = td;
1456 if (km->km_abandoned == TRUE)
1457 km->km_abandoned = FALSE;
1458 }
1459 break;
1460 /* Synchronization objects get reset to unsignalled. */
1461 case DISP_TYPE_SYNCHRONIZATION_EVENT:
1462 case DISP_TYPE_SYNCHRONIZATION_TIMER:
1463 obj->dh_sigstate = 0;
1464 break;
1465 case DISP_TYPE_SEMAPHORE:
1466 obj->dh_sigstate--;
1467 break;
1468 default:
1469 break;
1470 }
1471 }
1472
1473 static void
1474 ntoskrnl_satisfy_multiple_waits(wb)
1475 wait_block *wb;
1476 {
1477 wait_block *cur;
1478 struct thread *td;
1479
1480 cur = wb;
1481 td = wb->wb_kthread;
1482
1483 do {
1484 ntoskrnl_satisfy_wait(wb->wb_object, td);
1485 cur->wb_awakened = TRUE;
1486 cur = cur->wb_next;
1487 } while (cur != wb);
1488 }
1489
1490 /* Always called with dispatcher lock held. */
1491 static void
1492 ntoskrnl_waittest(obj, increment)
1493 nt_dispatch_header *obj;
1494 uint32_t increment;
1495 {
1496 wait_block *w, *next;
1497 list_entry *e;
1498 struct thread *td;
1499 wb_ext *we;
1500 int satisfied;
1501
1502 /*
1503 * Once an object has been signalled, we walk its list of
1504 * wait blocks. If a wait block can be awakened, then satisfy
1505 * waits as necessary and wake the thread.
1506 *
1507 * The rules work like this:
1508 *
1509 * If a wait block is marked as WAITTYPE_ANY, then
1510 * we can satisfy the wait conditions on the current
1511 * object and wake the thread right away. Satisfying
1512 * the wait also has the effect of breaking us out
1513 * of the search loop.
1514 *
1515 * If the object is marked as WAITTYLE_ALL, then the
1516 * wait block will be part of a circularly linked
1517 * list of wait blocks belonging to a waiting thread
1518 * that's sleeping in KeWaitForMultipleObjects(). In
1519 * order to wake the thread, all the objects in the
1520 * wait list must be in the signalled state. If they
1521 * are, we then satisfy all of them and wake the
1522 * thread.
1523 *
1524 */
1525
1526 e = obj->dh_waitlisthead.nle_flink;
1527
1528 while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) {
1529 w = CONTAINING_RECORD(e, wait_block, wb_waitlist);
1530 we = w->wb_ext;
1531 td = we->we_td;
1532 satisfied = FALSE;
1533 if (w->wb_waittype == WAITTYPE_ANY) {
1534 /*
1535 * Thread can be awakened if
1536 * any wait is satisfied.
1537 */
1538 ntoskrnl_satisfy_wait(obj, td);
1539 satisfied = TRUE;
1540 w->wb_awakened = TRUE;
1541 } else {
1542 /*
1543 * Thread can only be woken up
1544 * if all waits are satisfied.
1545 * If the thread is waiting on multiple
1546 * objects, they should all be linked
1547 * through the wb_next pointers in the
1548 * wait blocks.
1549 */
1550 satisfied = TRUE;
1551 next = w->wb_next;
1552 while (next != w) {
1553 if (ntoskrnl_is_signalled(obj, td) == FALSE) {
1554 satisfied = FALSE;
1555 break;
1556 }
1557 next = next->wb_next;
1558 }
1559 ntoskrnl_satisfy_multiple_waits(w);
1560 }
1561
1562 if (satisfied == TRUE)
1563 cv_broadcastpri(&we->we_cv,
1564 (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
1565 w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
1566
1567 e = e->nle_flink;
1568 }
1569 }
1570
1571 /*
1572 * Return the number of 100 nanosecond intervals since
1573 * January 1, 1601. (?!?!)
1574 */
1575 void
1576 ntoskrnl_time(tval)
1577 uint64_t *tval;
1578 {
1579 struct timespec ts;
1580
1581 nanotime(&ts);
1582 *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
1583 11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */
1584 }
1585
1586 static void
1587 KeQuerySystemTime(current_time)
1588 uint64_t *current_time;
1589 {
1590 ntoskrnl_time(current_time);
1591 }
1592
1593 static uint32_t
1594 KeTickCount(void)
1595 {
1596 struct timeval tv;
1597 getmicrouptime(&tv);
1598 return tvtohz(&tv);
1599 }
1600
1601 /*
1602 * KeWaitForSingleObject() is a tricky beast, because it can be used
1603 * with several different object types: semaphores, timers, events,
1604 * mutexes and threads. Semaphores don't appear very often, but the
1605 * other object types are quite common. KeWaitForSingleObject() is
1606 * what's normally used to acquire a mutex, and it can be used to
1607 * wait for a thread termination.
1608 *
1609 * The Windows NDIS API is implemented in terms of Windows kernel
1610 * primitives, and some of the object manipulation is duplicated in
1611 * NDIS. For example, NDIS has timers and events, which are actually
1612 * Windows kevents and ktimers. Now, you're supposed to only use the
1613 * NDIS variants of these objects within the confines of the NDIS API,
1614 * but there are some naughty developers out there who will use
1615 * KeWaitForSingleObject() on NDIS timer and event objects, so we
1616 * have to support that as well. Conseqently, our NDIS timer and event
1617 * code has to be closely tied into our ntoskrnl timer and event code,
1618 * just as it is in Windows.
1619 *
1620 * KeWaitForSingleObject() may do different things for different kinds
1621 * of objects:
1622 *
1623 * - For events, we check if the event has been signalled. If the
1624 * event is already in the signalled state, we just return immediately,
1625 * otherwise we wait for it to be set to the signalled state by someone
1626 * else calling KeSetEvent(). Events can be either synchronization or
1627 * notification events.
1628 *
1629 * - For timers, if the timer has already fired and the timer is in
1630 * the signalled state, we just return, otherwise we wait on the
1631 * timer. Unlike an event, timers get signalled automatically when
1632 * they expire rather than someone having to trip them manually.
1633 * Timers initialized with KeInitializeTimer() are always notification
1634 * events: KeInitializeTimerEx() lets you initialize a timer as
1635 * either a notification or synchronization event.
1636 *
1637 * - For mutexes, we try to acquire the mutex and if we can't, we wait
1638 * on the mutex until it's available and then grab it. When a mutex is
1639 * released, it enters the signalled state, which wakes up one of the
1640 * threads waiting to acquire it. Mutexes are always synchronization
1641 * events.
1642 *
1643 * - For threads, the only thing we do is wait until the thread object
1644 * enters a signalled state, which occurs when the thread terminates.
1645 * Threads are always notification events.
1646 *
1647 * A notification event wakes up all threads waiting on an object. A
1648 * synchronization event wakes up just one. Also, a synchronization event
1649 * is auto-clearing, which means we automatically set the event back to
1650 * the non-signalled state once the wakeup is done.
1651 */
1652
1653 uint32_t
1654 KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode,
1655 uint8_t alertable, int64_t *duetime)
1656 {
1657 wait_block w;
1658 struct thread *td = curthread;
1659 struct timeval tv;
1660 int error = 0;
1661 uint64_t curtime;
1662 wb_ext we;
1663 nt_dispatch_header *obj;
1664
1665 obj = arg;
1666
1667 if (obj == NULL)
1668 return (STATUS_INVALID_PARAMETER);
1669
1670 mtx_lock(&ntoskrnl_dispatchlock);
1671
1672 cv_init(&we.we_cv, "KeWFS");
1673 we.we_td = td;
1674
1675 /*
1676 * Check to see if this object is already signalled,
1677 * and just return without waiting if it is.
1678 */
1679 if (ntoskrnl_is_signalled(obj, td) == TRUE) {
1680 /* Sanity check the signal state value. */
1681 if (obj->dh_sigstate != INT32_MIN) {
1682 ntoskrnl_satisfy_wait(obj, curthread);
1683 mtx_unlock(&ntoskrnl_dispatchlock);
1684 return (STATUS_SUCCESS);
1685 } else {
1686 /*
1687 * There's a limit to how many times we can
1688 * recursively acquire a mutant. If we hit
1689 * the limit, something is very wrong.
1690 */
1691 if (obj->dh_type == DISP_TYPE_MUTANT) {
1692 mtx_unlock(&ntoskrnl_dispatchlock);
1693 panic("mutant limit exceeded");
1694 }
1695 }
1696 }
1697
1698 bzero((char *)&w, sizeof(wait_block));
1699 w.wb_object = obj;
1700 w.wb_ext = &we;
1701 w.wb_waittype = WAITTYPE_ANY;
1702 w.wb_next = &w;
1703 w.wb_waitkey = 0;
1704 w.wb_awakened = FALSE;
1705 w.wb_oldpri = td->td_priority;
1706
1707 InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist));
1708
1709 /*
1710 * The timeout value is specified in 100 nanosecond units
1711 * and can be a positive or negative number. If it's positive,
1712 * then the duetime is absolute, and we need to convert it
1713 * to an absolute offset relative to now in order to use it.
1714 * If it's negative, then the duetime is relative and we
1715 * just have to convert the units.
1716 */
1717
1718 if (duetime != NULL) {
1719 if (*duetime < 0) {
1720 tv.tv_sec = - (*duetime) / 10000000;
1721 tv.tv_usec = (- (*duetime) / 10) -
1722 (tv.tv_sec * 1000000);
1723 } else {
1724 ntoskrnl_time(&curtime);
1725 if (*duetime < curtime)
1726 tv.tv_sec = tv.tv_usec = 0;
1727 else {
1728 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1729 tv.tv_usec = ((*duetime) - curtime) / 10 -
1730 (tv.tv_sec * 1000000);
1731 }
1732 }
1733 }
1734
1735 if (duetime == NULL)
1736 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1737 else
1738 error = cv_timedwait(&we.we_cv,
1739 &ntoskrnl_dispatchlock, tvtohz(&tv));
1740
1741 RemoveEntryList(&w.wb_waitlist);
1742
1743 cv_destroy(&we.we_cv);
1744
1745 /* We timed out. Leave the object alone and return status. */
1746
1747 if (error == EWOULDBLOCK) {
1748 mtx_unlock(&ntoskrnl_dispatchlock);
1749 return (STATUS_TIMEOUT);
1750 }
1751
1752 mtx_unlock(&ntoskrnl_dispatchlock);
1753
1754 return (STATUS_SUCCESS);
1755 /*
1756 return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason,
1757 mode, alertable, duetime, &w));
1758 */
1759 }
1760
1761 static uint32_t
1762 KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype,
1763 uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime,
1764 wait_block *wb_array)
1765 {
1766 struct thread *td = curthread;
1767 wait_block *whead, *w;
1768 wait_block _wb_array[MAX_WAIT_OBJECTS];
1769 nt_dispatch_header *cur;
1770 struct timeval tv;
1771 int i, wcnt = 0, error = 0;
1772 uint64_t curtime;
1773 struct timespec t1, t2;
1774 uint32_t status = STATUS_SUCCESS;
1775 wb_ext we;
1776
1777 if (cnt > MAX_WAIT_OBJECTS)
1778 return (STATUS_INVALID_PARAMETER);
1779 if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
1780 return (STATUS_INVALID_PARAMETER);
1781
1782 mtx_lock(&ntoskrnl_dispatchlock);
1783
1784 cv_init(&we.we_cv, "KeWFM");
1785 we.we_td = td;
1786
1787 if (wb_array == NULL)
1788 whead = _wb_array;
1789 else
1790 whead = wb_array;
1791
1792 bzero((char *)whead, sizeof(wait_block) * cnt);
1793
1794 /* First pass: see if we can satisfy any waits immediately. */
1795
1796 wcnt = 0;
1797 w = whead;
1798
1799 for (i = 0; i < cnt; i++) {
1800 InsertTailList((&obj[i]->dh_waitlisthead),
1801 (&w->wb_waitlist));
1802 w->wb_ext = &we;
1803 w->wb_object = obj[i];
1804 w->wb_waittype = wtype;
1805 w->wb_waitkey = i;
1806 w->wb_awakened = FALSE;
1807 w->wb_oldpri = td->td_priority;
1808 w->wb_next = w + 1;
1809 w++;
1810 wcnt++;
1811 if (ntoskrnl_is_signalled(obj[i], td)) {
1812 /*
1813 * There's a limit to how many times
1814 * we can recursively acquire a mutant.
1815 * If we hit the limit, something
1816 * is very wrong.
1817 */
1818 if (obj[i]->dh_sigstate == INT32_MIN &&
1819 obj[i]->dh_type == DISP_TYPE_MUTANT) {
1820 mtx_unlock(&ntoskrnl_dispatchlock);
1821 panic("mutant limit exceeded");
1822 }
1823
1824 /*
1825 * If this is a WAITTYPE_ANY wait, then
1826 * satisfy the waited object and exit
1827 * right now.
1828 */
1829
1830 if (wtype == WAITTYPE_ANY) {
1831 ntoskrnl_satisfy_wait(obj[i], td);
1832 status = STATUS_WAIT_0 + i;
1833 goto wait_done;
1834 } else {
1835 w--;
1836 wcnt--;
1837 w->wb_object = NULL;
1838 RemoveEntryList(&w->wb_waitlist);
1839 }
1840 }
1841 }
1842
1843 /*
1844 * If this is a WAITTYPE_ALL wait and all objects are
1845 * already signalled, satisfy the waits and exit now.
1846 */
1847
1848 if (wtype == WAITTYPE_ALL && wcnt == 0) {
1849 for (i = 0; i < cnt; i++)
1850 ntoskrnl_satisfy_wait(obj[i], td);
1851 status = STATUS_SUCCESS;
1852 goto wait_done;
1853 }
1854
1855 /*
1856 * Create a circular waitblock list. The waitcount
1857 * must always be non-zero when we get here.
1858 */
1859
1860 (w - 1)->wb_next = whead;
1861
1862 /* Wait on any objects that aren't yet signalled. */
1863
1864 /* Calculate timeout, if any. */
1865
1866 if (duetime != NULL) {
1867 if (*duetime < 0) {
1868 tv.tv_sec = - (*duetime) / 10000000;
1869 tv.tv_usec = (- (*duetime) / 10) -
1870 (tv.tv_sec * 1000000);
1871 } else {
1872 ntoskrnl_time(&curtime);
1873 if (*duetime < curtime)
1874 tv.tv_sec = tv.tv_usec = 0;
1875 else {
1876 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1877 tv.tv_usec = ((*duetime) - curtime) / 10 -
1878 (tv.tv_sec * 1000000);
1879 }
1880 }
1881 }
1882
1883 while (wcnt) {
1884 nanotime(&t1);
1885
1886 if (duetime == NULL)
1887 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1888 else
1889 error = cv_timedwait(&we.we_cv,
1890 &ntoskrnl_dispatchlock, tvtohz(&tv));
1891
1892 /* Wait with timeout expired. */
1893
1894 if (error) {
1895 status = STATUS_TIMEOUT;
1896 goto wait_done;
1897 }
1898
1899 nanotime(&t2);
1900
1901 /* See what's been signalled. */
1902
1903 w = whead;
1904 do {
1905 cur = w->wb_object;
1906 if (ntoskrnl_is_signalled(cur, td) == TRUE ||
1907 w->wb_awakened == TRUE) {
1908 /* Sanity check the signal state value. */
1909 if (cur->dh_sigstate == INT32_MIN &&
1910 cur->dh_type == DISP_TYPE_MUTANT) {
1911 mtx_unlock(&ntoskrnl_dispatchlock);
1912 panic("mutant limit exceeded");
1913 }
1914 wcnt--;
1915 if (wtype == WAITTYPE_ANY) {
1916 status = w->wb_waitkey &
1917 STATUS_WAIT_0;
1918 goto wait_done;
1919 }
1920 }
1921 w = w->wb_next;
1922 } while (w != whead);
1923
1924 /*
1925 * If all objects have been signalled, or if this
1926 * is a WAITTYPE_ANY wait and we were woke up by
1927 * someone, we can bail.
1928 */
1929
1930 if (wcnt == 0) {
1931 status = STATUS_SUCCESS;
1932 goto wait_done;
1933 }
1934
1935 /*
1936 * If this is WAITTYPE_ALL wait, and there's still
1937 * objects that haven't been signalled, deduct the
1938 * time that's elapsed so far from the timeout and
1939 * wait again (or continue waiting indefinitely if
1940 * there's no timeout).
1941 */
1942
1943 if (duetime != NULL) {
1944 tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
1945 tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
1946 }
1947 }
1948
1949 wait_done:
1950
1951 cv_destroy(&we.we_cv);
1952
1953 for (i = 0; i < cnt; i++) {
1954 if (whead[i].wb_object != NULL)
1955 RemoveEntryList(&whead[i].wb_waitlist);
1956 }
1957 mtx_unlock(&ntoskrnl_dispatchlock);
1958
1959 return (status);
1960 }
1961
1962 static void
1963 WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val)
1964 {
1965 bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1966 }
1967
1968 static uint16_t
1969 READ_REGISTER_USHORT(reg)
1970 uint16_t *reg;
1971 {
1972 return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1973 }
1974
1975 static void
1976 WRITE_REGISTER_ULONG(reg, val)
1977 uint32_t *reg;
1978 uint32_t val;
1979 {
1980 bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1981 }
1982
1983 static uint32_t
1984 READ_REGISTER_ULONG(reg)
1985 uint32_t *reg;
1986 {
1987 return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1988 }
1989
1990 static uint8_t
1991 READ_REGISTER_UCHAR(uint8_t *reg)
1992 {
1993 return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1994 }
1995
1996 static void
1997 WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val)
1998 {
1999 bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
2000 }
2001
2002 static int64_t
2003 _allmul(a, b)
2004 int64_t a;
2005 int64_t b;
2006 {
2007 return (a * b);
2008 }
2009
2010 static int64_t
2011 _alldiv(a, b)
2012 int64_t a;
2013 int64_t b;
2014 {
2015 return (a / b);
2016 }
2017
2018 static int64_t
2019 _allrem(a, b)
2020 int64_t a;
2021 int64_t b;
2022 {
2023 return (a % b);
2024 }
2025
2026 static uint64_t
2027 _aullmul(a, b)
2028 uint64_t a;
2029 uint64_t b;
2030 {
2031 return (a * b);
2032 }
2033
2034 static uint64_t
2035 _aulldiv(a, b)
2036 uint64_t a;
2037 uint64_t b;
2038 {
2039 return (a / b);
2040 }
2041
2042 static uint64_t
2043 _aullrem(a, b)
2044 uint64_t a;
2045 uint64_t b;
2046 {
2047 return (a % b);
2048 }
2049
2050 static int64_t
2051 _allshl(int64_t a, uint8_t b)
2052 {
2053 return (a << b);
2054 }
2055
2056 static uint64_t
2057 _aullshl(uint64_t a, uint8_t b)
2058 {
2059 return (a << b);
2060 }
2061
2062 static int64_t
2063 _allshr(int64_t a, uint8_t b)
2064 {
2065 return (a >> b);
2066 }
2067
2068 static uint64_t
2069 _aullshr(uint64_t a, uint8_t b)
2070 {
2071 return (a >> b);
2072 }
2073
2074 static slist_entry *
2075 ntoskrnl_pushsl(head, entry)
2076 slist_header *head;
2077 slist_entry *entry;
2078 {
2079 slist_entry *oldhead;
2080
2081 oldhead = head->slh_list.slh_next;
2082 entry->sl_next = head->slh_list.slh_next;
2083 head->slh_list.slh_next = entry;
2084 head->slh_list.slh_depth++;
2085 head->slh_list.slh_seq++;
2086
2087 return (oldhead);
2088 }
2089
2090 static void
2091 InitializeSListHead(head)
2092 slist_header *head;
2093 {
2094 memset(head, 0, sizeof(*head));
2095 }
2096
2097 static slist_entry *
2098 ntoskrnl_popsl(head)
2099 slist_header *head;
2100 {
2101 slist_entry *first;
2102
2103 first = head->slh_list.slh_next;
2104 if (first != NULL) {
2105 head->slh_list.slh_next = first->sl_next;
2106 head->slh_list.slh_depth--;
2107 head->slh_list.slh_seq++;
2108 }
2109
2110 return (first);
2111 }
2112
2113 /*
2114 * We need this to make lookaside lists work for amd64.
2115 * We pass a pointer to ExAllocatePoolWithTag() the lookaside
2116 * list structure. For amd64 to work right, this has to be a
2117 * pointer to the wrapped version of the routine, not the
2118 * original. Letting the Windows driver invoke the original
2119 * function directly will result in a convention calling
2120 * mismatch and a pretty crash. On x86, this effectively
2121 * becomes a no-op since ipt_func and ipt_wrap are the same.
2122 */
2123
2124 static funcptr
2125 ntoskrnl_findwrap(func)
2126 funcptr func;
2127 {
2128 image_patch_table *patch;
2129
2130 patch = ntoskrnl_functbl;
2131 while (patch->ipt_func != NULL) {
2132 if ((funcptr)patch->ipt_func == func)
2133 return ((funcptr)patch->ipt_wrap);
2134 patch++;
2135 }
2136
2137 return (NULL);
2138 }
2139
2140 static void
2141 ExInitializePagedLookasideList(paged_lookaside_list *lookaside,
2142 lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2143 uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2144 {
2145 bzero((char *)lookaside, sizeof(paged_lookaside_list));
2146
2147 if (size < sizeof(slist_entry))
2148 lookaside->nll_l.gl_size = sizeof(slist_entry);
2149 else
2150 lookaside->nll_l.gl_size = size;
2151 lookaside->nll_l.gl_tag = tag;
2152 if (allocfunc == NULL)
2153 lookaside->nll_l.gl_allocfunc =
2154 ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2155 else
2156 lookaside->nll_l.gl_allocfunc = allocfunc;
2157
2158 if (freefunc == NULL)
2159 lookaside->nll_l.gl_freefunc =
2160 ntoskrnl_findwrap((funcptr)ExFreePool);
2161 else
2162 lookaside->nll_l.gl_freefunc = freefunc;
2163
2164 #ifdef __i386__
2165 KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2166 #endif
2167
2168 lookaside->nll_l.gl_type = NonPagedPool;
2169 lookaside->nll_l.gl_depth = depth;
2170 lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2171 }
2172
2173 static void
2174 ExDeletePagedLookasideList(lookaside)
2175 paged_lookaside_list *lookaside;
2176 {
2177 void *buf;
2178 void (*freefunc)(void *);
2179
2180 freefunc = lookaside->nll_l.gl_freefunc;
2181 while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2182 MSCALL1(freefunc, buf);
2183 }
2184
2185 static void
2186 ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside,
2187 lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2188 uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2189 {
2190 bzero((char *)lookaside, sizeof(npaged_lookaside_list));
2191
2192 if (size < sizeof(slist_entry))
2193 lookaside->nll_l.gl_size = sizeof(slist_entry);
2194 else
2195 lookaside->nll_l.gl_size = size;
2196 lookaside->nll_l.gl_tag = tag;
2197 if (allocfunc == NULL)
2198 lookaside->nll_l.gl_allocfunc =
2199 ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2200 else
2201 lookaside->nll_l.gl_allocfunc = allocfunc;
2202
2203 if (freefunc == NULL)
2204 lookaside->nll_l.gl_freefunc =
2205 ntoskrnl_findwrap((funcptr)ExFreePool);
2206 else
2207 lookaside->nll_l.gl_freefunc = freefunc;
2208
2209 #ifdef __i386__
2210 KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2211 #endif
2212
2213 lookaside->nll_l.gl_type = NonPagedPool;
2214 lookaside->nll_l.gl_depth = depth;
2215 lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2216 }
2217
2218 static void
2219 ExDeleteNPagedLookasideList(lookaside)
2220 npaged_lookaside_list *lookaside;
2221 {
2222 void *buf;
2223 void (*freefunc)(void *);
2224
2225 freefunc = lookaside->nll_l.gl_freefunc;
2226 while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2227 MSCALL1(freefunc, buf);
2228 }
2229
2230 slist_entry *
2231 InterlockedPushEntrySList(head, entry)
2232 slist_header *head;
2233 slist_entry *entry;
2234 {
2235 slist_entry *oldhead;
2236
2237 mtx_lock_spin(&ntoskrnl_interlock);
2238 oldhead = ntoskrnl_pushsl(head, entry);
2239 mtx_unlock_spin(&ntoskrnl_interlock);
2240
2241 return (oldhead);
2242 }
2243
2244 slist_entry *
2245 InterlockedPopEntrySList(head)
2246 slist_header *head;
2247 {
2248 slist_entry *first;
2249
2250 mtx_lock_spin(&ntoskrnl_interlock);
2251 first = ntoskrnl_popsl(head);
2252 mtx_unlock_spin(&ntoskrnl_interlock);
2253
2254 return (first);
2255 }
2256
2257 static slist_entry *
2258 ExInterlockedPushEntrySList(head, entry, lock)
2259 slist_header *head;
2260 slist_entry *entry;
2261 kspin_lock *lock;
2262 {
2263 return (InterlockedPushEntrySList(head, entry));
2264 }
2265
2266 static slist_entry *
2267 ExInterlockedPopEntrySList(head, lock)
2268 slist_header *head;
2269 kspin_lock *lock;
2270 {
2271 return (InterlockedPopEntrySList(head));
2272 }
2273
2274 uint16_t
2275 ExQueryDepthSList(head)
2276 slist_header *head;
2277 {
2278 uint16_t depth;
2279
2280 mtx_lock_spin(&ntoskrnl_interlock);
2281 depth = head->slh_list.slh_depth;
2282 mtx_unlock_spin(&ntoskrnl_interlock);
2283
2284 return (depth);
2285 }
2286
2287 void
2288 KeInitializeSpinLock(lock)
2289 kspin_lock *lock;
2290 {
2291 *lock = 0;
2292 }
2293
2294 #ifdef __i386__
2295 void
2296 KefAcquireSpinLockAtDpcLevel(lock)
2297 kspin_lock *lock;
2298 {
2299 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2300 int i = 0;
2301 #endif
2302
2303 while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) {
2304 /* sit and spin */;
2305 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2306 i++;
2307 if (i > 200000000)
2308 panic("DEADLOCK!");
2309 #endif
2310 }
2311 }
2312
2313 void
2314 KefReleaseSpinLockFromDpcLevel(lock)
2315 kspin_lock *lock;
2316 {
2317 atomic_store_rel_int((volatile u_int *)lock, 0);
2318 }
2319
2320 uint8_t
2321 KeAcquireSpinLockRaiseToDpc(kspin_lock *lock)
2322 {
2323 uint8_t oldirql;
2324
2325 if (KeGetCurrentIrql() > DISPATCH_LEVEL)
2326 panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
2327
2328 KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
2329 KeAcquireSpinLockAtDpcLevel(lock);
2330
2331 return (oldirql);
2332 }
2333 #else
2334 void
2335 KeAcquireSpinLockAtDpcLevel(kspin_lock *lock)
2336 {
2337 while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0)
2338 /* sit and spin */;
2339 }
2340
2341 void
2342 KeReleaseSpinLockFromDpcLevel(kspin_lock *lock)
2343 {
2344 atomic_store_rel_int((volatile u_int *)lock, 0);
2345 }
2346 #endif /* __i386__ */
2347
2348 uintptr_t
2349 InterlockedExchange(dst, val)
2350 volatile uint32_t *dst;
2351 uintptr_t val;
2352 {
2353 uintptr_t r;
2354
2355 mtx_lock_spin(&ntoskrnl_interlock);
2356 r = *dst;
2357 *dst = val;
2358 mtx_unlock_spin(&ntoskrnl_interlock);
2359
2360 return (r);
2361 }
2362
2363 static uint32_t
2364 InterlockedIncrement(addend)
2365 volatile uint32_t *addend;
2366 {
2367 atomic_add_long((volatile u_long *)addend, 1);
2368 return (*addend);
2369 }
2370
2371 static uint32_t
2372 InterlockedDecrement(addend)
2373 volatile uint32_t *addend;
2374 {
2375 atomic_subtract_long((volatile u_long *)addend, 1);
2376 return (*addend);
2377 }
2378
2379 static void
2380 ExInterlockedAddLargeStatistic(addend, inc)
2381 uint64_t *addend;
2382 uint32_t inc;
2383 {
2384 mtx_lock_spin(&ntoskrnl_interlock);
2385 *addend += inc;
2386 mtx_unlock_spin(&ntoskrnl_interlock);
2387 };
2388
2389 mdl *
2390 IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf,
2391 uint8_t chargequota, irp *iopkt)
2392 {
2393 mdl *m;
2394 int zone = 0;
2395
2396 if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE)
2397 m = ExAllocatePoolWithTag(NonPagedPool,
2398 MmSizeOfMdl(vaddr, len), 0);
2399 else {
2400 m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO);
2401 zone++;
2402 }
2403
2404 if (m == NULL)
2405 return (NULL);
2406
2407 MmInitializeMdl(m, vaddr, len);
2408
2409 /*
2410 * MmInitializMdl() clears the flags field, so we
2411 * have to set this here. If the MDL came from the
2412 * MDL UMA zone, tag it so we can release it to
2413 * the right place later.
2414 */
2415 if (zone)
2416 m->mdl_flags = MDL_ZONE_ALLOCED;
2417
2418 if (iopkt != NULL) {
2419 if (secondarybuf == TRUE) {
2420 mdl *last;
2421 last = iopkt->irp_mdl;
2422 while (last->mdl_next != NULL)
2423 last = last->mdl_next;
2424 last->mdl_next = m;
2425 } else {
2426 if (iopkt->irp_mdl != NULL)
2427 panic("leaking an MDL in IoAllocateMdl()");
2428 iopkt->irp_mdl = m;
2429 }
2430 }
2431
2432 return (m);
2433 }
2434
2435 void
2436 IoFreeMdl(m)
2437 mdl *m;
2438 {
2439 if (m == NULL)
2440 return;
2441
2442 if (m->mdl_flags & MDL_ZONE_ALLOCED)
2443 uma_zfree(mdl_zone, m);
2444 else
2445 ExFreePool(m);
2446 }
2447
2448 static void *
2449 MmAllocateContiguousMemory(size, highest)
2450 uint32_t size;
2451 uint64_t highest;
2452 {
2453 void *addr;
2454 size_t pagelength = roundup(size, PAGE_SIZE);
2455
2456 addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0);
2457
2458 return (addr);
2459 }
2460
2461 static void *
2462 MmAllocateContiguousMemorySpecifyCache(size, lowest, highest,
2463 boundary, cachetype)
2464 uint32_t size;
2465 uint64_t lowest;
2466 uint64_t highest;
2467 uint64_t boundary;
2468 enum nt_caching_type cachetype;
2469 {
2470 vm_memattr_t memattr;
2471 void *ret;
2472
2473 switch (cachetype) {
2474 case MmNonCached:
2475 memattr = VM_MEMATTR_UNCACHEABLE;
2476 break;
2477 case MmWriteCombined:
2478 memattr = VM_MEMATTR_WRITE_COMBINING;
2479 break;
2480 case MmNonCachedUnordered:
2481 memattr = VM_MEMATTR_UNCACHEABLE;
2482 break;
2483 case MmCached:
2484 case MmHardwareCoherentCached:
2485 case MmUSWCCached:
2486 default:
2487 memattr = VM_MEMATTR_DEFAULT;
2488 break;
2489 }
2490
2491 ret = (void *)kmem_alloc_contig(size, M_ZERO | M_NOWAIT, lowest,
2492 highest, PAGE_SIZE, boundary, memattr);
2493 if (ret != NULL)
2494 malloc_type_allocated(M_DEVBUF, round_page(size));
2495 return (ret);
2496 }
2497
2498 static void
2499 MmFreeContiguousMemory(base)
2500 void *base;
2501 {
2502 ExFreePool(base);
2503 }
2504
2505 static void
2506 MmFreeContiguousMemorySpecifyCache(base, size, cachetype)
2507 void *base;
2508 uint32_t size;
2509 enum nt_caching_type cachetype;
2510 {
2511 contigfree(base, size, M_DEVBUF);
2512 }
2513
2514 static uint32_t
2515 MmSizeOfMdl(vaddr, len)
2516 void *vaddr;
2517 size_t len;
2518 {
2519 uint32_t l;
2520
2521 l = sizeof(struct mdl) +
2522 (sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len));
2523
2524 return (l);
2525 }
2526
2527 /*
2528 * The Microsoft documentation says this routine fills in the
2529 * page array of an MDL with the _physical_ page addresses that
2530 * comprise the buffer, but we don't really want to do that here.
2531 * Instead, we just fill in the page array with the kernel virtual
2532 * addresses of the buffers.
2533 */
2534 void
2535 MmBuildMdlForNonPagedPool(m)
2536 mdl *m;
2537 {
2538 vm_offset_t *mdl_pages;
2539 int pagecnt, i;
2540
2541 pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount);
2542
2543 if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *))
2544 panic("not enough pages in MDL to describe buffer");
2545
2546 mdl_pages = MmGetMdlPfnArray(m);
2547
2548 for (i = 0; i < pagecnt; i++)
2549 *mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE);
2550
2551 m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL;
2552 m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m);
2553 }
2554
2555 static void *
2556 MmMapLockedPages(mdl *buf, uint8_t accessmode)
2557 {
2558 buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA;
2559 return (MmGetMdlVirtualAddress(buf));
2560 }
2561
2562 static void *
2563 MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype,
2564 void *vaddr, uint32_t bugcheck, uint32_t prio)
2565 {
2566 return (MmMapLockedPages(buf, accessmode));
2567 }
2568
2569 static void
2570 MmUnmapLockedPages(vaddr, buf)
2571 void *vaddr;
2572 mdl *buf;
2573 {
2574 buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA;
2575 }
2576
2577 /*
2578 * This function has a problem in that it will break if you
2579 * compile this module without PAE and try to use it on a PAE
2580 * kernel. Unfortunately, there's no way around this at the
2581 * moment. It's slightly less broken that using pmap_kextract().
2582 * You'd think the virtual memory subsystem would help us out
2583 * here, but it doesn't.
2584 */
2585
2586 static uint64_t
2587 MmGetPhysicalAddress(void *base)
2588 {
2589 return (pmap_extract(kernel_map->pmap, (vm_offset_t)base));
2590 }
2591
2592 void *
2593 MmGetSystemRoutineAddress(ustr)
2594 unicode_string *ustr;
2595 {
2596 ansi_string astr;
2597
2598 if (RtlUnicodeStringToAnsiString(&astr, ustr, TRUE))
2599 return (NULL);
2600 return (ndis_get_routine_address(ntoskrnl_functbl, astr.as_buf));
2601 }
2602
2603 uint8_t
2604 MmIsAddressValid(vaddr)
2605 void *vaddr;
2606 {
2607 if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr))
2608 return (TRUE);
2609
2610 return (FALSE);
2611 }
2612
2613 void *
2614 MmMapIoSpace(paddr, len, cachetype)
2615 uint64_t paddr;
2616 uint32_t len;
2617 uint32_t cachetype;
2618 {
2619 devclass_t nexus_class;
2620 device_t *nexus_devs, devp;
2621 int nexus_count = 0;
2622 device_t matching_dev = NULL;
2623 struct resource *res;
2624 int i;
2625 vm_offset_t v;
2626
2627 /* There will always be at least one nexus. */
2628
2629 nexus_class = devclass_find("nexus");
2630 devclass_get_devices(nexus_class, &nexus_devs, &nexus_count);
2631
2632 for (i = 0; i < nexus_count; i++) {
2633 devp = nexus_devs[i];
2634 matching_dev = ntoskrnl_finddev(devp, paddr, &res);
2635 if (matching_dev)
2636 break;
2637 }
2638
2639 free(nexus_devs, M_TEMP);
2640
2641 if (matching_dev == NULL)
2642 return (NULL);
2643
2644 v = (vm_offset_t)rman_get_virtual(res);
2645 if (paddr > rman_get_start(res))
2646 v += paddr - rman_get_start(res);
2647
2648 return ((void *)v);
2649 }
2650
2651 void
2652 MmUnmapIoSpace(vaddr, len)
2653 void *vaddr;
2654 size_t len;
2655 {
2656 }
2657
2658 static device_t
2659 ntoskrnl_finddev(dev, paddr, res)
2660 device_t dev;
2661 uint64_t paddr;
2662 struct resource **res;
2663 {
2664 device_t *children = NULL;
2665 device_t matching_dev;
2666 int childcnt;
2667 struct resource *r;
2668 struct resource_list *rl;
2669 struct resource_list_entry *rle;
2670 uint32_t flags;
2671 int i;
2672
2673 /* We only want devices that have been successfully probed. */
2674
2675 if (device_is_alive(dev) == FALSE)
2676 return (NULL);
2677
2678 rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
2679 if (rl != NULL) {
2680 STAILQ_FOREACH(rle, rl, link) {
2681 r = rle->res;
2682
2683 if (r == NULL)
2684 continue;
2685
2686 flags = rman_get_flags(r);
2687
2688 if (rle->type == SYS_RES_MEMORY &&
2689 paddr >= rman_get_start(r) &&
2690 paddr <= rman_get_end(r)) {
2691 if (!(flags & RF_ACTIVE))
2692 bus_activate_resource(dev,
2693 SYS_RES_MEMORY, 0, r);
2694 *res = r;
2695 return (dev);
2696 }
2697 }
2698 }
2699
2700 /*
2701 * If this device has children, do another
2702 * level of recursion to inspect them.
2703 */
2704
2705 device_get_children(dev, &children, &childcnt);
2706
2707 for (i = 0; i < childcnt; i++) {
2708 matching_dev = ntoskrnl_finddev(children[i], paddr, res);
2709 if (matching_dev != NULL) {
2710 free(children, M_TEMP);
2711 return (matching_dev);
2712 }
2713 }
2714
2715 /* Won't somebody please think of the children! */
2716
2717 if (children != NULL)
2718 free(children, M_TEMP);
2719
2720 return (NULL);
2721 }
2722
2723 /*
2724 * Workitems are unlike DPCs, in that they run in a user-mode thread
2725 * context rather than at DISPATCH_LEVEL in kernel context. In our
2726 * case we run them in kernel context anyway.
2727 */
2728 static void
2729 ntoskrnl_workitem_thread(arg)
2730 void *arg;
2731 {
2732 kdpc_queue *kq;
2733 list_entry *l;
2734 io_workitem *iw;
2735 uint8_t irql;
2736
2737 kq = arg;
2738
2739 InitializeListHead(&kq->kq_disp);
2740 kq->kq_td = curthread;
2741 kq->kq_exit = 0;
2742 KeInitializeSpinLock(&kq->kq_lock);
2743 KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
2744
2745 while (1) {
2746 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
2747
2748 KeAcquireSpinLock(&kq->kq_lock, &irql);
2749
2750 if (kq->kq_exit) {
2751 kq->kq_exit = 0;
2752 KeReleaseSpinLock(&kq->kq_lock, irql);
2753 break;
2754 }
2755
2756 while (!IsListEmpty(&kq->kq_disp)) {
2757 l = RemoveHeadList(&kq->kq_disp);
2758 iw = CONTAINING_RECORD(l,
2759 io_workitem, iw_listentry);
2760 InitializeListHead((&iw->iw_listentry));
2761 if (iw->iw_func == NULL)
2762 continue;
2763 KeReleaseSpinLock(&kq->kq_lock, irql);
2764 MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx);
2765 KeAcquireSpinLock(&kq->kq_lock, &irql);
2766 }
2767
2768 KeReleaseSpinLock(&kq->kq_lock, irql);
2769 }
2770
2771 kproc_exit(0);
2772 return; /* notreached */
2773 }
2774
2775 static ndis_status
2776 RtlCharToInteger(src, base, val)
2777 const char *src;
2778 uint32_t base;
2779 uint32_t *val;
2780 {
2781 int negative = 0;
2782 uint32_t res;
2783
2784 if (!src || !val)
2785 return (STATUS_ACCESS_VIOLATION);
2786 while (*src != '\0' && *src <= ' ')
2787 src++;
2788 if (*src == '+')
2789 src++;
2790 else if (*src == '-') {
2791 src++;
2792 negative = 1;
2793 }
2794 if (base == 0) {
2795 base = 10;
2796 if (*src == '0') {
2797 src++;
2798 if (*src == 'b') {
2799 base = 2;
2800 src++;
2801 } else if (*src == 'o') {
2802 base = 8;
2803 src++;
2804 } else if (*src == 'x') {
2805 base = 16;
2806 src++;
2807 }
2808 }
2809 } else if (!(base == 2 || base == 8 || base == 10 || base == 16))
2810 return (STATUS_INVALID_PARAMETER);
2811
2812 for (res = 0; *src; src++) {
2813 int v;
2814 if (isdigit(*src))
2815 v = *src - '0';
2816 else if (isxdigit(*src))
2817 v = tolower(*src) - 'a' + 10;
2818 else
2819 v = base;
2820 if (v >= base)
2821 return (STATUS_INVALID_PARAMETER);
2822 res = res * base + v;
2823 }
2824 *val = negative ? -res : res;
2825 return (STATUS_SUCCESS);
2826 }
2827
2828 static void
2829 ntoskrnl_destroy_workitem_threads(void)
2830 {
2831 kdpc_queue *kq;
2832 int i;
2833
2834 for (i = 0; i < WORKITEM_THREADS; i++) {
2835 kq = wq_queues + i;
2836 kq->kq_exit = 1;
2837 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2838 while (kq->kq_exit)
2839 tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10);
2840 }
2841 }
2842
2843 io_workitem *
2844 IoAllocateWorkItem(dobj)
2845 device_object *dobj;
2846 {
2847 io_workitem *iw;
2848
2849 iw = uma_zalloc(iw_zone, M_NOWAIT);
2850 if (iw == NULL)
2851 return (NULL);
2852
2853 InitializeListHead(&iw->iw_listentry);
2854 iw->iw_dobj = dobj;
2855
2856 mtx_lock(&ntoskrnl_dispatchlock);
2857 iw->iw_idx = wq_idx;
2858 WORKIDX_INC(wq_idx);
2859 mtx_unlock(&ntoskrnl_dispatchlock);
2860
2861 return (iw);
2862 }
2863
2864 void
2865 IoFreeWorkItem(iw)
2866 io_workitem *iw;
2867 {
2868 uma_zfree(iw_zone, iw);
2869 }
2870
2871 void
2872 IoQueueWorkItem(iw, iw_func, qtype, ctx)
2873 io_workitem *iw;
2874 io_workitem_func iw_func;
2875 uint32_t qtype;
2876 void *ctx;
2877 {
2878 kdpc_queue *kq;
2879 list_entry *l;
2880 io_workitem *cur;
2881 uint8_t irql;
2882
2883 kq = wq_queues + iw->iw_idx;
2884
2885 KeAcquireSpinLock(&kq->kq_lock, &irql);
2886
2887 /*
2888 * Traverse the list and make sure this workitem hasn't
2889 * already been inserted. Queuing the same workitem
2890 * twice will hose the list but good.
2891 */
2892
2893 l = kq->kq_disp.nle_flink;
2894 while (l != &kq->kq_disp) {
2895 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2896 if (cur == iw) {
2897 /* Already queued -- do nothing. */
2898 KeReleaseSpinLock(&kq->kq_lock, irql);
2899 return;
2900 }
2901 l = l->nle_flink;
2902 }
2903
2904 iw->iw_func = iw_func;
2905 iw->iw_ctx = ctx;
2906
2907 InsertTailList((&kq->kq_disp), (&iw->iw_listentry));
2908 KeReleaseSpinLock(&kq->kq_lock, irql);
2909
2910 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2911 }
2912
2913 static void
2914 ntoskrnl_workitem(dobj, arg)
2915 device_object *dobj;
2916 void *arg;
2917 {
2918 io_workitem *iw;
2919 work_queue_item *w;
2920 work_item_func f;
2921
2922 iw = arg;
2923 w = (work_queue_item *)dobj;
2924 f = (work_item_func)w->wqi_func;
2925 uma_zfree(iw_zone, iw);
2926 MSCALL2(f, w, w->wqi_ctx);
2927 }
2928
2929 /*
2930 * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft
2931 * warns that it's unsafe and to use IoQueueWorkItem() instead. The
2932 * problem with ExQueueWorkItem() is that it can't guard against
2933 * the condition where a driver submits a job to the work queue and
2934 * is then unloaded before the job is able to run. IoQueueWorkItem()
2935 * acquires a reference to the device's device_object via the
2936 * object manager and retains it until after the job has completed,
2937 * which prevents the driver from being unloaded before the job
2938 * runs. (We don't currently support this behavior, though hopefully
2939 * that will change once the object manager API is fleshed out a bit.)
2940 *
2941 * Having said all that, the ExQueueWorkItem() API remains, because
2942 * there are still other parts of Windows that use it, including
2943 * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem().
2944 * We fake up the ExQueueWorkItem() API on top of our implementation
2945 * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively
2946 * for ExQueueWorkItem() jobs, and we pass a pointer to the work
2947 * queue item (provided by the caller) in to IoAllocateWorkItem()
2948 * instead of the device_object. We need to save this pointer so
2949 * we can apply a sanity check: as with the DPC queue and other
2950 * workitem queues, we can't allow the same work queue item to
2951 * be queued twice. If it's already pending, we silently return
2952 */
2953
2954 void
2955 ExQueueWorkItem(w, qtype)
2956 work_queue_item *w;
2957 uint32_t qtype;
2958 {
2959 io_workitem *iw;
2960 io_workitem_func iwf;
2961 kdpc_queue *kq;
2962 list_entry *l;
2963 io_workitem *cur;
2964 uint8_t irql;
2965
2966 /*
2967 * We need to do a special sanity test to make sure
2968 * the ExQueueWorkItem() API isn't used to queue
2969 * the same workitem twice. Rather than checking the
2970 * io_workitem pointer itself, we test the attached
2971 * device object, which is really a pointer to the
2972 * legacy work queue item structure.
2973 */
2974
2975 kq = wq_queues + WORKITEM_LEGACY_THREAD;
2976 KeAcquireSpinLock(&kq->kq_lock, &irql);
2977 l = kq->kq_disp.nle_flink;
2978 while (l != &kq->kq_disp) {
2979 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2980 if (cur->iw_dobj == (device_object *)w) {
2981 /* Already queued -- do nothing. */
2982 KeReleaseSpinLock(&kq->kq_lock, irql);
2983 return;
2984 }
2985 l = l->nle_flink;
2986 }
2987 KeReleaseSpinLock(&kq->kq_lock, irql);
2988
2989 iw = IoAllocateWorkItem((device_object *)w);
2990 if (iw == NULL)
2991 return;
2992
2993 iw->iw_idx = WORKITEM_LEGACY_THREAD;
2994 iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem);
2995 IoQueueWorkItem(iw, iwf, qtype, iw);
2996 }
2997
2998 static void
2999 RtlZeroMemory(dst, len)
3000 void *dst;
3001 size_t len;
3002 {
3003 bzero(dst, len);
3004 }
3005
3006 static void
3007 RtlSecureZeroMemory(dst, len)
3008 void *dst;
3009 size_t len;
3010 {
3011 memset(dst, 0, len);
3012 }
3013
3014 static void
3015 RtlFillMemory(void *dst, size_t len, uint8_t c)
3016 {
3017 memset(dst, c, len);
3018 }
3019
3020 static void
3021 RtlMoveMemory(dst, src, len)
3022 void *dst;
3023 const void *src;
3024 size_t len;
3025 {
3026 memmove(dst, src, len);
3027 }
3028
3029 static void
3030 RtlCopyMemory(dst, src, len)
3031 void *dst;
3032 const void *src;
3033 size_t len;
3034 {
3035 bcopy(src, dst, len);
3036 }
3037
3038 static size_t
3039 RtlCompareMemory(s1, s2, len)
3040 const void *s1;
3041 const void *s2;
3042 size_t len;
3043 {
3044 size_t i;
3045 uint8_t *m1, *m2;
3046
3047 m1 = __DECONST(char *, s1);
3048 m2 = __DECONST(char *, s2);
3049
3050 for (i = 0; i < len && m1[i] == m2[i]; i++);
3051 return (i);
3052 }
3053
3054 void
3055 RtlInitAnsiString(dst, src)
3056 ansi_string *dst;
3057 char *src;
3058 {
3059 ansi_string *a;
3060
3061 a = dst;
3062 if (a == NULL)
3063 return;
3064 if (src == NULL) {
3065 a->as_len = a->as_maxlen = 0;
3066 a->as_buf = NULL;
3067 } else {
3068 a->as_buf = src;
3069 a->as_len = a->as_maxlen = strlen(src);
3070 }
3071 }
3072
3073 void
3074 RtlInitUnicodeString(dst, src)
3075 unicode_string *dst;
3076 uint16_t *src;
3077 {
3078 unicode_string *u;
3079 int i;
3080
3081 u = dst;
3082 if (u == NULL)
3083 return;
3084 if (src == NULL) {
3085 u->us_len = u->us_maxlen = 0;
3086 u->us_buf = NULL;
3087 } else {
3088 i = 0;
3089 while(src[i] != 0)
3090 i++;
3091 u->us_buf = src;
3092 u->us_len = u->us_maxlen = i * 2;
3093 }
3094 }
3095
3096 ndis_status
3097 RtlUnicodeStringToInteger(ustr, base, val)
3098 unicode_string *ustr;
3099 uint32_t base;
3100 uint32_t *val;
3101 {
3102 uint16_t *uchr;
3103 int len, neg = 0;
3104 char abuf[64];
3105 char *astr;
3106
3107 uchr = ustr->us_buf;
3108 len = ustr->us_len;
3109 bzero(abuf, sizeof(abuf));
3110
3111 if ((char)((*uchr) & 0xFF) == '-') {
3112 neg = 1;
3113 uchr++;
3114 len -= 2;
3115 } else if ((char)((*uchr) & 0xFF) == '+') {
3116 neg = 0;
3117 uchr++;
3118 len -= 2;
3119 }
3120
3121 if (base == 0) {
3122 if ((char)((*uchr) & 0xFF) == 'b') {
3123 base = 2;
3124 uchr++;
3125 len -= 2;
3126 } else if ((char)((*uchr) & 0xFF) == 'o') {
3127 base = 8;
3128 uchr++;
3129 len -= 2;
3130 } else if ((char)((*uchr) & 0xFF) == 'x') {
3131 base = 16;
3132 uchr++;
3133 len -= 2;
3134 } else
3135 base = 10;
3136 }
3137
3138 astr = abuf;
3139 if (neg) {
3140 strcpy(astr, "-");
3141 astr++;
3142 }
3143
3144 ntoskrnl_unicode_to_ascii(uchr, astr, len);
3145 *val = strtoul(abuf, NULL, base);
3146
3147 return (STATUS_SUCCESS);
3148 }
3149
3150 void
3151 RtlFreeUnicodeString(ustr)
3152 unicode_string *ustr;
3153 {
3154 if (ustr->us_buf == NULL)
3155 return;
3156 ExFreePool(ustr->us_buf);
3157 ustr->us_buf = NULL;
3158 }
3159
3160 void
3161 RtlFreeAnsiString(astr)
3162 ansi_string *astr;
3163 {
3164 if (astr->as_buf == NULL)
3165 return;
3166 ExFreePool(astr->as_buf);
3167 astr->as_buf = NULL;
3168 }
3169
3170 static int
3171 atoi(str)
3172 const char *str;
3173 {
3174 return (int)strtol(str, (char **)NULL, 10);
3175 }
3176
3177 static long
3178 atol(str)
3179 const char *str;
3180 {
3181 return strtol(str, (char **)NULL, 10);
3182 }
3183
3184 static int
3185 rand(void)
3186 {
3187
3188 return (random());
3189 }
3190
3191 static void
3192 srand(unsigned int seed __unused)
3193 {
3194 }
3195
3196 static uint8_t
3197 IoIsWdmVersionAvailable(uint8_t major, uint8_t minor)
3198 {
3199 if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
3200 return (TRUE);
3201 return (FALSE);
3202 }
3203
3204 static int32_t
3205 IoOpenDeviceRegistryKey(struct device_object *devobj, uint32_t type,
3206 uint32_t mask, void **key)
3207 {
3208 return (NDIS_STATUS_INVALID_DEVICE_REQUEST);
3209 }
3210
3211 static ndis_status
3212 IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj)
3213 unicode_string *name;
3214 uint32_t reqaccess;
3215 void *fileobj;
3216 device_object *devobj;
3217 {
3218 return (STATUS_SUCCESS);
3219 }
3220
3221 static ndis_status
3222 IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen)
3223 device_object *devobj;
3224 uint32_t regprop;
3225 uint32_t buflen;
3226 void *prop;
3227 uint32_t *reslen;
3228 {
3229 driver_object *drv;
3230 uint16_t **name;
3231
3232 drv = devobj->do_drvobj;
3233
3234 switch (regprop) {
3235 case DEVPROP_DRIVER_KEYNAME:
3236 name = prop;
3237 *name = drv->dro_drivername.us_buf;
3238 *reslen = drv->dro_drivername.us_len;
3239 break;
3240 default:
3241 return (STATUS_INVALID_PARAMETER_2);
3242 break;
3243 }
3244
3245 return (STATUS_SUCCESS);
3246 }
3247
3248 static void
3249 KeInitializeMutex(kmutex, level)
3250 kmutant *kmutex;
3251 uint32_t level;
3252 {
3253 InitializeListHead((&kmutex->km_header.dh_waitlisthead));
3254 kmutex->km_abandoned = FALSE;
3255 kmutex->km_apcdisable = 1;
3256 kmutex->km_header.dh_sigstate = 1;
3257 kmutex->km_header.dh_type = DISP_TYPE_MUTANT;
3258 kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t);
3259 kmutex->km_ownerthread = NULL;
3260 }
3261
3262 static uint32_t
3263 KeReleaseMutex(kmutant *kmutex, uint8_t kwait)
3264 {
3265 uint32_t prevstate;
3266
3267 mtx_lock(&ntoskrnl_dispatchlock);
3268 prevstate = kmutex->km_header.dh_sigstate;
3269 if (kmutex->km_ownerthread != curthread) {
3270 mtx_unlock(&ntoskrnl_dispatchlock);
3271 return (STATUS_MUTANT_NOT_OWNED);
3272 }
3273
3274 kmutex->km_header.dh_sigstate++;
3275 kmutex->km_abandoned = FALSE;
3276
3277 if (kmutex->km_header.dh_sigstate == 1) {
3278 kmutex->km_ownerthread = NULL;
3279 ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT);
3280 }
3281
3282 mtx_unlock(&ntoskrnl_dispatchlock);
3283
3284 return (prevstate);
3285 }
3286
3287 static uint32_t
3288 KeReadStateMutex(kmutex)
3289 kmutant *kmutex;
3290 {
3291 return (kmutex->km_header.dh_sigstate);
3292 }
3293
3294 void
3295 KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state)
3296 {
3297 InitializeListHead((&kevent->k_header.dh_waitlisthead));
3298 kevent->k_header.dh_sigstate = state;
3299 if (type == EVENT_TYPE_NOTIFY)
3300 kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT;
3301 else
3302 kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT;
3303 kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t);
3304 }
3305
3306 uint32_t
3307 KeResetEvent(kevent)
3308 nt_kevent *kevent;
3309 {
3310 uint32_t prevstate;
3311
3312 mtx_lock(&ntoskrnl_dispatchlock);
3313 prevstate = kevent->k_header.dh_sigstate;
3314 kevent->k_header.dh_sigstate = FALSE;
3315 mtx_unlock(&ntoskrnl_dispatchlock);
3316
3317 return (prevstate);
3318 }
3319
3320 uint32_t
3321 KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
3322 {
3323 uint32_t prevstate;
3324 wait_block *w;
3325 nt_dispatch_header *dh;
3326 struct thread *td;
3327 wb_ext *we;
3328
3329 mtx_lock(&ntoskrnl_dispatchlock);
3330 prevstate = kevent->k_header.dh_sigstate;
3331 dh = &kevent->k_header;
3332
3333 if (IsListEmpty(&dh->dh_waitlisthead))
3334 /*
3335 * If there's nobody in the waitlist, just set
3336 * the state to signalled.
3337 */
3338 dh->dh_sigstate = 1;
3339 else {
3340 /*
3341 * Get the first waiter. If this is a synchronization
3342 * event, just wake up that one thread (don't bother
3343 * setting the state to signalled since we're supposed
3344 * to automatically clear synchronization events anyway).
3345 *
3346 * If it's a notification event, or the first
3347 * waiter is doing a WAITTYPE_ALL wait, go through
3348 * the full wait satisfaction process.
3349 */
3350 w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink,
3351 wait_block, wb_waitlist);
3352 we = w->wb_ext;
3353 td = we->we_td;
3354 if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT ||
3355 w->wb_waittype == WAITTYPE_ALL) {
3356 if (prevstate == 0) {
3357 dh->dh_sigstate = 1;
3358 ntoskrnl_waittest(dh, increment);
3359 }
3360 } else {
3361 w->wb_awakened |= TRUE;
3362 cv_broadcastpri(&we->we_cv,
3363 (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
3364 w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
3365 }
3366 }
3367
3368 mtx_unlock(&ntoskrnl_dispatchlock);
3369
3370 return (prevstate);
3371 }
3372
3373 void
3374 KeClearEvent(kevent)
3375 nt_kevent *kevent;
3376 {
3377 kevent->k_header.dh_sigstate = FALSE;
3378 }
3379
3380 uint32_t
3381 KeReadStateEvent(kevent)
3382 nt_kevent *kevent;
3383 {
3384 return (kevent->k_header.dh_sigstate);
3385 }
3386
3387 /*
3388 * The object manager in Windows is responsible for managing
3389 * references and access to various types of objects, including
3390 * device_objects, events, threads, timers and so on. However,
3391 * there's a difference in the way objects are handled in user
3392 * mode versus kernel mode.
3393 *
3394 * In user mode (i.e. Win32 applications), all objects are
3395 * managed by the object manager. For example, when you create
3396 * a timer or event object, you actually end up with an
3397 * object_header (for the object manager's bookkeeping
3398 * purposes) and an object body (which contains the actual object
3399 * structure, e.g. ktimer, kevent, etc...). This allows Windows
3400 * to manage resource quotas and to enforce access restrictions
3401 * on basically every kind of system object handled by the kernel.
3402 *
3403 * However, in kernel mode, you only end up using the object
3404 * manager some of the time. For example, in a driver, you create
3405 * a timer object by simply allocating the memory for a ktimer
3406 * structure and initializing it with KeInitializeTimer(). Hence,
3407 * the timer has no object_header and no reference counting or
3408 * security/resource checks are done on it. The assumption in
3409 * this case is that if you're running in kernel mode, you know
3410 * what you're doing, and you're already at an elevated privilege
3411 * anyway.
3412 *
3413 * There are some exceptions to this. The two most important ones
3414 * for our purposes are device_objects and threads. We need to use
3415 * the object manager to do reference counting on device_objects,
3416 * and for threads, you can only get a pointer to a thread's
3417 * dispatch header by using ObReferenceObjectByHandle() on the
3418 * handle returned by PsCreateSystemThread().
3419 */
3420
3421 static ndis_status
3422 ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype,
3423 uint8_t accessmode, void **object, void **handleinfo)
3424 {
3425 nt_objref *nr;
3426
3427 nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO);
3428 if (nr == NULL)
3429 return (STATUS_INSUFFICIENT_RESOURCES);
3430
3431 InitializeListHead((&nr->no_dh.dh_waitlisthead));
3432 nr->no_obj = handle;
3433 nr->no_dh.dh_type = DISP_TYPE_THREAD;
3434 nr->no_dh.dh_sigstate = 0;
3435 nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) /
3436 sizeof(uint32_t));
3437 TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
3438 *object = nr;
3439
3440 return (STATUS_SUCCESS);
3441 }
3442
3443 static void
3444 ObfDereferenceObject(object)
3445 void *object;
3446 {
3447 nt_objref *nr;
3448
3449 nr = object;
3450 TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
3451 free(nr, M_DEVBUF);
3452 }
3453
3454 static uint32_t
3455 ZwClose(handle)
3456 ndis_handle handle;
3457 {
3458 return (STATUS_SUCCESS);
3459 }
3460
3461 static uint32_t
3462 WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf)
3463 uint32_t traceclass;
3464 void *traceinfo;
3465 uint32_t infolen;
3466 uint32_t reqlen;
3467 void *buf;
3468 {
3469 return (STATUS_NOT_FOUND);
3470 }
3471
3472 static uint32_t
3473 WmiTraceMessage(uint64_t loghandle, uint32_t messageflags,
3474 void *guid, uint16_t messagenum, ...)
3475 {
3476 return (STATUS_SUCCESS);
3477 }
3478
3479 static uint32_t
3480 IoWMIRegistrationControl(dobj, action)
3481 device_object *dobj;
3482 uint32_t action;
3483 {
3484 return (STATUS_SUCCESS);
3485 }
3486
3487 /*
3488 * This is here just in case the thread returns without calling
3489 * PsTerminateSystemThread().
3490 */
3491 static void
3492 ntoskrnl_thrfunc(arg)
3493 void *arg;
3494 {
3495 thread_context *thrctx;
3496 uint32_t (*tfunc)(void *);
3497 void *tctx;
3498 uint32_t rval;
3499
3500 thrctx = arg;
3501 tfunc = thrctx->tc_thrfunc;
3502 tctx = thrctx->tc_thrctx;
3503 free(thrctx, M_TEMP);
3504
3505 rval = MSCALL1(tfunc, tctx);
3506
3507 PsTerminateSystemThread(rval);
3508 return; /* notreached */
3509 }
3510
3511 static ndis_status
3512 PsCreateSystemThread(handle, reqaccess, objattrs, phandle,
3513 clientid, thrfunc, thrctx)
3514 ndis_handle *handle;
3515 uint32_t reqaccess;
3516 void *objattrs;
3517 ndis_handle phandle;
3518 void *clientid;
3519 void *thrfunc;
3520 void *thrctx;
3521 {
3522 int error;
3523 thread_context *tc;
3524 struct proc *p;
3525
3526 tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT);
3527 if (tc == NULL)
3528 return (STATUS_INSUFFICIENT_RESOURCES);
3529
3530 tc->tc_thrctx = thrctx;
3531 tc->tc_thrfunc = thrfunc;
3532
3533 error = kproc_create(ntoskrnl_thrfunc, tc, &p,
3534 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth);
3535
3536 if (error) {
3537 free(tc, M_TEMP);
3538 return (STATUS_INSUFFICIENT_RESOURCES);
3539 }
3540
3541 *handle = p;
3542 ntoskrnl_kth++;
3543
3544 return (STATUS_SUCCESS);
3545 }
3546
3547 /*
3548 * In Windows, the exit of a thread is an event that you're allowed
3549 * to wait on, assuming you've obtained a reference to the thread using
3550 * ObReferenceObjectByHandle(). Unfortunately, the only way we can
3551 * simulate this behavior is to register each thread we create in a
3552 * reference list, and if someone holds a reference to us, we poke
3553 * them.
3554 */
3555 static ndis_status
3556 PsTerminateSystemThread(status)
3557 ndis_status status;
3558 {
3559 struct nt_objref *nr;
3560
3561 mtx_lock(&ntoskrnl_dispatchlock);
3562 TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
3563 if (nr->no_obj != curthread->td_proc)
3564 continue;
3565 nr->no_dh.dh_sigstate = 1;
3566 ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT);
3567 break;
3568 }
3569 mtx_unlock(&ntoskrnl_dispatchlock);
3570
3571 ntoskrnl_kth--;
3572
3573 kproc_exit(0);
3574 return (0); /* notreached */
3575 }
3576
3577 static uint32_t
3578 DbgPrint(char *fmt, ...)
3579 {
3580 va_list ap;
3581
3582 if (bootverbose) {
3583 va_start(ap, fmt);
3584 vprintf(fmt, ap);
3585 va_end(ap);
3586 }
3587
3588 return (STATUS_SUCCESS);
3589 }
3590
3591 static void
3592 DbgBreakPoint(void)
3593 {
3594
3595 kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint");
3596 }
3597
3598 static void
3599 KeBugCheckEx(code, param1, param2, param3, param4)
3600 uint32_t code;
3601 u_long param1;
3602 u_long param2;
3603 u_long param3;
3604 u_long param4;
3605 {
3606 panic("KeBugCheckEx: STOP 0x%X", code);
3607 }
3608
3609 static void
3610 ntoskrnl_timercall(arg)
3611 void *arg;
3612 {
3613 ktimer *timer;
3614 struct timeval tv;
3615 kdpc *dpc;
3616
3617 mtx_lock(&ntoskrnl_dispatchlock);
3618
3619 timer = arg;
3620
3621 #ifdef NTOSKRNL_DEBUG_TIMERS
3622 ntoskrnl_timer_fires++;
3623 #endif
3624 ntoskrnl_remove_timer(timer);
3625
3626 /*
3627 * This should never happen, but complain
3628 * if it does.
3629 */
3630
3631 if (timer->k_header.dh_inserted == FALSE) {
3632 mtx_unlock(&ntoskrnl_dispatchlock);
3633 printf("NTOS: timer %p fired even though "
3634 "it was canceled\n", timer);
3635 return;
3636 }
3637
3638 /* Mark the timer as no longer being on the timer queue. */
3639
3640 timer->k_header.dh_inserted = FALSE;
3641
3642 /* Now signal the object and satisfy any waits on it. */
3643
3644 timer->k_header.dh_sigstate = 1;
3645 ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT);
3646
3647 /*
3648 * If this is a periodic timer, re-arm it
3649 * so it will fire again. We do this before
3650 * calling any deferred procedure calls because
3651 * it's possible the DPC might cancel the timer,
3652 * in which case it would be wrong for us to
3653 * re-arm it again afterwards.
3654 */
3655
3656 if (timer->k_period) {
3657 tv.tv_sec = 0;
3658 tv.tv_usec = timer->k_period * 1000;
3659 timer->k_header.dh_inserted = TRUE;
3660 ntoskrnl_insert_timer(timer, tvtohz(&tv));
3661 #ifdef NTOSKRNL_DEBUG_TIMERS
3662 ntoskrnl_timer_reloads++;
3663 #endif
3664 }
3665
3666 dpc = timer->k_dpc;
3667
3668 mtx_unlock(&ntoskrnl_dispatchlock);
3669
3670 /* If there's a DPC associated with the timer, queue it up. */
3671
3672 if (dpc != NULL)
3673 KeInsertQueueDpc(dpc, NULL, NULL);
3674 }
3675
3676 #ifdef NTOSKRNL_DEBUG_TIMERS
3677 static int
3678 sysctl_show_timers(SYSCTL_HANDLER_ARGS)
3679 {
3680 int ret;
3681
3682 ret = 0;
3683 ntoskrnl_show_timers();
3684 return (sysctl_handle_int(oidp, &ret, 0, req));
3685 }
3686
3687 static void
3688 ntoskrnl_show_timers()
3689 {
3690 int i = 0;
3691 list_entry *l;
3692
3693 mtx_lock_spin(&ntoskrnl_calllock);
3694 l = ntoskrnl_calllist.nle_flink;
3695 while(l != &ntoskrnl_calllist) {
3696 i++;
3697 l = l->nle_flink;
3698 }
3699 mtx_unlock_spin(&ntoskrnl_calllock);
3700
3701 printf("\n");
3702 printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS);
3703 printf("timer sets: %qu\n", ntoskrnl_timer_sets);
3704 printf("timer reloads: %qu\n", ntoskrnl_timer_reloads);
3705 printf("timer cancels: %qu\n", ntoskrnl_timer_cancels);
3706 printf("timer fires: %qu\n", ntoskrnl_timer_fires);
3707 printf("\n");
3708 }
3709 #endif
3710
3711 /*
3712 * Must be called with dispatcher lock held.
3713 */
3714
3715 static void
3716 ntoskrnl_insert_timer(timer, ticks)
3717 ktimer *timer;
3718 int ticks;
3719 {
3720 callout_entry *e;
3721 list_entry *l;
3722 struct callout *c;
3723
3724 /*
3725 * Try and allocate a timer.
3726 */
3727 mtx_lock_spin(&ntoskrnl_calllock);
3728 if (IsListEmpty(&ntoskrnl_calllist)) {
3729 mtx_unlock_spin(&ntoskrnl_calllock);
3730 #ifdef NTOSKRNL_DEBUG_TIMERS
3731 ntoskrnl_show_timers();
3732 #endif
3733 panic("out of timers!");
3734 }
3735 l = RemoveHeadList(&ntoskrnl_calllist);
3736 mtx_unlock_spin(&ntoskrnl_calllock);
3737
3738 e = CONTAINING_RECORD(l, callout_entry, ce_list);
3739 c = &e->ce_callout;
3740
3741 timer->k_callout = c;
3742
3743 callout_init(c, 1);
3744 callout_reset(c, ticks, ntoskrnl_timercall, timer);
3745 }
3746
3747 static void
3748 ntoskrnl_remove_timer(timer)
3749 ktimer *timer;
3750 {
3751 callout_entry *e;
3752
3753 e = (callout_entry *)timer->k_callout;
3754 callout_stop(timer->k_callout);
3755
3756 mtx_lock_spin(&ntoskrnl_calllock);
3757 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
3758 mtx_unlock_spin(&ntoskrnl_calllock);
3759 }
3760
3761 void
3762 KeInitializeTimer(timer)
3763 ktimer *timer;
3764 {
3765 if (timer == NULL)
3766 return;
3767
3768 KeInitializeTimerEx(timer, EVENT_TYPE_NOTIFY);
3769 }
3770
3771 void
3772 KeInitializeTimerEx(timer, type)
3773 ktimer *timer;
3774 uint32_t type;
3775 {
3776 if (timer == NULL)
3777 return;
3778
3779 bzero((char *)timer, sizeof(ktimer));
3780 InitializeListHead((&timer->k_header.dh_waitlisthead));
3781 timer->k_header.dh_sigstate = FALSE;
3782 timer->k_header.dh_inserted = FALSE;
3783 if (type == EVENT_TYPE_NOTIFY)
3784 timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER;
3785 else
3786 timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER;
3787 timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t);
3788 }
3789
3790 /*
3791 * DPC subsystem. A Windows Defered Procedure Call has the following
3792 * properties:
3793 * - It runs at DISPATCH_LEVEL.
3794 * - It can have one of 3 importance values that control when it
3795 * runs relative to other DPCs in the queue.
3796 * - On SMP systems, it can be set to run on a specific processor.
3797 * In order to satisfy the last property, we create a DPC thread for
3798 * each CPU in the system and bind it to that CPU. Each thread
3799 * maintains three queues with different importance levels, which
3800 * will be processed in order from lowest to highest.
3801 *
3802 * In Windows, interrupt handlers run as DPCs. (Not to be confused
3803 * with ISRs, which run in interrupt context and can preempt DPCs.)
3804 * ISRs are given the highest importance so that they'll take
3805 * precedence over timers and other things.
3806 */
3807
3808 static void
3809 ntoskrnl_dpc_thread(arg)
3810 void *arg;
3811 {
3812 kdpc_queue *kq;
3813 kdpc *d;
3814 list_entry *l;
3815 uint8_t irql;
3816
3817 kq = arg;
3818
3819 InitializeListHead(&kq->kq_disp);
3820 kq->kq_td = curthread;
3821 kq->kq_exit = 0;
3822 kq->kq_running = FALSE;
3823 KeInitializeSpinLock(&kq->kq_lock);
3824 KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
3825 KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE);
3826
3827 /*
3828 * Elevate our priority. DPCs are used to run interrupt
3829 * handlers, and they should trigger as soon as possible
3830 * once scheduled by an ISR.
3831 */
3832
3833 thread_lock(curthread);
3834 #ifdef NTOSKRNL_MULTIPLE_DPCS
3835 sched_bind(curthread, kq->kq_cpu);
3836 #endif
3837 sched_prio(curthread, PRI_MIN_KERN);
3838 thread_unlock(curthread);
3839
3840 while (1) {
3841 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
3842
3843 KeAcquireSpinLock(&kq->kq_lock, &irql);
3844
3845 if (kq->kq_exit) {
3846 kq->kq_exit = 0;
3847 KeReleaseSpinLock(&kq->kq_lock, irql);
3848 break;
3849 }
3850
3851 kq->kq_running = TRUE;
3852
3853 while (!IsListEmpty(&kq->kq_disp)) {
3854 l = RemoveHeadList((&kq->kq_disp));
3855 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3856 InitializeListHead((&d->k_dpclistentry));
3857 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
3858 MSCALL4(d->k_deferedfunc, d, d->k_deferredctx,
3859 d->k_sysarg1, d->k_sysarg2);
3860 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3861 }
3862
3863 kq->kq_running = FALSE;
3864
3865 KeReleaseSpinLock(&kq->kq_lock, irql);
3866
3867 KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE);
3868 }
3869
3870 kproc_exit(0);
3871 return; /* notreached */
3872 }
3873
3874 static void
3875 ntoskrnl_destroy_dpc_threads(void)
3876 {
3877 kdpc_queue *kq;
3878 kdpc dpc;
3879 int i;
3880
3881 kq = kq_queues;
3882 #ifdef NTOSKRNL_MULTIPLE_DPCS
3883 for (i = 0; i < mp_ncpus; i++) {
3884 #else
3885 for (i = 0; i < 1; i++) {
3886 #endif
3887 kq += i;
3888
3889 kq->kq_exit = 1;
3890 KeInitializeDpc(&dpc, NULL, NULL);
3891 KeSetTargetProcessorDpc(&dpc, i);
3892 KeInsertQueueDpc(&dpc, NULL, NULL);
3893 while (kq->kq_exit)
3894 tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10);
3895 }
3896 }
3897
3898 static uint8_t
3899 ntoskrnl_insert_dpc(head, dpc)
3900 list_entry *head;
3901 kdpc *dpc;
3902 {
3903 list_entry *l;
3904 kdpc *d;
3905
3906 l = head->nle_flink;
3907 while (l != head) {
3908 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3909 if (d == dpc)
3910 return (FALSE);
3911 l = l->nle_flink;
3912 }
3913
3914 if (dpc->k_importance == KDPC_IMPORTANCE_LOW)
3915 InsertTailList((head), (&dpc->k_dpclistentry));
3916 else
3917 InsertHeadList((head), (&dpc->k_dpclistentry));
3918
3919 return (TRUE);
3920 }
3921
3922 void
3923 KeInitializeDpc(dpc, dpcfunc, dpcctx)
3924 kdpc *dpc;
3925 void *dpcfunc;
3926 void *dpcctx;
3927 {
3928
3929 if (dpc == NULL)
3930 return;
3931
3932 dpc->k_deferedfunc = dpcfunc;
3933 dpc->k_deferredctx = dpcctx;
3934 dpc->k_num = KDPC_CPU_DEFAULT;
3935 dpc->k_importance = KDPC_IMPORTANCE_MEDIUM;
3936 InitializeListHead((&dpc->k_dpclistentry));
3937 }
3938
3939 uint8_t
3940 KeInsertQueueDpc(dpc, sysarg1, sysarg2)
3941 kdpc *dpc;
3942 void *sysarg1;
3943 void *sysarg2;
3944 {
3945 kdpc_queue *kq;
3946 uint8_t r;
3947 uint8_t irql;
3948
3949 if (dpc == NULL)
3950 return (FALSE);
3951
3952 kq = kq_queues;
3953
3954 #ifdef NTOSKRNL_MULTIPLE_DPCS
3955 KeRaiseIrql(DISPATCH_LEVEL, &irql);
3956
3957 /*
3958 * By default, the DPC is queued to run on the same CPU
3959 * that scheduled it.
3960 */
3961
3962 if (dpc->k_num == KDPC_CPU_DEFAULT)
3963 kq += curthread->td_oncpu;
3964 else
3965 kq += dpc->k_num;
3966 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3967 #else
3968 KeAcquireSpinLock(&kq->kq_lock, &irql);
3969 #endif
3970
3971 r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc);
3972 if (r == TRUE) {
3973 dpc->k_sysarg1 = sysarg1;
3974 dpc->k_sysarg2 = sysarg2;
3975 }
3976 KeReleaseSpinLock(&kq->kq_lock, irql);
3977
3978 if (r == FALSE)
3979 return (r);
3980
3981 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
3982
3983 return (r);
3984 }
3985
3986 uint8_t
3987 KeRemoveQueueDpc(dpc)
3988 kdpc *dpc;
3989 {
3990 kdpc_queue *kq;
3991 uint8_t irql;
3992
3993 if (dpc == NULL)
3994 return (FALSE);
3995
3996 #ifdef NTOSKRNL_MULTIPLE_DPCS
3997 KeRaiseIrql(DISPATCH_LEVEL, &irql);
3998
3999 kq = kq_queues + dpc->k_num;
4000
4001 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
4002 #else
4003 kq = kq_queues;
4004 KeAcquireSpinLock(&kq->kq_lock, &irql);
4005 #endif
4006
4007 if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) {
4008 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
4009 KeLowerIrql(irql);
4010 return (FALSE);
4011 }
4012
4013 RemoveEntryList((&dpc->k_dpclistentry));
4014 InitializeListHead((&dpc->k_dpclistentry));
4015
4016 KeReleaseSpinLock(&kq->kq_lock, irql);
4017
4018 return (TRUE);
4019 }
4020
4021 void
4022 KeSetImportanceDpc(dpc, imp)
4023 kdpc *dpc;
4024 uint32_t imp;
4025 {
4026 if (imp != KDPC_IMPORTANCE_LOW &&
4027 imp != KDPC_IMPORTANCE_MEDIUM &&
4028 imp != KDPC_IMPORTANCE_HIGH)
4029 return;
4030
4031 dpc->k_importance = (uint8_t)imp;
4032 }
4033
4034 void
4035 KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu)
4036 {
4037 if (cpu > mp_ncpus)
4038 return;
4039
4040 dpc->k_num = cpu;
4041 }
4042
4043 void
4044 KeFlushQueuedDpcs(void)
4045 {
4046 kdpc_queue *kq;
4047 int i;
4048
4049 /*
4050 * Poke each DPC queue and wait
4051 * for them to drain.
4052 */
4053
4054 #ifdef NTOSKRNL_MULTIPLE_DPCS
4055 for (i = 0; i < mp_ncpus; i++) {
4056 #else
4057 for (i = 0; i < 1; i++) {
4058 #endif
4059 kq = kq_queues + i;
4060 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
4061 KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL);
4062 }
4063 }
4064
4065 uint32_t
4066 KeGetCurrentProcessorNumber(void)
4067 {
4068 return ((uint32_t)curthread->td_oncpu);
4069 }
4070
4071 uint8_t
4072 KeSetTimerEx(timer, duetime, period, dpc)
4073 ktimer *timer;
4074 int64_t duetime;
4075 uint32_t period;
4076 kdpc *dpc;
4077 {
4078 struct timeval tv;
4079 uint64_t curtime;
4080 uint8_t pending;
4081
4082 if (timer == NULL)
4083 return (FALSE);
4084
4085 mtx_lock(&ntoskrnl_dispatchlock);
4086
4087 if (timer->k_header.dh_inserted == TRUE) {
4088 ntoskrnl_remove_timer(timer);
4089 #ifdef NTOSKRNL_DEBUG_TIMERS
4090 ntoskrnl_timer_cancels++;
4091 #endif
4092 timer->k_header.dh_inserted = FALSE;
4093 pending = TRUE;
4094 } else
4095 pending = FALSE;
4096
4097 timer->k_duetime = duetime;
4098 timer->k_period = period;
4099 timer->k_header.dh_sigstate = FALSE;
4100 timer->k_dpc = dpc;
4101
4102 if (duetime < 0) {
4103 tv.tv_sec = - (duetime) / 10000000;
4104 tv.tv_usec = (- (duetime) / 10) -
4105 (tv.tv_sec * 1000000);
4106 } else {
4107 ntoskrnl_time(&curtime);
4108 if (duetime < curtime)
4109 tv.tv_sec = tv.tv_usec = 0;
4110 else {
4111 tv.tv_sec = ((duetime) - curtime) / 10000000;
4112 tv.tv_usec = ((duetime) - curtime) / 10 -
4113 (tv.tv_sec * 1000000);
4114 }
4115 }
4116
4117 timer->k_header.dh_inserted = TRUE;
4118 ntoskrnl_insert_timer(timer, tvtohz(&tv));
4119 #ifdef NTOSKRNL_DEBUG_TIMERS
4120 ntoskrnl_timer_sets++;
4121 #endif
4122
4123 mtx_unlock(&ntoskrnl_dispatchlock);
4124
4125 return (pending);
4126 }
4127
4128 uint8_t
4129 KeSetTimer(timer, duetime, dpc)
4130 ktimer *timer;
4131 int64_t duetime;
4132 kdpc *dpc;
4133 {
4134 return (KeSetTimerEx(timer, duetime, 0, dpc));
4135 }
4136
4137 /*
4138 * The Windows DDK documentation seems to say that cancelling
4139 * a timer that has a DPC will result in the DPC also being
4140 * cancelled, but this isn't really the case.
4141 */
4142
4143 uint8_t
4144 KeCancelTimer(timer)
4145 ktimer *timer;
4146 {
4147 uint8_t pending;
4148
4149 if (timer == NULL)
4150 return (FALSE);
4151
4152 mtx_lock(&ntoskrnl_dispatchlock);
4153
4154 pending = timer->k_header.dh_inserted;
4155
4156 if (timer->k_header.dh_inserted == TRUE) {
4157 timer->k_header.dh_inserted = FALSE;
4158 ntoskrnl_remove_timer(timer);
4159 #ifdef NTOSKRNL_DEBUG_TIMERS
4160 ntoskrnl_timer_cancels++;
4161 #endif
4162 }
4163
4164 mtx_unlock(&ntoskrnl_dispatchlock);
4165
4166 return (pending);
4167 }
4168
4169 uint8_t
4170 KeReadStateTimer(timer)
4171 ktimer *timer;
4172 {
4173 return (timer->k_header.dh_sigstate);
4174 }
4175
4176 static int32_t
4177 KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval)
4178 {
4179 ktimer timer;
4180
4181 if (wait_mode != 0)
4182 panic("invalid wait_mode %d", wait_mode);
4183
4184 KeInitializeTimer(&timer);
4185 KeSetTimer(&timer, *interval, NULL);
4186 KeWaitForSingleObject(&timer, 0, 0, alertable, NULL);
4187
4188 return STATUS_SUCCESS;
4189 }
4190
4191 static uint64_t
4192 KeQueryInterruptTime(void)
4193 {
4194 int ticks;
4195 struct timeval tv;
4196
4197 getmicrouptime(&tv);
4198
4199 ticks = tvtohz(&tv);
4200
4201 return ticks * howmany(10000000, hz);
4202 }
4203
4204 static struct thread *
4205 KeGetCurrentThread(void)
4206 {
4207
4208 return curthread;
4209 }
4210
4211 static int32_t
4212 KeSetPriorityThread(td, pri)
4213 struct thread *td;
4214 int32_t pri;
4215 {
4216 int32_t old;
4217
4218 if (td == NULL)
4219 return LOW_REALTIME_PRIORITY;
4220
4221 if (td->td_priority <= PRI_MIN_KERN)
4222 old = HIGH_PRIORITY;
4223 else if (td->td_priority >= PRI_MAX_KERN)
4224 old = LOW_PRIORITY;
4225 else
4226 old = LOW_REALTIME_PRIORITY;
4227
4228 thread_lock(td);
4229 if (pri == HIGH_PRIORITY)
4230 sched_prio(td, PRI_MIN_KERN);
4231 if (pri == LOW_REALTIME_PRIORITY)
4232 sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2);
4233 if (pri == LOW_PRIORITY)
4234 sched_prio(td, PRI_MAX_KERN);
4235 thread_unlock(td);
4236
4237 return old;
4238 }
4239
4240 static void
4241 dummy()
4242 {
4243 printf("ntoskrnl dummy called...\n");
4244 }
4245
4246 image_patch_table ntoskrnl_functbl[] = {
4247 IMPORT_SFUNC(RtlZeroMemory, 2),
4248 IMPORT_SFUNC(RtlSecureZeroMemory, 2),
4249 IMPORT_SFUNC(RtlFillMemory, 3),
4250 IMPORT_SFUNC(RtlMoveMemory, 3),
4251 IMPORT_SFUNC(RtlCharToInteger, 3),
4252 IMPORT_SFUNC(RtlCopyMemory, 3),
4253 IMPORT_SFUNC(RtlCopyString, 2),
4254 IMPORT_SFUNC(RtlCompareMemory, 3),
4255 IMPORT_SFUNC(RtlEqualUnicodeString, 3),
4256 IMPORT_SFUNC(RtlCopyUnicodeString, 2),
4257 IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3),
4258 IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3),
4259 IMPORT_SFUNC(RtlInitAnsiString, 2),
4260 IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2),
4261 IMPORT_SFUNC(RtlInitUnicodeString, 2),
4262 IMPORT_SFUNC(RtlFreeAnsiString, 1),
4263 IMPORT_SFUNC(RtlFreeUnicodeString, 1),
4264 IMPORT_SFUNC(RtlUnicodeStringToInteger, 3),
4265 IMPORT_CFUNC(sprintf, 0),
4266 IMPORT_CFUNC(vsprintf, 0),
4267 IMPORT_CFUNC_MAP(_snprintf, snprintf, 0),
4268 IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0),
4269 IMPORT_CFUNC(DbgPrint, 0),
4270 IMPORT_SFUNC(DbgBreakPoint, 0),
4271 IMPORT_SFUNC(KeBugCheckEx, 5),
4272 IMPORT_CFUNC(strncmp, 0),
4273 IMPORT_CFUNC(strcmp, 0),
4274 IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0),
4275 IMPORT_CFUNC(strncpy, 0),
4276 IMPORT_CFUNC(strcpy, 0),
4277 IMPORT_CFUNC(strlen, 0),
4278 IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0),
4279 IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0),
4280 IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0),
4281 IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0),
4282 IMPORT_CFUNC_MAP(strchr, index, 0),
4283 IMPORT_CFUNC_MAP(strrchr, rindex, 0),
4284 IMPORT_CFUNC(memcpy, 0),
4285 IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0),
4286 IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0),
4287 IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0),
4288 IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4),
4289 IMPORT_SFUNC(IoGetDriverObjectExtension, 2),
4290 IMPORT_FFUNC(IofCallDriver, 2),
4291 IMPORT_FFUNC(IofCompleteRequest, 2),
4292 IMPORT_SFUNC(IoAcquireCancelSpinLock, 1),
4293 IMPORT_SFUNC(IoReleaseCancelSpinLock, 1),
4294 IMPORT_SFUNC(IoCancelIrp, 1),
4295 IMPORT_SFUNC(IoConnectInterrupt, 11),
4296 IMPORT_SFUNC(IoDisconnectInterrupt, 1),
4297 IMPORT_SFUNC(IoCreateDevice, 7),
4298 IMPORT_SFUNC(IoDeleteDevice, 1),
4299 IMPORT_SFUNC(IoGetAttachedDevice, 1),
4300 IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2),
4301 IMPORT_SFUNC(IoDetachDevice, 1),
4302 IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7),
4303 IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6),
4304 IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9),
4305 IMPORT_SFUNC(IoAllocateIrp, 2),
4306 IMPORT_SFUNC(IoReuseIrp, 2),
4307 IMPORT_SFUNC(IoMakeAssociatedIrp, 2),
4308 IMPORT_SFUNC(IoFreeIrp, 1),
4309 IMPORT_SFUNC(IoInitializeIrp, 3),
4310 IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1),
4311 IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2),
4312 IMPORT_SFUNC(KeSynchronizeExecution, 3),
4313 IMPORT_SFUNC(KeWaitForSingleObject, 5),
4314 IMPORT_SFUNC(KeWaitForMultipleObjects, 8),
4315 IMPORT_SFUNC(_allmul, 4),
4316 IMPORT_SFUNC(_alldiv, 4),
4317 IMPORT_SFUNC(_allrem, 4),
4318 IMPORT_RFUNC(_allshr, 0),
4319 IMPORT_RFUNC(_allshl, 0),
4320 IMPORT_SFUNC(_aullmul, 4),
4321 IMPORT_SFUNC(_aulldiv, 4),
4322 IMPORT_SFUNC(_aullrem, 4),
4323 IMPORT_RFUNC(_aullshr, 0),
4324 IMPORT_RFUNC(_aullshl, 0),
4325 IMPORT_CFUNC(atoi, 0),
4326 IMPORT_CFUNC(atol, 0),
4327 IMPORT_CFUNC(rand, 0),
4328 IMPORT_CFUNC(srand, 0),
4329 IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2),
4330 IMPORT_SFUNC(READ_REGISTER_USHORT, 1),
4331 IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2),
4332 IMPORT_SFUNC(READ_REGISTER_ULONG, 1),
4333 IMPORT_SFUNC(READ_REGISTER_UCHAR, 1),
4334 IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2),
4335 IMPORT_SFUNC(ExInitializePagedLookasideList, 7),
4336 IMPORT_SFUNC(ExDeletePagedLookasideList, 1),
4337 IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7),
4338 IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1),
4339 IMPORT_FFUNC(InterlockedPopEntrySList, 1),
4340 IMPORT_FFUNC(InitializeSListHead, 1),
4341 IMPORT_FFUNC(InterlockedPushEntrySList, 2),
4342 IMPORT_SFUNC(ExQueryDepthSList, 1),
4343 IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList,
4344 InterlockedPopEntrySList, 1),
4345 IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList,
4346 InterlockedPushEntrySList, 2),
4347 IMPORT_FFUNC(ExInterlockedPopEntrySList, 2),
4348 IMPORT_FFUNC(ExInterlockedPushEntrySList, 3),
4349 IMPORT_SFUNC(ExAllocatePoolWithTag, 3),
4350 IMPORT_SFUNC(ExFreePoolWithTag, 2),
4351 IMPORT_SFUNC(ExFreePool, 1),
4352 #ifdef __i386__
4353 IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1),
4354 IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1),
4355 IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1),
4356 #else
4357 /*
4358 * For AMD64, we can get away with just mapping
4359 * KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock()
4360 * because the calling conventions end up being the same.
4361 * On i386, we have to be careful because KfAcquireSpinLock()
4362 * is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't.
4363 */
4364 IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1),
4365 IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1),
4366 IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1),
4367 #endif
4368 IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1),
4369 IMPORT_FFUNC(InterlockedIncrement, 1),
4370 IMPORT_FFUNC(InterlockedDecrement, 1),
4371 IMPORT_FFUNC(InterlockedExchange, 2),
4372 IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2),
4373 IMPORT_SFUNC(IoAllocateMdl, 5),
4374 IMPORT_SFUNC(IoFreeMdl, 1),
4375 IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1),
4376 IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3),
4377 IMPORT_SFUNC(MmFreeContiguousMemory, 1),
4378 IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3),
4379 IMPORT_SFUNC(MmSizeOfMdl, 1),
4380 IMPORT_SFUNC(MmMapLockedPages, 2),
4381 IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6),
4382 IMPORT_SFUNC(MmUnmapLockedPages, 2),
4383 IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1),
4384 IMPORT_SFUNC(MmGetPhysicalAddress, 1),
4385 IMPORT_SFUNC(MmGetSystemRoutineAddress, 1),
4386 IMPORT_SFUNC(MmIsAddressValid, 1),
4387 IMPORT_SFUNC(MmMapIoSpace, 3 + 1),
4388 IMPORT_SFUNC(MmUnmapIoSpace, 2),
4389 IMPORT_SFUNC(KeInitializeSpinLock, 1),
4390 IMPORT_SFUNC(IoIsWdmVersionAvailable, 2),
4391 IMPORT_SFUNC(IoOpenDeviceRegistryKey, 4),
4392 IMPORT_SFUNC(IoGetDeviceObjectPointer, 4),
4393 IMPORT_SFUNC(IoGetDeviceProperty, 5),
4394 IMPORT_SFUNC(IoAllocateWorkItem, 1),
4395 IMPORT_SFUNC(IoFreeWorkItem, 1),
4396 IMPORT_SFUNC(IoQueueWorkItem, 4),
4397 IMPORT_SFUNC(ExQueueWorkItem, 2),
4398 IMPORT_SFUNC(ntoskrnl_workitem, 2),
4399 IMPORT_SFUNC(KeInitializeMutex, 2),
4400 IMPORT_SFUNC(KeReleaseMutex, 2),
4401 IMPORT_SFUNC(KeReadStateMutex, 1),
4402 IMPORT_SFUNC(KeInitializeEvent, 3),
4403 IMPORT_SFUNC(KeSetEvent, 3),
4404 IMPORT_SFUNC(KeResetEvent, 1),
4405 IMPORT_SFUNC(KeClearEvent, 1),
4406 IMPORT_SFUNC(KeReadStateEvent, 1),
4407 IMPORT_SFUNC(KeInitializeTimer, 1),
4408 IMPORT_SFUNC(KeInitializeTimerEx, 2),
4409 IMPORT_SFUNC(KeSetTimer, 3),
4410 IMPORT_SFUNC(KeSetTimerEx, 4),
4411 IMPORT_SFUNC(KeCancelTimer, 1),
4412 IMPORT_SFUNC(KeReadStateTimer, 1),
4413 IMPORT_SFUNC(KeInitializeDpc, 3),
4414 IMPORT_SFUNC(KeInsertQueueDpc, 3),
4415 IMPORT_SFUNC(KeRemoveQueueDpc, 1),
4416 IMPORT_SFUNC(KeSetImportanceDpc, 2),
4417 IMPORT_SFUNC(KeSetTargetProcessorDpc, 2),
4418 IMPORT_SFUNC(KeFlushQueuedDpcs, 0),
4419 IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1),
4420 IMPORT_SFUNC(ObReferenceObjectByHandle, 6),
4421 IMPORT_FFUNC(ObfDereferenceObject, 1),
4422 IMPORT_SFUNC(ZwClose, 1),
4423 IMPORT_SFUNC(PsCreateSystemThread, 7),
4424 IMPORT_SFUNC(PsTerminateSystemThread, 1),
4425 IMPORT_SFUNC(IoWMIRegistrationControl, 2),
4426 IMPORT_SFUNC(WmiQueryTraceInformation, 5),
4427 IMPORT_CFUNC(WmiTraceMessage, 0),
4428 IMPORT_SFUNC(KeQuerySystemTime, 1),
4429 IMPORT_CFUNC(KeTickCount, 0),
4430 IMPORT_SFUNC(KeDelayExecutionThread, 3),
4431 IMPORT_SFUNC(KeQueryInterruptTime, 0),
4432 IMPORT_SFUNC(KeGetCurrentThread, 0),
4433 IMPORT_SFUNC(KeSetPriorityThread, 2),
4434
4435 /*
4436 * This last entry is a catch-all for any function we haven't
4437 * implemented yet. The PE import list patching routine will
4438 * use it for any function that doesn't have an explicit match
4439 * in this table.
4440 */
4441
4442 { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
4443
4444 /* End of list. */
4445 { NULL, NULL, NULL }
4446 };
4447