1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2010-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was developed in part by Philip Paeps under contract for 8 * Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * The views and conclusions contained in the software and documentation are 32 * those of the authors and should not be interpreted as representing official 33 * policies, either expressed or implied, of the FreeBSD Project. 34 * 35 * $FreeBSD$ 36 */ 37 38 #ifndef _SYS_EFSYS_H 39 #define _SYS_EFSYS_H 40 41 #ifdef __cplusplus 42 extern "C" { 43 #endif 44 45 #include <sys/param.h> 46 #include <sys/bus.h> 47 #include <sys/endian.h> 48 #include <sys/lock.h> 49 #include <sys/malloc.h> 50 #include <sys/mbuf.h> 51 #include <sys/mutex.h> 52 #include <sys/rwlock.h> 53 #include <sys/sdt.h> 54 #include <sys/systm.h> 55 56 #include <machine/bus.h> 57 #include <machine/endian.h> 58 59 #define EFSYS_HAS_UINT64 1 60 #if defined(__x86_64__) 61 #define EFSYS_USE_UINT64 1 62 #else 63 #define EFSYS_USE_UINT64 0 64 #endif 65 #define EFSYS_HAS_SSE2_M128 0 66 #if _BYTE_ORDER == _BIG_ENDIAN 67 #define EFSYS_IS_BIG_ENDIAN 1 68 #define EFSYS_IS_LITTLE_ENDIAN 0 69 #elif _BYTE_ORDER == _LITTLE_ENDIAN 70 #define EFSYS_IS_BIG_ENDIAN 0 71 #define EFSYS_IS_LITTLE_ENDIAN 1 72 #endif 73 #include "efx_types.h" 74 75 /* Common code requires this */ 76 #if __FreeBSD_version < 800068 77 #define memmove(d, s, l) bcopy(s, d, l) 78 #endif 79 80 /* FreeBSD equivalents of Solaris things */ 81 #ifndef _NOTE 82 #define _NOTE(s) 83 #endif 84 85 #ifndef B_FALSE 86 #define B_FALSE FALSE 87 #endif 88 #ifndef B_TRUE 89 #define B_TRUE TRUE 90 #endif 91 92 #ifndef IS_P2ALIGNED 93 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 94 #endif 95 96 #ifndef P2ROUNDUP 97 #define P2ROUNDUP(x, align) (-(-(x) & -(align))) 98 #endif 99 100 #ifndef P2ALIGN 101 #define P2ALIGN(_x, _a) ((_x) & -(_a)) 102 #endif 103 104 #ifndef IS2P 105 #define ISP2(x) (((x) & ((x) - 1)) == 0) 106 #endif 107 108 #if defined(__x86_64__) && __FreeBSD_version >= 1000000 109 110 #define SFXGE_USE_BUS_SPACE_8 1 111 112 #if !defined(bus_space_read_stream_8) 113 114 #define bus_space_read_stream_8(t, h, o) \ 115 bus_space_read_8((t), (h), (o)) 116 117 #define bus_space_write_stream_8(t, h, o, v) \ 118 bus_space_write_8((t), (h), (o), (v)) 119 120 #endif 121 122 #endif 123 124 #define ENOTACTIVE EINVAL 125 126 /* Memory type to use on FreeBSD */ 127 MALLOC_DECLARE(M_SFXGE); 128 129 /* Machine dependend prefetch wrappers */ 130 #if defined(__i386__) || defined(__amd64__) 131 static __inline void 132 prefetch_read_many(void *addr) 133 { 134 135 __asm__( 136 "prefetcht0 (%0)" 137 : 138 : "r" (addr)); 139 } 140 141 static __inline void 142 prefetch_read_once(void *addr) 143 { 144 145 __asm__( 146 "prefetchnta (%0)" 147 : 148 : "r" (addr)); 149 } 150 #elif defined(__sparc64__) 151 static __inline void 152 prefetch_read_many(void *addr) 153 { 154 155 __asm__( 156 "prefetch [%0], 0" 157 : 158 : "r" (addr)); 159 } 160 161 static __inline void 162 prefetch_read_once(void *addr) 163 { 164 165 __asm__( 166 "prefetch [%0], 1" 167 : 168 : "r" (addr)); 169 } 170 #else 171 static __inline void 172 prefetch_read_many(void *addr) 173 { 174 175 } 176 177 static __inline void 178 prefetch_read_once(void *addr) 179 { 180 181 } 182 #endif 183 184 #if defined(__i386__) || defined(__amd64__) 185 #include <vm/vm.h> 186 #include <vm/pmap.h> 187 #endif 188 static __inline void 189 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 190 struct mbuf *m, bus_dma_segment_t *seg) 191 { 192 #if defined(__i386__) || defined(__amd64__) 193 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 194 seg->ds_len = m->m_len; 195 #else 196 int nsegstmp; 197 198 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 199 #endif 200 } 201 202 /* Modifiers used for Windows builds */ 203 #define __in 204 #define __in_opt 205 #define __in_ecount(_n) 206 #define __in_ecount_opt(_n) 207 #define __in_bcount(_n) 208 #define __in_bcount_opt(_n) 209 210 #define __out 211 #define __out_opt 212 #define __out_ecount(_n) 213 #define __out_ecount_opt(_n) 214 #define __out_bcount(_n) 215 #define __out_bcount_opt(_n) 216 #define __out_bcount_part(_n, _l) 217 #define __out_bcount_part_opt(_n, _l) 218 219 #define __deref_out 220 221 #define __inout 222 #define __inout_opt 223 #define __inout_ecount(_n) 224 #define __inout_ecount_opt(_n) 225 #define __inout_bcount(_n) 226 #define __inout_bcount_opt(_n) 227 #define __inout_bcount_full_opt(_n) 228 229 #define __deref_out_bcount_opt(n) 230 231 #define __checkReturn 232 #define __success(_x) 233 234 #define __drv_when(_p, _c) 235 236 /* Code inclusion options */ 237 238 239 #define EFSYS_OPT_NAMES 1 240 241 #define EFSYS_OPT_SIENA 1 242 #define EFSYS_OPT_HUNTINGTON 1 243 #define EFSYS_OPT_MEDFORD 1 244 #ifdef DEBUG 245 #define EFSYS_OPT_CHECK_REG 1 246 #else 247 #define EFSYS_OPT_CHECK_REG 0 248 #endif 249 250 #define EFSYS_OPT_MCDI 1 251 #define EFSYS_OPT_MCDI_LOGGING 0 252 #define EFSYS_OPT_MCDI_PROXY_AUTH 0 253 254 #define EFSYS_OPT_MAC_STATS 1 255 256 #define EFSYS_OPT_LOOPBACK 0 257 258 #define EFSYS_OPT_MON_MCDI 0 259 #define EFSYS_OPT_MON_STATS 0 260 261 #define EFSYS_OPT_PHY_STATS 1 262 #define EFSYS_OPT_BIST 1 263 #define EFSYS_OPT_PHY_LED_CONTROL 1 264 #define EFSYS_OPT_PHY_FLAGS 0 265 266 #define EFSYS_OPT_VPD 1 267 #define EFSYS_OPT_NVRAM 1 268 #define EFSYS_OPT_BOOTCFG 0 269 270 #define EFSYS_OPT_DIAG 0 271 #define EFSYS_OPT_RX_SCALE 1 272 #define EFSYS_OPT_QSTATS 1 273 #define EFSYS_OPT_FILTER 1 274 #define EFSYS_OPT_RX_SCATTER 0 275 276 #define EFSYS_OPT_EV_PREFETCH 0 277 278 #define EFSYS_OPT_DECODE_INTR_FATAL 1 279 280 #define EFSYS_OPT_LICENSING 0 281 282 #define EFSYS_OPT_ALLOW_UNCONFIGURED_NIC 0 283 284 #define EFSYS_OPT_RX_PACKED_STREAM 0 285 286 #define EFSYS_OPT_TUNNEL 0 287 288 /* ID */ 289 290 typedef struct __efsys_identifier_s efsys_identifier_t; 291 292 /* PROBE */ 293 294 #ifndef DTRACE_PROBE 295 296 #define EFSYS_PROBE(_name) 297 298 #define EFSYS_PROBE1(_name, _type1, _arg1) 299 300 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 301 302 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 303 _type3, _arg3) 304 305 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 306 _type3, _arg3, _type4, _arg4) 307 308 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 309 _type3, _arg3, _type4, _arg4, _type5, _arg5) 310 311 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 312 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 313 _type6, _arg6) 314 315 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 316 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 317 _type6, _arg6, _type7, _arg7) 318 319 #else /* DTRACE_PROBE */ 320 321 #define EFSYS_PROBE(_name) \ 322 DTRACE_PROBE(_name) 323 324 #define EFSYS_PROBE1(_name, _type1, _arg1) \ 325 DTRACE_PROBE1(_name, _type1, _arg1) 326 327 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 328 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 329 330 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 331 _type3, _arg3) \ 332 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 333 _type3, _arg3) 334 335 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 336 _type3, _arg3, _type4, _arg4) \ 337 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 338 _type3, _arg3, _type4, _arg4) 339 340 #ifdef DTRACE_PROBE5 341 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 342 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 343 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 344 _type3, _arg3, _type4, _arg4, _type5, _arg5) 345 #else 346 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 347 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 348 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 349 _type3, _arg3, _type4, _arg4) 350 #endif 351 352 #ifdef DTRACE_PROBE6 353 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 354 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 355 _type6, _arg6) \ 356 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 357 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 358 _type6, _arg6) 359 #else 360 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 361 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 362 _type6, _arg6) \ 363 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 364 _type3, _arg3, _type4, _arg4, _type5, _arg5) 365 #endif 366 367 #ifdef DTRACE_PROBE7 368 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 369 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 370 _type6, _arg6, _type7, _arg7) \ 371 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 372 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 373 _type6, _arg6, _type7, _arg7) 374 #else 375 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 376 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 377 _type6, _arg6, _type7, _arg7) \ 378 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 379 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 380 _type6, _arg6) 381 #endif 382 383 #endif /* DTRACE_PROBE */ 384 385 /* DMA */ 386 387 typedef uint64_t efsys_dma_addr_t; 388 389 typedef struct efsys_mem_s { 390 bus_dma_tag_t esm_tag; 391 bus_dmamap_t esm_map; 392 caddr_t esm_base; 393 efsys_dma_addr_t esm_addr; 394 } efsys_mem_t; 395 396 397 #define EFSYS_MEM_ZERO(_esmp, _size) \ 398 do { \ 399 (void) memset((_esmp)->esm_base, 0, (_size)); \ 400 \ 401 _NOTE(CONSTANTCONDITION) \ 402 } while (B_FALSE) 403 404 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 405 do { \ 406 uint32_t *addr; \ 407 \ 408 _NOTE(CONSTANTCONDITION) \ 409 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 410 ("not power of 2 aligned")); \ 411 \ 412 addr = (void *)((_esmp)->esm_base + (_offset)); \ 413 \ 414 (_edp)->ed_u32[0] = *addr; \ 415 \ 416 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 417 uint32_t, (_edp)->ed_u32[0]); \ 418 \ 419 _NOTE(CONSTANTCONDITION) \ 420 } while (B_FALSE) 421 422 #if defined(__x86_64__) 423 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 424 do { \ 425 uint64_t *addr; \ 426 \ 427 _NOTE(CONSTANTCONDITION) \ 428 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 429 ("not power of 2 aligned")); \ 430 \ 431 addr = (void *)((_esmp)->esm_base + (_offset)); \ 432 \ 433 (_eqp)->eq_u64[0] = *addr; \ 434 \ 435 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 436 uint32_t, (_eqp)->eq_u32[1], \ 437 uint32_t, (_eqp)->eq_u32[0]); \ 438 \ 439 _NOTE(CONSTANTCONDITION) \ 440 } while (B_FALSE) 441 #else 442 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 443 do { \ 444 uint32_t *addr; \ 445 \ 446 _NOTE(CONSTANTCONDITION) \ 447 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 448 ("not power of 2 aligned")); \ 449 \ 450 addr = (void *)((_esmp)->esm_base + (_offset)); \ 451 \ 452 (_eqp)->eq_u32[0] = *addr++; \ 453 (_eqp)->eq_u32[1] = *addr; \ 454 \ 455 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 456 uint32_t, (_eqp)->eq_u32[1], \ 457 uint32_t, (_eqp)->eq_u32[0]); \ 458 \ 459 _NOTE(CONSTANTCONDITION) \ 460 } while (B_FALSE) 461 #endif 462 463 #if defined(__x86_64__) 464 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 465 do { \ 466 uint64_t *addr; \ 467 \ 468 _NOTE(CONSTANTCONDITION) \ 469 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 470 ("not power of 2 aligned")); \ 471 \ 472 addr = (void *)((_esmp)->esm_base + (_offset)); \ 473 \ 474 (_eop)->eo_u64[0] = *addr++; \ 475 (_eop)->eo_u64[1] = *addr; \ 476 \ 477 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 478 uint32_t, (_eop)->eo_u32[3], \ 479 uint32_t, (_eop)->eo_u32[2], \ 480 uint32_t, (_eop)->eo_u32[1], \ 481 uint32_t, (_eop)->eo_u32[0]); \ 482 \ 483 _NOTE(CONSTANTCONDITION) \ 484 } while (B_FALSE) 485 #else 486 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 487 do { \ 488 uint32_t *addr; \ 489 \ 490 _NOTE(CONSTANTCONDITION) \ 491 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 492 ("not power of 2 aligned")); \ 493 \ 494 addr = (void *)((_esmp)->esm_base + (_offset)); \ 495 \ 496 (_eop)->eo_u32[0] = *addr++; \ 497 (_eop)->eo_u32[1] = *addr++; \ 498 (_eop)->eo_u32[2] = *addr++; \ 499 (_eop)->eo_u32[3] = *addr; \ 500 \ 501 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 502 uint32_t, (_eop)->eo_u32[3], \ 503 uint32_t, (_eop)->eo_u32[2], \ 504 uint32_t, (_eop)->eo_u32[1], \ 505 uint32_t, (_eop)->eo_u32[0]); \ 506 \ 507 _NOTE(CONSTANTCONDITION) \ 508 } while (B_FALSE) 509 #endif 510 511 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 512 do { \ 513 uint32_t *addr; \ 514 \ 515 _NOTE(CONSTANTCONDITION) \ 516 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 517 ("not power of 2 aligned")); \ 518 \ 519 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 520 uint32_t, (_edp)->ed_u32[0]); \ 521 \ 522 addr = (void *)((_esmp)->esm_base + (_offset)); \ 523 \ 524 *addr = (_edp)->ed_u32[0]; \ 525 \ 526 _NOTE(CONSTANTCONDITION) \ 527 } while (B_FALSE) 528 529 #if defined(__x86_64__) 530 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 531 do { \ 532 uint64_t *addr; \ 533 \ 534 _NOTE(CONSTANTCONDITION) \ 535 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 536 ("not power of 2 aligned")); \ 537 \ 538 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 539 uint32_t, (_eqp)->eq_u32[1], \ 540 uint32_t, (_eqp)->eq_u32[0]); \ 541 \ 542 addr = (void *)((_esmp)->esm_base + (_offset)); \ 543 \ 544 *addr = (_eqp)->eq_u64[0]; \ 545 \ 546 _NOTE(CONSTANTCONDITION) \ 547 } while (B_FALSE) 548 549 #else 550 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 551 do { \ 552 uint32_t *addr; \ 553 \ 554 _NOTE(CONSTANTCONDITION) \ 555 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 556 ("not power of 2 aligned")); \ 557 \ 558 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 559 uint32_t, (_eqp)->eq_u32[1], \ 560 uint32_t, (_eqp)->eq_u32[0]); \ 561 \ 562 addr = (void *)((_esmp)->esm_base + (_offset)); \ 563 \ 564 *addr++ = (_eqp)->eq_u32[0]; \ 565 *addr = (_eqp)->eq_u32[1]; \ 566 \ 567 _NOTE(CONSTANTCONDITION) \ 568 } while (B_FALSE) 569 #endif 570 571 #if defined(__x86_64__) 572 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 573 do { \ 574 uint64_t *addr; \ 575 \ 576 _NOTE(CONSTANTCONDITION) \ 577 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 578 ("not power of 2 aligned")); \ 579 \ 580 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 581 uint32_t, (_eop)->eo_u32[3], \ 582 uint32_t, (_eop)->eo_u32[2], \ 583 uint32_t, (_eop)->eo_u32[1], \ 584 uint32_t, (_eop)->eo_u32[0]); \ 585 \ 586 addr = (void *)((_esmp)->esm_base + (_offset)); \ 587 \ 588 *addr++ = (_eop)->eo_u64[0]; \ 589 *addr = (_eop)->eo_u64[1]; \ 590 \ 591 _NOTE(CONSTANTCONDITION) \ 592 } while (B_FALSE) 593 #else 594 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 595 do { \ 596 uint32_t *addr; \ 597 \ 598 _NOTE(CONSTANTCONDITION) \ 599 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 600 ("not power of 2 aligned")); \ 601 \ 602 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 603 uint32_t, (_eop)->eo_u32[3], \ 604 uint32_t, (_eop)->eo_u32[2], \ 605 uint32_t, (_eop)->eo_u32[1], \ 606 uint32_t, (_eop)->eo_u32[0]); \ 607 \ 608 addr = (void *)((_esmp)->esm_base + (_offset)); \ 609 \ 610 *addr++ = (_eop)->eo_u32[0]; \ 611 *addr++ = (_eop)->eo_u32[1]; \ 612 *addr++ = (_eop)->eo_u32[2]; \ 613 *addr = (_eop)->eo_u32[3]; \ 614 \ 615 _NOTE(CONSTANTCONDITION) \ 616 } while (B_FALSE) 617 #endif 618 619 #define EFSYS_MEM_ADDR(_esmp) \ 620 ((_esmp)->esm_addr) 621 622 #define EFSYS_MEM_IS_NULL(_esmp) \ 623 ((_esmp)->esm_base == NULL) 624 625 /* BAR */ 626 627 #define SFXGE_LOCK_NAME_MAX 16 628 629 typedef struct efsys_bar_s { 630 struct mtx esb_lock; 631 char esb_lock_name[SFXGE_LOCK_NAME_MAX]; 632 bus_space_tag_t esb_tag; 633 bus_space_handle_t esb_handle; 634 int esb_rid; 635 struct resource *esb_res; 636 } efsys_bar_t; 637 638 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ 639 do { \ 640 snprintf((_esbp)->esb_lock_name, \ 641 sizeof((_esbp)->esb_lock_name), \ 642 "%s:bar", (_ifname)); \ 643 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ 644 NULL, MTX_DEF); \ 645 _NOTE(CONSTANTCONDITION) \ 646 } while (B_FALSE) 647 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \ 648 mtx_destroy(&(_esbp)->esb_lock) 649 #define SFXGE_BAR_LOCK(_esbp) \ 650 mtx_lock(&(_esbp)->esb_lock) 651 #define SFXGE_BAR_UNLOCK(_esbp) \ 652 mtx_unlock(&(_esbp)->esb_lock) 653 654 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 655 do { \ 656 _NOTE(CONSTANTCONDITION) \ 657 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 658 ("not power of 2 aligned")); \ 659 \ 660 _NOTE(CONSTANTCONDITION) \ 661 if (_lock) \ 662 SFXGE_BAR_LOCK(_esbp); \ 663 \ 664 (_edp)->ed_u32[0] = bus_space_read_stream_4( \ 665 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 666 (_offset)); \ 667 \ 668 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 669 uint32_t, (_edp)->ed_u32[0]); \ 670 \ 671 _NOTE(CONSTANTCONDITION) \ 672 if (_lock) \ 673 SFXGE_BAR_UNLOCK(_esbp); \ 674 _NOTE(CONSTANTCONDITION) \ 675 } while (B_FALSE) 676 677 #if defined(SFXGE_USE_BUS_SPACE_8) 678 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 679 do { \ 680 _NOTE(CONSTANTCONDITION) \ 681 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 682 ("not power of 2 aligned")); \ 683 \ 684 SFXGE_BAR_LOCK(_esbp); \ 685 \ 686 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \ 687 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 688 (_offset)); \ 689 \ 690 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 691 uint32_t, (_eqp)->eq_u32[1], \ 692 uint32_t, (_eqp)->eq_u32[0]); \ 693 \ 694 SFXGE_BAR_UNLOCK(_esbp); \ 695 _NOTE(CONSTANTCONDITION) \ 696 } while (B_FALSE) 697 698 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 699 do { \ 700 _NOTE(CONSTANTCONDITION) \ 701 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 702 ("not power of 2 aligned")); \ 703 \ 704 _NOTE(CONSTANTCONDITION) \ 705 if (_lock) \ 706 SFXGE_BAR_LOCK(_esbp); \ 707 \ 708 (_eop)->eo_u64[0] = bus_space_read_stream_8( \ 709 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 710 (_offset)); \ 711 (_eop)->eo_u64[1] = bus_space_read_stream_8( \ 712 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 713 (_offset) + 8); \ 714 \ 715 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 716 uint32_t, (_eop)->eo_u32[3], \ 717 uint32_t, (_eop)->eo_u32[2], \ 718 uint32_t, (_eop)->eo_u32[1], \ 719 uint32_t, (_eop)->eo_u32[0]); \ 720 \ 721 _NOTE(CONSTANTCONDITION) \ 722 if (_lock) \ 723 SFXGE_BAR_UNLOCK(_esbp); \ 724 _NOTE(CONSTANTCONDITION) \ 725 } while (B_FALSE) 726 727 #else 728 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 729 do { \ 730 _NOTE(CONSTANTCONDITION) \ 731 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 732 ("not power of 2 aligned")); \ 733 \ 734 SFXGE_BAR_LOCK(_esbp); \ 735 \ 736 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \ 737 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 738 (_offset)); \ 739 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \ 740 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 741 (_offset) + 4); \ 742 \ 743 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 744 uint32_t, (_eqp)->eq_u32[1], \ 745 uint32_t, (_eqp)->eq_u32[0]); \ 746 \ 747 SFXGE_BAR_UNLOCK(_esbp); \ 748 _NOTE(CONSTANTCONDITION) \ 749 } while (B_FALSE) 750 751 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 752 do { \ 753 _NOTE(CONSTANTCONDITION) \ 754 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 755 ("not power of 2 aligned")); \ 756 \ 757 _NOTE(CONSTANTCONDITION) \ 758 if (_lock) \ 759 SFXGE_BAR_LOCK(_esbp); \ 760 \ 761 (_eop)->eo_u32[0] = bus_space_read_stream_4( \ 762 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 763 (_offset)); \ 764 (_eop)->eo_u32[1] = bus_space_read_stream_4( \ 765 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 766 (_offset) + 4); \ 767 (_eop)->eo_u32[2] = bus_space_read_stream_4( \ 768 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 769 (_offset) + 8); \ 770 (_eop)->eo_u32[3] = bus_space_read_stream_4( \ 771 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 772 (_offset) + 12); \ 773 \ 774 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 775 uint32_t, (_eop)->eo_u32[3], \ 776 uint32_t, (_eop)->eo_u32[2], \ 777 uint32_t, (_eop)->eo_u32[1], \ 778 uint32_t, (_eop)->eo_u32[0]); \ 779 \ 780 _NOTE(CONSTANTCONDITION) \ 781 if (_lock) \ 782 SFXGE_BAR_UNLOCK(_esbp); \ 783 _NOTE(CONSTANTCONDITION) \ 784 } while (B_FALSE) 785 #endif 786 787 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 788 do { \ 789 _NOTE(CONSTANTCONDITION) \ 790 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 791 ("not power of 2 aligned")); \ 792 \ 793 _NOTE(CONSTANTCONDITION) \ 794 if (_lock) \ 795 SFXGE_BAR_LOCK(_esbp); \ 796 \ 797 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 798 uint32_t, (_edp)->ed_u32[0]); \ 799 \ 800 /* \ 801 * Make sure that previous writes to the dword have \ 802 * been done. It should be cheaper than barrier just \ 803 * after the write below. \ 804 */ \ 805 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 806 (_offset), sizeof (efx_dword_t), \ 807 BUS_SPACE_BARRIER_WRITE); \ 808 bus_space_write_stream_4((_esbp)->esb_tag, \ 809 (_esbp)->esb_handle, \ 810 (_offset), (_edp)->ed_u32[0]); \ 811 \ 812 _NOTE(CONSTANTCONDITION) \ 813 if (_lock) \ 814 SFXGE_BAR_UNLOCK(_esbp); \ 815 _NOTE(CONSTANTCONDITION) \ 816 } while (B_FALSE) 817 818 #if defined(SFXGE_USE_BUS_SPACE_8) 819 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 820 do { \ 821 _NOTE(CONSTANTCONDITION) \ 822 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 823 ("not power of 2 aligned")); \ 824 \ 825 SFXGE_BAR_LOCK(_esbp); \ 826 \ 827 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 828 uint32_t, (_eqp)->eq_u32[1], \ 829 uint32_t, (_eqp)->eq_u32[0]); \ 830 \ 831 /* \ 832 * Make sure that previous writes to the qword have \ 833 * been done. It should be cheaper than barrier just \ 834 * after the write below. \ 835 */ \ 836 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 837 (_offset), sizeof (efx_qword_t), \ 838 BUS_SPACE_BARRIER_WRITE); \ 839 bus_space_write_stream_8((_esbp)->esb_tag, \ 840 (_esbp)->esb_handle, \ 841 (_offset), (_eqp)->eq_u64[0]); \ 842 \ 843 SFXGE_BAR_UNLOCK(_esbp); \ 844 _NOTE(CONSTANTCONDITION) \ 845 } while (B_FALSE) 846 #else 847 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 848 do { \ 849 _NOTE(CONSTANTCONDITION) \ 850 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 851 ("not power of 2 aligned")); \ 852 \ 853 SFXGE_BAR_LOCK(_esbp); \ 854 \ 855 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 856 uint32_t, (_eqp)->eq_u32[1], \ 857 uint32_t, (_eqp)->eq_u32[0]); \ 858 \ 859 /* \ 860 * Make sure that previous writes to the qword have \ 861 * been done. It should be cheaper than barrier just \ 862 * after the last write below. \ 863 */ \ 864 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 865 (_offset), sizeof (efx_qword_t), \ 866 BUS_SPACE_BARRIER_WRITE); \ 867 bus_space_write_stream_4((_esbp)->esb_tag, \ 868 (_esbp)->esb_handle, \ 869 (_offset), (_eqp)->eq_u32[0]); \ 870 /* \ 871 * It should be guaranteed that the last dword comes \ 872 * the last, so barrier entire qword to be sure that \ 873 * neither above nor below writes are reordered. \ 874 */ \ 875 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 876 (_offset), sizeof (efx_qword_t), \ 877 BUS_SPACE_BARRIER_WRITE); \ 878 bus_space_write_stream_4((_esbp)->esb_tag, \ 879 (_esbp)->esb_handle, \ 880 (_offset) + 4, (_eqp)->eq_u32[1]); \ 881 \ 882 SFXGE_BAR_UNLOCK(_esbp); \ 883 _NOTE(CONSTANTCONDITION) \ 884 } while (B_FALSE) 885 #endif 886 887 /* 888 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping 889 * (required by PIO hardware) 890 */ 891 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \ 892 do { \ 893 _NOTE(CONSTANTCONDITION) \ 894 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 895 ("not power of 2 aligned")); \ 896 \ 897 (void) (_esbp); \ 898 \ 899 /* FIXME: Perform a 64-bit write */ \ 900 KASSERT(0, ("not implemented")); \ 901 \ 902 _NOTE(CONSTANTCONDITION) \ 903 } while (B_FALSE) 904 905 #if defined(SFXGE_USE_BUS_SPACE_8) 906 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 907 do { \ 908 _NOTE(CONSTANTCONDITION) \ 909 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 910 ("not power of 2 aligned")); \ 911 \ 912 _NOTE(CONSTANTCONDITION) \ 913 if (_lock) \ 914 SFXGE_BAR_LOCK(_esbp); \ 915 \ 916 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 917 uint32_t, (_eop)->eo_u32[3], \ 918 uint32_t, (_eop)->eo_u32[2], \ 919 uint32_t, (_eop)->eo_u32[1], \ 920 uint32_t, (_eop)->eo_u32[0]); \ 921 \ 922 /* \ 923 * Make sure that previous writes to the oword have \ 924 * been done. It should be cheaper than barrier just \ 925 * after the last write below. \ 926 */ \ 927 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 928 (_offset), sizeof (efx_oword_t), \ 929 BUS_SPACE_BARRIER_WRITE); \ 930 bus_space_write_stream_8((_esbp)->esb_tag, \ 931 (_esbp)->esb_handle, \ 932 (_offset), (_eop)->eo_u64[0]); \ 933 /* \ 934 * It should be guaranteed that the last qword comes \ 935 * the last, so barrier entire oword to be sure that \ 936 * neither above nor below writes are reordered. \ 937 */ \ 938 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 939 (_offset), sizeof (efx_oword_t), \ 940 BUS_SPACE_BARRIER_WRITE); \ 941 bus_space_write_stream_8((_esbp)->esb_tag, \ 942 (_esbp)->esb_handle, \ 943 (_offset) + 8, (_eop)->eo_u64[1]); \ 944 \ 945 _NOTE(CONSTANTCONDITION) \ 946 if (_lock) \ 947 SFXGE_BAR_UNLOCK(_esbp); \ 948 _NOTE(CONSTANTCONDITION) \ 949 } while (B_FALSE) 950 951 #else 952 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 953 do { \ 954 _NOTE(CONSTANTCONDITION) \ 955 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 956 ("not power of 2 aligned")); \ 957 \ 958 _NOTE(CONSTANTCONDITION) \ 959 if (_lock) \ 960 SFXGE_BAR_LOCK(_esbp); \ 961 \ 962 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 963 uint32_t, (_eop)->eo_u32[3], \ 964 uint32_t, (_eop)->eo_u32[2], \ 965 uint32_t, (_eop)->eo_u32[1], \ 966 uint32_t, (_eop)->eo_u32[0]); \ 967 \ 968 /* \ 969 * Make sure that previous writes to the oword have \ 970 * been done. It should be cheaper than barrier just \ 971 * after the last write below. \ 972 */ \ 973 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 974 (_offset), sizeof (efx_oword_t), \ 975 BUS_SPACE_BARRIER_WRITE); \ 976 bus_space_write_stream_4((_esbp)->esb_tag, \ 977 (_esbp)->esb_handle, \ 978 (_offset), (_eop)->eo_u32[0]); \ 979 bus_space_write_stream_4((_esbp)->esb_tag, \ 980 (_esbp)->esb_handle, \ 981 (_offset) + 4, (_eop)->eo_u32[1]); \ 982 bus_space_write_stream_4((_esbp)->esb_tag, \ 983 (_esbp)->esb_handle, \ 984 (_offset) + 8, (_eop)->eo_u32[2]); \ 985 /* \ 986 * It should be guaranteed that the last dword comes \ 987 * the last, so barrier entire oword to be sure that \ 988 * neither above nor below writes are reordered. \ 989 */ \ 990 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 991 (_offset), sizeof (efx_oword_t), \ 992 BUS_SPACE_BARRIER_WRITE); \ 993 bus_space_write_stream_4((_esbp)->esb_tag, \ 994 (_esbp)->esb_handle, \ 995 (_offset) + 12, (_eop)->eo_u32[3]); \ 996 \ 997 _NOTE(CONSTANTCONDITION) \ 998 if (_lock) \ 999 SFXGE_BAR_UNLOCK(_esbp); \ 1000 _NOTE(CONSTANTCONDITION) \ 1001 } while (B_FALSE) 1002 #endif 1003 1004 /* Use the standard octo-word write for doorbell writes */ 1005 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \ 1006 do { \ 1007 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \ 1008 _NOTE(CONSTANTCONDITION) \ 1009 } while (B_FALSE) 1010 1011 /* SPIN */ 1012 1013 #define EFSYS_SPIN(_us) \ 1014 do { \ 1015 DELAY(_us); \ 1016 _NOTE(CONSTANTCONDITION) \ 1017 } while (B_FALSE) 1018 1019 #define EFSYS_SLEEP EFSYS_SPIN 1020 1021 /* BARRIERS */ 1022 1023 #define EFSYS_MEM_READ_BARRIER() rmb() 1024 #define EFSYS_PIO_WRITE_BARRIER() 1025 1026 /* DMA SYNC */ 1027 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \ 1028 do { \ 1029 bus_dmamap_sync((_esmp)->esm_tag, \ 1030 (_esmp)->esm_map, \ 1031 BUS_DMASYNC_POSTREAD); \ 1032 _NOTE(CONSTANTCONDITION) \ 1033 } while (B_FALSE) 1034 1035 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \ 1036 do { \ 1037 bus_dmamap_sync((_esmp)->esm_tag, \ 1038 (_esmp)->esm_map, \ 1039 BUS_DMASYNC_PREWRITE); \ 1040 _NOTE(CONSTANTCONDITION) \ 1041 } while (B_FALSE) 1042 1043 /* TIMESTAMP */ 1044 1045 typedef clock_t efsys_timestamp_t; 1046 1047 #define EFSYS_TIMESTAMP(_usp) \ 1048 do { \ 1049 clock_t now; \ 1050 \ 1051 now = ticks; \ 1052 *(_usp) = now * hz / 1000000; \ 1053 _NOTE(CONSTANTCONDITION) \ 1054 } while (B_FALSE) 1055 1056 /* KMEM */ 1057 1058 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 1059 do { \ 1060 (_esip) = (_esip); \ 1061 /* \ 1062 * The macro is used in non-sleepable contexts, for \ 1063 * example, holding a mutex. \ 1064 */ \ 1065 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \ 1066 _NOTE(CONSTANTCONDITION) \ 1067 } while (B_FALSE) 1068 1069 #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 1070 do { \ 1071 (void) (_esip); \ 1072 (void) (_size); \ 1073 free((_p), M_SFXGE); \ 1074 _NOTE(CONSTANTCONDITION) \ 1075 } while (B_FALSE) 1076 1077 /* LOCK */ 1078 1079 typedef struct efsys_lock_s { 1080 struct mtx lock; 1081 char lock_name[SFXGE_LOCK_NAME_MAX]; 1082 } efsys_lock_t; 1083 1084 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ 1085 do { \ 1086 efsys_lock_t *__eslp = (_eslp); \ 1087 \ 1088 snprintf((__eslp)->lock_name, \ 1089 sizeof((__eslp)->lock_name), \ 1090 "%s:%s", (_ifname), (_label)); \ 1091 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ 1092 NULL, MTX_DEF); \ 1093 } while (B_FALSE) 1094 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ 1095 mtx_destroy(&(_eslp)->lock) 1096 #define SFXGE_EFSYS_LOCK(_eslp) \ 1097 mtx_lock(&(_eslp)->lock) 1098 #define SFXGE_EFSYS_UNLOCK(_eslp) \ 1099 mtx_unlock(&(_eslp)->lock) 1100 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ 1101 mtx_assert(&(_eslp)->lock, MA_OWNED) 1102 1103 typedef int efsys_lock_state_t; 1104 1105 #define EFSYS_LOCK_MAGIC 0x000010c4 1106 1107 #define EFSYS_LOCK(_lockp, _state) \ 1108 do { \ 1109 SFXGE_EFSYS_LOCK(_lockp); \ 1110 (_state) = EFSYS_LOCK_MAGIC; \ 1111 _NOTE(CONSTANTCONDITION) \ 1112 } while (B_FALSE) 1113 1114 #define EFSYS_UNLOCK(_lockp, _state) \ 1115 do { \ 1116 if ((_state) != EFSYS_LOCK_MAGIC) \ 1117 KASSERT(B_FALSE, ("not locked")); \ 1118 SFXGE_EFSYS_UNLOCK(_lockp); \ 1119 _NOTE(CONSTANTCONDITION) \ 1120 } while (B_FALSE) 1121 1122 /* STAT */ 1123 1124 typedef uint64_t efsys_stat_t; 1125 1126 #define EFSYS_STAT_INCR(_knp, _delta) \ 1127 do { \ 1128 *(_knp) += (_delta); \ 1129 _NOTE(CONSTANTCONDITION) \ 1130 } while (B_FALSE) 1131 1132 #define EFSYS_STAT_DECR(_knp, _delta) \ 1133 do { \ 1134 *(_knp) -= (_delta); \ 1135 _NOTE(CONSTANTCONDITION) \ 1136 } while (B_FALSE) 1137 1138 #define EFSYS_STAT_SET(_knp, _val) \ 1139 do { \ 1140 *(_knp) = (_val); \ 1141 _NOTE(CONSTANTCONDITION) \ 1142 } while (B_FALSE) 1143 1144 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 1145 do { \ 1146 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 1147 _NOTE(CONSTANTCONDITION) \ 1148 } while (B_FALSE) 1149 1150 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 1151 do { \ 1152 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 1153 _NOTE(CONSTANTCONDITION) \ 1154 } while (B_FALSE) 1155 1156 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 1157 do { \ 1158 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 1159 _NOTE(CONSTANTCONDITION) \ 1160 } while (B_FALSE) 1161 1162 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 1163 do { \ 1164 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 1165 _NOTE(CONSTANTCONDITION) \ 1166 } while (B_FALSE) 1167 1168 /* ERR */ 1169 1170 extern void sfxge_err(efsys_identifier_t *, unsigned int, 1171 uint32_t, uint32_t); 1172 1173 #if EFSYS_OPT_DECODE_INTR_FATAL 1174 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 1175 do { \ 1176 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 1177 _NOTE(CONSTANTCONDITION) \ 1178 } while (B_FALSE) 1179 #endif 1180 1181 /* ASSERT */ 1182 1183 #define EFSYS_ASSERT(_exp) do { \ 1184 if (!(_exp)) \ 1185 panic("%s", #_exp); \ 1186 } while (0) 1187 1188 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 1189 const _t __x = (_t)(_x); \ 1190 const _t __y = (_t)(_y); \ 1191 if (!(__x _op __y)) \ 1192 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 1193 } while(0) 1194 1195 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 1196 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 1197 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 1198 1199 /* ROTATE */ 1200 1201 #define EFSYS_HAS_ROTL_DWORD 0 1202 1203 #ifdef __cplusplus 1204 } 1205 #endif 1206 1207 #endif /* _SYS_EFSYS_H */ 1208