1*d4a07e70Sfengbojiang /*- 2*d4a07e70Sfengbojiang * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3*d4a07e70Sfengbojiang * 4*d4a07e70Sfengbojiang * Copyright (c) 2018, Matthew Macy <[email protected]> 5*d4a07e70Sfengbojiang * 6*d4a07e70Sfengbojiang * Redistribution and use in source and binary forms, with or without 7*d4a07e70Sfengbojiang * modification, are permitted provided that the following conditions 8*d4a07e70Sfengbojiang * are met: 9*d4a07e70Sfengbojiang * 1. Redistributions of source code must retain the above copyright 10*d4a07e70Sfengbojiang * notice, this list of conditions and the following disclaimer. 11*d4a07e70Sfengbojiang * 2. Redistributions in binary form must reproduce the above copyright 12*d4a07e70Sfengbojiang * notice, this list of conditions and the following disclaimer in the 13*d4a07e70Sfengbojiang * documentation and/or other materials provided with the distribution. 14*d4a07e70Sfengbojiang * 15*d4a07e70Sfengbojiang * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16*d4a07e70Sfengbojiang * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17*d4a07e70Sfengbojiang * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18*d4a07e70Sfengbojiang * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19*d4a07e70Sfengbojiang * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20*d4a07e70Sfengbojiang * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21*d4a07e70Sfengbojiang * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22*d4a07e70Sfengbojiang * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23*d4a07e70Sfengbojiang * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24*d4a07e70Sfengbojiang * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25*d4a07e70Sfengbojiang * SUCH DAMAGE. 26*d4a07e70Sfengbojiang * 27*d4a07e70Sfengbojiang * $FreeBSD$ 28*d4a07e70Sfengbojiang */ 29*d4a07e70Sfengbojiang 30*d4a07e70Sfengbojiang #ifndef _SYS_EPOCH_H_ 31*d4a07e70Sfengbojiang #define _SYS_EPOCH_H_ 32*d4a07e70Sfengbojiang 33*d4a07e70Sfengbojiang struct epoch_context { 34*d4a07e70Sfengbojiang void *data[2]; 35*d4a07e70Sfengbojiang } __attribute__((__aligned__(sizeof(void *)))); 36*d4a07e70Sfengbojiang 37*d4a07e70Sfengbojiang typedef struct epoch_context *epoch_context_t; 38*d4a07e70Sfengbojiang typedef void epoch_callback_t(epoch_context_t); 39*d4a07e70Sfengbojiang 40*d4a07e70Sfengbojiang #ifdef _KERNEL 41*d4a07e70Sfengbojiang #include <sys/lock.h> 42*d4a07e70Sfengbojiang #include <sys/pcpu.h> 43*d4a07e70Sfengbojiang #include <ck_epoch.h> 44*d4a07e70Sfengbojiang 45*d4a07e70Sfengbojiang struct epoch; 46*d4a07e70Sfengbojiang typedef struct epoch *epoch_t; 47*d4a07e70Sfengbojiang 48*d4a07e70Sfengbojiang #define EPOCH_PREEMPT 0x1 49*d4a07e70Sfengbojiang #define EPOCH_LOCKED 0x2 50*d4a07e70Sfengbojiang 51*d4a07e70Sfengbojiang extern epoch_t global_epoch; 52*d4a07e70Sfengbojiang extern epoch_t global_epoch_preempt; 53*d4a07e70Sfengbojiang 54*d4a07e70Sfengbojiang struct epoch_tracker { 55*d4a07e70Sfengbojiang TAILQ_ENTRY(epoch_tracker) et_link; 56*d4a07e70Sfengbojiang struct thread *et_td; 57*d4a07e70Sfengbojiang ck_epoch_section_t et_section; 58*d4a07e70Sfengbojiang #ifdef EPOCH_TRACE 59*d4a07e70Sfengbojiang struct epoch *et_epoch; 60*d4a07e70Sfengbojiang SLIST_ENTRY(epoch_tracker) et_tlink; 61*d4a07e70Sfengbojiang const char *et_file; 62*d4a07e70Sfengbojiang int et_line; 63*d4a07e70Sfengbojiang #endif 64*d4a07e70Sfengbojiang } __attribute__((__aligned__(sizeof(void *)))); 65*d4a07e70Sfengbojiang typedef struct epoch_tracker *epoch_tracker_t; 66*d4a07e70Sfengbojiang 67*d4a07e70Sfengbojiang epoch_t epoch_alloc(const char *name, int flags); 68*d4a07e70Sfengbojiang void epoch_free(epoch_t epoch); 69*d4a07e70Sfengbojiang void epoch_wait(epoch_t epoch); 70*d4a07e70Sfengbojiang void epoch_wait_preempt(epoch_t epoch); 71*d4a07e70Sfengbojiang void epoch_drain_callbacks(epoch_t epoch); 72*d4a07e70Sfengbojiang void epoch_call(epoch_t epoch, epoch_callback_t cb, epoch_context_t ctx); 73*d4a07e70Sfengbojiang int in_epoch(epoch_t epoch); 74*d4a07e70Sfengbojiang int in_epoch_verbose(epoch_t epoch, int dump_onfail); 75*d4a07e70Sfengbojiang DPCPU_DECLARE(int, epoch_cb_count); 76*d4a07e70Sfengbojiang DPCPU_DECLARE(struct grouptask, epoch_cb_task); 77*d4a07e70Sfengbojiang 78*d4a07e70Sfengbojiang #ifdef EPOCH_TRACE 79*d4a07e70Sfengbojiang #define EPOCH_FILE_LINE , const char *file, int line 80*d4a07e70Sfengbojiang #else 81*d4a07e70Sfengbojiang #define EPOCH_FILE_LINE 82*d4a07e70Sfengbojiang #endif 83*d4a07e70Sfengbojiang 84*d4a07e70Sfengbojiang void _epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et EPOCH_FILE_LINE); 85*d4a07e70Sfengbojiang void _epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et EPOCH_FILE_LINE); 86*d4a07e70Sfengbojiang #ifdef EPOCH_TRACE 87*d4a07e70Sfengbojiang void epoch_trace_list(struct thread *); 88*d4a07e70Sfengbojiang #define epoch_enter_preempt(epoch, et) _epoch_enter_preempt(epoch, et, __FILE__, __LINE__) 89*d4a07e70Sfengbojiang #define epoch_exit_preempt(epoch, et) _epoch_exit_preempt(epoch, et, __FILE__, __LINE__) 90*d4a07e70Sfengbojiang #else 91*d4a07e70Sfengbojiang #define epoch_enter_preempt(epoch, et) _epoch_enter_preempt(epoch, et) 92*d4a07e70Sfengbojiang #define epoch_exit_preempt(epoch, et) _epoch_exit_preempt(epoch, et) 93*d4a07e70Sfengbojiang #endif 94*d4a07e70Sfengbojiang void epoch_enter(epoch_t epoch); 95*d4a07e70Sfengbojiang void epoch_exit(epoch_t epoch); 96*d4a07e70Sfengbojiang 97*d4a07e70Sfengbojiang /* 98*d4a07e70Sfengbojiang * Globally recognized epochs in the FreeBSD kernel. 99*d4a07e70Sfengbojiang */ 100*d4a07e70Sfengbojiang /* Network preemptible epoch, declared in sys/net/if.c. */ 101*d4a07e70Sfengbojiang extern epoch_t net_epoch_preempt; 102*d4a07e70Sfengbojiang #define NET_EPOCH_ENTER(et) epoch_enter_preempt(net_epoch_preempt, &(et)) 103*d4a07e70Sfengbojiang #define NET_EPOCH_EXIT(et) epoch_exit_preempt(net_epoch_preempt, &(et)) 104*d4a07e70Sfengbojiang #define NET_EPOCH_WAIT() epoch_wait_preempt(net_epoch_preempt) 105*d4a07e70Sfengbojiang #define NET_EPOCH_CALL(f, c) epoch_call(net_epoch_preempt, (f), (c)) 106*d4a07e70Sfengbojiang #define NET_EPOCH_ASSERT() MPASS(in_epoch(net_epoch_preempt)) 107*d4a07e70Sfengbojiang #define NET_TASK_INIT(t, p, f, c) TASK_INIT_FLAGS(t, p, f, c, TASK_NETWORK) 108*d4a07e70Sfengbojiang #define NET_GROUPTASK_INIT(gtask, prio, func, ctx) \ 109*d4a07e70Sfengbojiang GTASK_INIT(&(gtask)->gt_task, TASK_NETWORK, (prio), (func), (ctx)) 110*d4a07e70Sfengbojiang 111*d4a07e70Sfengbojiang #endif /* _KERNEL */ 112*d4a07e70Sfengbojiang #endif /* _SYS_EPOCH_H_ */ 113