1 /*
2 * Copyright (c) 2021, 2023, 2024 Apple Inc. All rights reserved.
3 * @APPLE_LICENSE_HEADER_START@
4 *
5 * This file contains Original Code and/or Modifications of Original Code
6 * as defined in and that are subject to the Apple Public Source License
7 * Version 2.0 (the 'License'). You may not use this file except in
8 * compliance with the License. Please obtain a copy of the License at
9 * http://www.opensource.apple.com/apsl/ and read it before using this
10 * file.
11 *
12 * The Original Code and all software distributed under the License are
13 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
17 * Please see the License for the specific language governing rights and
18 * limitations under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22
23 /*
24 * LOCKING STRATEGY
25 *
26 * The struct socket's so_flow_db field (struct soflow_db and its hash entries
27 * struct soflow_hash_entry) is protected by the socket lock. This covers all the
28 * socket paths that calls soflow_get_flow() as well as the garbage collection.
29 * For the socket detach path, soflow_detach() cannot assume the socket lock is
30 * held. Thus, reference counts are added to both struct soflow_db and struct
31 * soflow_hash_entry to avoid access after freed issues.
32 *
33 * The global list, soflow_entry_head, keeps track of all struct soflow_hash_entry
34 * entries which is used by garbage collection when detecting idle entries. This list
35 * is protected by the global lock soflow_lck_rw.
36 *
37 */
38
39 #include <sys/types.h>
40 #include <sys/kern_control.h>
41 #include <sys/queue.h>
42 #include <sys/domain.h>
43 #include <sys/protosw.h>
44 #include <sys/syslog.h>
45 #include <sys/systm.h>
46 #include <sys/sysproto.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
49
50 #include <kern/sched_prim.h>
51 #include <kern/locks.h>
52 #include <kern/zalloc.h>
53 #include <kern/debug.h>
54 #include <net/ntstat.h>
55 #include <netinet6/in6_var.h>
56
57 #define _IP_VHL
58 #include <netinet/ip.h>
59 #include <netinet/in_pcb.h>
60 #include <netinet/udp.h>
61 #include <netinet/udp_var.h>
62
63 #include <string.h>
64 #include <libkern/libkern.h>
65 #include <kern/socket_flows.h>
66 #include <net/sockaddr_utils.h>
67
68 extern struct inpcbinfo ripcbinfo;
69
70 /*
71 * Per-Socket Flow Management
72 */
73
74 static int soflow_log_level = LOG_ERR;
75 static int soflow_log_port = 0;
76 static int soflow_log_pid = 0;
77 static int soflow_log_proto = 0;
78 static int soflow_nstat_disable = 0;
79 static int soflow_disable = 0;
80 static long soflow_attached_count = 0;
81 static long soflow_attached_high_water_mark = 0;
82 static os_log_t soflow_log_handle = NULL;
83
84 /*
85 * Sysctls for debug logs control
86 */
87 SYSCTL_NODE(_net, OID_AUTO, soflow, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "soflow");
88
89 SYSCTL_INT(_net_soflow, OID_AUTO, log_level, CTLFLAG_RW | CTLFLAG_LOCKED,
90 &soflow_log_level, 0, "");
91
92 SYSCTL_INT(_net_soflow, OID_AUTO, log_port, CTLFLAG_RW | CTLFLAG_LOCKED,
93 &soflow_log_port, 0, "");
94
95 SYSCTL_INT(_net_soflow, OID_AUTO, log_pid, CTLFLAG_RW | CTLFLAG_LOCKED,
96 &soflow_log_pid, 0, "");
97
98 SYSCTL_INT(_net_soflow, OID_AUTO, log_proto, CTLFLAG_RW | CTLFLAG_LOCKED,
99 &soflow_log_proto, 0, "");
100
101 SYSCTL_INT(_net_soflow, OID_AUTO, nstat_disable, CTLFLAG_RW | CTLFLAG_LOCKED,
102 &soflow_nstat_disable, 0, "");
103
104 SYSCTL_INT(_net_soflow, OID_AUTO, disable, CTLFLAG_RW | CTLFLAG_LOCKED,
105 &soflow_disable, 0, "");
106
107 SYSCTL_LONG(_net_soflow, OID_AUTO, count, CTLFLAG_LOCKED | CTLFLAG_RD, &soflow_attached_count, "");
108 SYSCTL_LONG(_net_soflow, OID_AUTO, high_water_mark, CTLFLAG_LOCKED | CTLFLAG_RD, &soflow_attached_high_water_mark, "");
109
110 #define SOFLOW_LOG(level, so, debug, fmt, ...) \
111 do { \
112 if (soflow_log_level >= level && debug && soflow_log_handle) { \
113 if (level == LOG_ERR) { \
114 os_log_error(soflow_log_handle, "SOFLOW - %s:%d <pid %d so %llx> " fmt "\n", __FUNCTION__, __LINE__, \
115 so ? SOFLOW_SOCKET_PID(so) : 0, so ? (uint64_t)VM_KERNEL_ADDRPERM(so) : 0, ##__VA_ARGS__); \
116 } else { \
117 os_log(soflow_log_handle, "SOFLOW - %s:%d <pid %d so %llx> " fmt "\n", __FUNCTION__, __LINE__, \
118 so ? SOFLOW_SOCKET_PID(so) : 0, so ? (uint64_t)VM_KERNEL_ADDRPERM(so) : 0, ##__VA_ARGS__); \
119 } \
120 } \
121 } while (0)
122
123 #define SOFLOW_ENTRY_LOG(level, so, entry, debug, msg) \
124 do { \
125 if (soflow_log_level >= level && entry && debug) { \
126 soflow_entry_log(level, so, entry, msg); \
127 } \
128 } while (0)
129
130 #define SOFLOW_HASH(laddr, faddr, lport, fport) ((faddr) ^ ((laddr) >> 16) ^ (fport) ^ (lport))
131
132 #define SOFLOW_IS_UDP(so) (so && SOCK_CHECK_TYPE(so, SOCK_DGRAM) && SOCK_CHECK_PROTO(so, IPPROTO_UDP))
133 #define SOFLOW_GET_SO_PROTO(so) (so ? SOCK_PROTO(so) : IPPROTO_MAX)
134
135 #define SOFLOW_SOCKET_PID(so) ((so->so_flags & SOF_DELEGATED) ? so->e_pid : so->last_pid)
136
137 #define SOFLOW_ENABLE_DEBUG(so, entry) \
138 ((soflow_log_port == 0 || !entry || soflow_log_port == ntohs(entry->soflow_lport) || soflow_log_port == ntohs(entry->soflow_fport)) && \
139 (soflow_log_pid == 0 || !so || soflow_log_pid == SOFLOW_SOCKET_PID(so)) && \
140 (soflow_log_proto == 0 || !so || soflow_log_proto == SOFLOW_GET_SO_PROTO(so)))
141
142 os_refgrp_decl(static, soflow_refgrp, "soflow_ref_group", NULL);
143
144 #define SOFLOW_ENTRY_FREE(entry) \
145 if (entry && (os_ref_release(&entry->soflow_ref_count) == 0)) { \
146 soflow_entry_free(entry); \
147 }
148
149 #define SOFLOW_DB_FREE(db) \
150 if (db && (os_ref_release(&db->soflow_db_ref_count) == 0)) { \
151 soflow_db_free(db); \
152 }
153
154 static int soflow_initialized = 0;
155
156 TAILQ_HEAD(soflow_entry_head, soflow_hash_entry) soflow_entry_head;
157 static LCK_GRP_DECLARE(soflow_lck_grp, "Socket Flow");
158 static LCK_RW_DECLARE(soflow_lck_rw, &soflow_lck_grp);
159
160 #define SOFLOW_LOCK_EXCLUSIVE lck_rw_lock_exclusive(&soflow_lck_rw)
161 #define SOFLOW_UNLOCK_EXCLUSIVE lck_rw_unlock_exclusive(&soflow_lck_rw)
162 #define SOFLOW_LOCK_SHARED lck_rw_lock_shared(&soflow_lck_rw)
163 #define SOFLOW_UNLOCK_SHARED lck_rw_unlock_shared(&soflow_lck_rw)
164
165 /*
166 * Flow Garbage Collection:
167 */
168 static struct thread *soflow_gc_thread;
169 static soflow_feat_gc_needed_func soflow_feat_gc_needed_func_ptr = NULL;
170 static soflow_feat_gc_perform_func soflow_feat_gc_perform_func_ptr = NULL;
171
172 #define SOFLOW_GC_IDLE_TO 30 // Flow Idle Timeout in seconds
173 #define SOFLOW_GC_MAX_COUNT 100 // Max sockets to be handled per run
174 #define SOFLOW_GC_RUN_INTERVAL_NSEC (10 * NSEC_PER_SEC) // GC wakes up every 10 seconds
175
176 /*
177 * Feature Context Handling:
178 */
179 static soflow_feat_detach_entry_func soflow_feat_detach_entry_func_ptr = NULL;
180 static soflow_feat_detach_db_func soflow_feat_detach_db_func_ptr = NULL;
181
182 static void soflow_gc_thread_func(void *v, wait_result_t w);
183 static void soflow_gc_expire(void *v, wait_result_t w);
184 static boolean_t soflow_entry_local_address_needs_update(struct soflow_hash_entry *);
185 static boolean_t soflow_entry_local_port_needs_update(struct socket *, struct soflow_hash_entry *);
186
187 static void
soflow_init(void)188 soflow_init(void)
189 {
190 if (soflow_initialized) {
191 return;
192 }
193 soflow_initialized = 1;
194
195 if (soflow_log_handle == NULL) {
196 soflow_log_handle = os_log_create("com.apple.xnu.net.soflow", "soflow");
197 }
198
199 TAILQ_INIT(&soflow_entry_head);
200
201 // Spawn thread for gargage collection
202 if (kernel_thread_start(soflow_gc_thread_func, NULL,
203 &soflow_gc_thread) != KERN_SUCCESS) {
204 panic_plain("%s: Can't create SOFLOW GC thread", __func__);
205 /* NOTREACHED */
206 }
207 /* this must not fail */
208 VERIFY(soflow_gc_thread != NULL);
209 }
210
211 static void
soflow_entry_log(int level,struct socket * so,struct soflow_hash_entry * entry,const char * msg)212 soflow_entry_log(int level, struct socket *so, struct soflow_hash_entry *entry, const char* msg)
213 {
214 #pragma unused(level, msg)
215 char local[MAX_IPv6_STR_LEN + 6] = { 0 };
216 char remote[MAX_IPv6_STR_LEN + 6] = { 0 };
217 const void *addr;
218
219 // No sock or not UDP, no-op
220 if (entry == NULL) {
221 return;
222 }
223
224 switch (entry->soflow_family) {
225 case AF_INET6:
226 addr = &entry->soflow_laddr.addr6;
227 inet_ntop(AF_INET6, addr, local, sizeof(local));
228 addr = &entry->soflow_faddr.addr6;
229 inet_ntop(AF_INET6, addr, remote, sizeof(local));
230 break;
231 case AF_INET:
232 addr = &entry->soflow_laddr.addr46.ia46_addr4.s_addr;
233 inet_ntop(AF_INET, addr, local, sizeof(local));
234 addr = &entry->soflow_faddr.addr46.ia46_addr4.s_addr;
235 inet_ntop(AF_INET, addr, remote, sizeof(local));
236 break;
237 default:
238 return;
239 }
240
241 SOFLOW_LOG(level, so, entry->soflow_debug, "<%s>: %s <%s(%d) entry %p, featureID %llu, filter_ctl 0x%x> outifp %d lport %d fport %d laddr %s faddr %s hash %X "
242 "<rx p %llu b %llu, tx p %llu b %llu>",
243 msg, entry->soflow_outgoing ? "OUT" : "IN ",
244 SOFLOW_IS_UDP(so) ? "UDP" : "proto", SOFLOW_GET_SO_PROTO(so),
245 entry, entry->soflow_feat_ctxt_id,
246 entry->soflow_filter_control_unit,
247 entry->soflow_outifindex,
248 ntohs(entry->soflow_lport), ntohs(entry->soflow_fport), local, remote,
249 entry->soflow_flowhash,
250 entry->soflow_rxpackets, entry->soflow_rxbytes, entry->soflow_txpackets, entry->soflow_txbytes);
251 }
252
253 bool
soflow_fill_hash_entry_from_address(struct soflow_hash_entry * entry,bool isLocal,struct sockaddr * addr,bool islocalUpdate)254 soflow_fill_hash_entry_from_address(struct soflow_hash_entry *entry, bool isLocal, struct sockaddr *addr, bool islocalUpdate)
255 {
256 struct sockaddr_in *sin = NULL;
257 struct sockaddr_in6 *sin6 = NULL;
258
259 if (entry == NULL || addr == NULL) {
260 return FALSE;
261 }
262
263 switch (addr->sa_family) {
264 case AF_INET:
265 sin = satosin(addr);
266 if (sin->sin_len < sizeof(*sin)) {
267 return FALSE;
268 }
269 if (isLocal == TRUE) {
270 if (sin->sin_port != 0) {
271 entry->soflow_lport = sin->sin_port;
272 if (islocalUpdate) {
273 entry->soflow_lport_updated = TRUE;
274 }
275 }
276 if (sin->sin_addr.s_addr != INADDR_ANY) {
277 entry->soflow_laddr.addr46.ia46_addr4.s_addr = sin->sin_addr.s_addr;
278 if (islocalUpdate) {
279 entry->soflow_laddr_updated = TRUE;
280 }
281 }
282 } else {
283 if (sin->sin_port != 0) {
284 entry->soflow_fport = sin->sin_port;
285 }
286 if (sin->sin_addr.s_addr != INADDR_ANY) {
287 entry->soflow_faddr.addr46.ia46_addr4.s_addr = sin->sin_addr.s_addr;
288 }
289 }
290 entry->soflow_family = AF_INET;
291 return TRUE;
292 case AF_INET6:
293 sin6 = satosin6(addr);
294 if (sin6->sin6_len < sizeof(*sin6)) {
295 return FALSE;
296 }
297 if (isLocal == TRUE) {
298 if (sin6->sin6_port != 0) {
299 entry->soflow_lport = sin6->sin6_port;
300 if (islocalUpdate) {
301 entry->soflow_lport_updated = TRUE;
302 }
303 }
304 if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
305 entry->soflow_laddr.addr6 = sin6->sin6_addr;
306 entry->soflow_laddr6_ifscope = sin6->sin6_scope_id;
307 in6_verify_ifscope(&sin6->sin6_addr, sin6->sin6_scope_id);
308 if (islocalUpdate) {
309 entry->soflow_laddr_updated = TRUE;
310 }
311 }
312 } else {
313 if (sin6->sin6_port != 0) {
314 entry->soflow_fport = sin6->sin6_port;
315 }
316 if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
317 entry->soflow_faddr.addr6 = sin6->sin6_addr;
318 entry->soflow_faddr6_ifscope = sin6->sin6_scope_id;
319 in6_verify_ifscope(&sin6->sin6_addr, sin6->sin6_scope_id);
320 }
321 }
322 entry->soflow_family = AF_INET6;
323 return TRUE;
324 default:
325 return FALSE;
326 }
327 }
328
329 bool
soflow_fill_hash_entry_from_inp(struct soflow_hash_entry * entry,bool isLocal,struct inpcb * inp,bool islocalUpdate)330 soflow_fill_hash_entry_from_inp(struct soflow_hash_entry *entry, bool isLocal, struct inpcb *inp, bool islocalUpdate)
331 {
332 if (entry == NULL || inp == NULL) {
333 return FALSE;
334 }
335
336 if (inp->inp_vflag & INP_IPV6) {
337 if (isLocal == TRUE) {
338 if (inp->inp_lport) {
339 entry->soflow_lport = inp->inp_lport;
340 if (islocalUpdate) {
341 entry->soflow_lport_updated = TRUE;
342 }
343 }
344 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
345 entry->soflow_laddr.addr6 = inp->in6p_laddr;
346 entry->soflow_laddr6_ifscope = inp->inp_lifscope;
347 in6_verify_ifscope(&entry->soflow_laddr.addr6, inp->inp_lifscope);
348 if (islocalUpdate) {
349 entry->soflow_laddr_updated = TRUE;
350 }
351 }
352 } else {
353 if (inp->inp_fport) {
354 entry->soflow_fport = inp->inp_fport;
355 }
356 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
357 entry->soflow_faddr.addr6 = inp->in6p_faddr;
358 entry->soflow_faddr6_ifscope = inp->inp_fifscope;
359 in6_verify_ifscope(&entry->soflow_faddr.addr6, inp->inp_fifscope);
360 }
361 }
362 entry->soflow_family = AF_INET6;
363 return TRUE;
364 } else if (inp->inp_vflag & INP_IPV4) {
365 if (isLocal == TRUE) {
366 if (inp->inp_lport) {
367 entry->soflow_lport = inp->inp_lport;
368 if (islocalUpdate) {
369 entry->soflow_lport_updated = TRUE;
370 }
371 }
372 if (inp->inp_laddr.s_addr) {
373 entry->soflow_laddr.addr46.ia46_addr4.s_addr = inp->inp_laddr.s_addr;
374 if (islocalUpdate) {
375 entry->soflow_laddr_updated = TRUE;
376 }
377 }
378 } else {
379 if (inp->inp_fport) {
380 entry->soflow_fport = inp->inp_fport;
381 }
382 if (inp->inp_faddr.s_addr) {
383 entry->soflow_faddr.addr46.ia46_addr4.s_addr = inp->inp_faddr.s_addr;
384 }
385 }
386 entry->soflow_family = AF_INET;
387 return TRUE;
388 }
389 return FALSE;
390 }
391
392 static errno_t
soflow_db_init(struct socket * so)393 soflow_db_init(struct socket *so)
394 {
395 errno_t error = 0;
396 struct soflow_db * __single db = NULL;
397 struct soflow_hash_entry *hash_entry = NULL;
398
399 db = kalloc_type(struct soflow_db, Z_WAITOK | Z_ZERO | Z_NOFAIL);
400 db->soflow_db_so = so;
401 void * __single hash = hashinit(SOFLOW_HASH_SIZE, M_CFIL, &db->soflow_db_hashmask);
402 db->soflow_db_hashbase = __unsafe_forge_bidi_indexable(struct soflow_hash_head *, hash, SOFLOW_HASH_SIZE * sizeof(void*));
403 if (db->soflow_db_hashbase == NULL) {
404 kfree_type(struct soflow_db, db);
405 error = ENOMEM;
406 goto done;
407 }
408 db->soflow_db_debug = SOFLOW_ENABLE_DEBUG(so, hash_entry);
409 os_ref_init(&db->soflow_db_ref_count, &soflow_refgrp);
410 so->so_flow_db = db;
411 done:
412 return error;
413 }
414
415 static void
soflow_entry_free(struct soflow_hash_entry * hash_entry)416 soflow_entry_free(struct soflow_hash_entry *hash_entry)
417 {
418 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
419
420 if (hash_entry == NULL) {
421 return;
422 }
423
424 SOFLOW_ENTRY_LOG(LOG_INFO, so, hash_entry, hash_entry->soflow_debug, "Free entry");
425 kfree_type(struct soflow_hash_entry, hash_entry);
426 }
427
428 static void
soflow_db_remove_entry(struct soflow_db * db,struct soflow_hash_entry * hash_entry)429 soflow_db_remove_entry(struct soflow_db *db, struct soflow_hash_entry *hash_entry)
430 {
431 if (hash_entry == NULL) {
432 return;
433 }
434 if (db == NULL || db->soflow_db_count == 0) {
435 return;
436 }
437
438 #if defined(NSTAT_EXTENSION_FILTER_DOMAIN_INFO)
439 if (hash_entry->soflow_nstat_context != NULL) {
440 SOFLOW_LOG(LOG_INFO, db->soflow_db_so, hash_entry->soflow_debug, "<Close nstat> - context %lX", (unsigned long)hash_entry->soflow_nstat_context);
441 nstat_provider_stats_close(hash_entry->soflow_nstat_context);
442 hash_entry->soflow_nstat_context = NULL;
443 SOFLOW_ENTRY_FREE(hash_entry);
444 }
445 #endif
446
447 db->soflow_db_count--;
448 if (db->soflow_db_only_entry == hash_entry) {
449 db->soflow_db_only_entry = NULL;
450 }
451 LIST_REMOVE(hash_entry, soflow_entry_link);
452
453 // Feature context present, give feature a chance to detach and clean up
454 if (hash_entry->soflow_feat_ctxt != NULL && soflow_feat_detach_entry_func_ptr != NULL) {
455 soflow_feat_detach_entry_func_ptr(db->soflow_db_so, hash_entry);
456 hash_entry->soflow_feat_ctxt = NULL;
457 hash_entry->soflow_feat_ctxt_id = 0;
458 }
459
460 hash_entry->soflow_db = NULL;
461
462 SOFLOW_LOCK_EXCLUSIVE;
463 if (soflow_initialized) {
464 TAILQ_REMOVE(&soflow_entry_head, hash_entry, soflow_entry_list_link);
465 soflow_attached_count--;
466 }
467 SOFLOW_UNLOCK_EXCLUSIVE;
468
469 SOFLOW_ENTRY_FREE(hash_entry);
470 }
471
472 static void
soflow_db_free(struct soflow_db * db)473 soflow_db_free(struct soflow_db *db)
474 {
475 struct soflow_hash_entry *entry = NULL;
476 struct soflow_hash_entry *temp_entry = NULL;
477 struct soflow_hash_head *flowhash = NULL;
478
479 if (db == NULL) {
480 return;
481 }
482
483 SOFLOW_LOG(LOG_INFO, db->soflow_db_so, db->soflow_db_debug, "<db %p> freeing db (count == %d)", db, db->soflow_db_count);
484
485 for (int i = 0; i < SOFLOW_HASH_SIZE; i++) {
486 flowhash = &db->soflow_db_hashbase[i];
487 LIST_FOREACH_SAFE(entry, flowhash, soflow_entry_link, temp_entry) {
488 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Remove entry");
489 soflow_db_remove_entry(db, entry);
490 }
491 }
492
493 if (soflow_feat_detach_db_func_ptr != NULL) {
494 soflow_feat_detach_db_func_ptr(db->soflow_db_so, db);
495 }
496
497 // Make sure all entries are cleaned up!
498 VERIFY(db->soflow_db_count == 0);
499 hashdestroy(db->soflow_db_hashbase, M_CFIL, db->soflow_db_hashmask);
500 kfree_type(struct soflow_db, db);
501 }
502
503 void
soflow_detach(struct socket * so)504 soflow_detach(struct socket *so)
505 {
506 if (so == NULL || so->so_flow_db == NULL) {
507 return;
508 }
509 SOFLOW_DB_FREE(so->so_flow_db);
510 so->so_flow_db = NULL;
511 }
512
513 static boolean_t
soflow_match_entries_v4(struct soflow_hash_entry * entry1,struct soflow_hash_entry * entry2,boolean_t remoteOnly)514 soflow_match_entries_v4(struct soflow_hash_entry *entry1, struct soflow_hash_entry *entry2, boolean_t remoteOnly)
515 {
516 if (entry1 == NULL || entry2 == NULL) {
517 return false;
518 }
519
520 // Ignore local match if remoteOnly or if local has been updated since entry added
521 boolean_t lport_matched = (remoteOnly || entry1->soflow_lport_updated || entry1->soflow_lport == entry2->soflow_lport);
522 boolean_t laddr_matched = (remoteOnly || entry1->soflow_laddr_updated ||
523 entry1->soflow_laddr.addr46.ia46_addr4.s_addr == entry2->soflow_laddr.addr46.ia46_addr4.s_addr);
524
525 // Entries match if local and remote ports and addresses all matched
526 return lport_matched && entry1->soflow_fport == entry2->soflow_fport &&
527 laddr_matched && entry1->soflow_faddr.addr46.ia46_addr4.s_addr == entry2->soflow_faddr.addr46.ia46_addr4.s_addr;
528 }
529
530 static boolean_t
soflow_match_entries_v6(struct soflow_hash_entry * entry1,struct soflow_hash_entry * entry2,boolean_t remoteOnly)531 soflow_match_entries_v6(struct soflow_hash_entry *entry1, struct soflow_hash_entry *entry2, boolean_t remoteOnly)
532 {
533 if (entry1 == NULL || entry2 == NULL) {
534 return false;
535 }
536
537 // Ignore local match if remoteOnly or if local has been updated since entry added
538 boolean_t lport_matched = (remoteOnly || entry1->soflow_lport_updated || entry1->soflow_lport == entry2->soflow_lport);
539 boolean_t laddr_matched = (remoteOnly || entry1->soflow_laddr_updated ||
540 in6_are_addr_equal_scoped(&entry1->soflow_laddr.addr6, &entry2->soflow_laddr.addr6, entry1->soflow_laddr6_ifscope, entry2->soflow_laddr6_ifscope));
541
542 // Entries match if local and remote ports and addresses all matched
543 return lport_matched && entry1->soflow_fport == entry2->soflow_fport &&
544 laddr_matched && in6_are_addr_equal_scoped(&entry1->soflow_faddr.addr6, &entry2->soflow_faddr.addr6, entry1->soflow_faddr6_ifscope, entry2->soflow_faddr6_ifscope);
545 }
546
547 static struct soflow_hash_entry *
soflow_db_lookup_entry_internal(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote,boolean_t remoteOnly,boolean_t withLocalPort)548 soflow_db_lookup_entry_internal(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote, boolean_t remoteOnly, boolean_t withLocalPort)
549 {
550 struct soflow_hash_entry matchentry = { };
551 struct soflow_hash_entry *nextentry = NULL;
552 struct inpcb *inp = sotoinpcb(db->soflow_db_so);
553 u_int32_t hashkey_faddr = 0, hashkey_laddr = 0;
554 u_int16_t hashkey_fport = 0, hashkey_lport = 0;
555 int inp_hash_element = 0;
556 struct soflow_hash_head *flowhash = NULL;
557
558 if (inp == NULL || db == NULL) {
559 return NULL;
560 }
561
562 if (local != NULL) {
563 soflow_fill_hash_entry_from_address(&matchentry, TRUE, local, FALSE);
564 } else {
565 soflow_fill_hash_entry_from_inp(&matchentry, TRUE, inp, FALSE);
566 }
567 if (remote != NULL) {
568 soflow_fill_hash_entry_from_address(&matchentry, FALSE, remote, FALSE);
569 } else {
570 soflow_fill_hash_entry_from_inp(&matchentry, FALSE, inp, FALSE);
571 }
572 matchentry.soflow_debug = SOFLOW_ENABLE_DEBUG(db->soflow_db_so, (&matchentry));
573 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, &matchentry, true, "Looking for entry");
574
575 if (inp->inp_vflag & INP_IPV6) {
576 hashkey_faddr = matchentry.soflow_faddr.addr6.s6_addr32[3];
577 hashkey_laddr = (remoteOnly == false) ? matchentry.soflow_laddr.addr6.s6_addr32[3] : 0;
578 } else {
579 hashkey_faddr = matchentry.soflow_faddr.addr46.ia46_addr4.s_addr;
580 hashkey_laddr = (remoteOnly == false) ? matchentry.soflow_laddr.addr46.ia46_addr4.s_addr : 0;
581 }
582
583 hashkey_fport = matchentry.soflow_fport;
584 hashkey_lport = (remoteOnly == false || withLocalPort == true) ? matchentry.soflow_lport : 0;
585
586 inp_hash_element = SOFLOW_HASH(hashkey_laddr, hashkey_faddr, hashkey_lport, hashkey_fport);
587 inp_hash_element &= db->soflow_db_hashmask;
588 flowhash = &db->soflow_db_hashbase[inp_hash_element];
589
590 LIST_FOREACH(nextentry, flowhash, soflow_entry_link) {
591 if (inp->inp_vflag & INP_IPV6) {
592 if (soflow_match_entries_v6(nextentry, &matchentry, remoteOnly)) {
593 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Found entry v6");
594 break;
595 }
596 } else if (inp->inp_vflag & INP_IPV4) {
597 if (soflow_match_entries_v4(nextentry, &matchentry, remoteOnly)) {
598 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Found entry v4");
599 break;
600 }
601 }
602 }
603
604 if (nextentry == NULL) {
605 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, &matchentry, matchentry.soflow_debug, "Entry not found");
606 }
607 return nextentry;
608 }
609
610 static struct soflow_hash_entry *
soflow_db_lookup_entry(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote,boolean_t remoteOnly)611 soflow_db_lookup_entry(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote, boolean_t remoteOnly)
612 {
613 struct soflow_hash_entry *entry = soflow_db_lookup_entry_internal(db, local, remote, remoteOnly, false);
614 if (entry == NULL && remoteOnly == true) {
615 entry = soflow_db_lookup_entry_internal(db, local, remote, remoteOnly, true);
616 }
617 return entry;
618 }
619
620 static struct soflow_hash_entry *
soflow_db_lookup_by_feature_context_id(struct soflow_db * db,u_int64_t feature_context_id)621 soflow_db_lookup_by_feature_context_id(struct soflow_db *db, u_int64_t feature_context_id)
622 {
623 struct soflow_hash_head *flowhash = NULL;
624 u_int32_t inp_hash_element = (u_int32_t)(feature_context_id & 0x0ffffffff);
625 struct soflow_hash_entry *nextentry;
626
627 inp_hash_element &= db->soflow_db_hashmask;
628 flowhash = &db->soflow_db_hashbase[inp_hash_element];
629
630 LIST_FOREACH(nextentry, flowhash, soflow_entry_link) {
631 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Looking at entry");
632 if (nextentry->soflow_feat_ctxt != NULL &&
633 nextentry->soflow_feat_ctxt_id == feature_context_id) {
634 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, nextentry, nextentry->soflow_debug, "Found entry by feature context id");
635 break;
636 }
637 }
638
639 if (nextentry == NULL) {
640 SOFLOW_LOG(LOG_DEBUG, db->soflow_db_so, db->soflow_db_debug, "No entry found for featureID %llu <count %d hash %X %X>",
641 feature_context_id, db->soflow_db_count, inp_hash_element, (u_int32_t)(feature_context_id & 0x0ffffffff));
642 }
643 return nextentry;
644 }
645
646 void *
soflow_db_get_feature_context(struct soflow_db * db,u_int64_t feature_context_id)647 soflow_db_get_feature_context(struct soflow_db *db, u_int64_t feature_context_id)
648 {
649 struct soflow_hash_entry *hash_entry = NULL;
650 void * __single context = NULL;
651
652 if (db == NULL || db->soflow_db_so == NULL || feature_context_id == 0) {
653 return NULL;
654 }
655
656 socket_lock_assert_owned(db->soflow_db_so);
657
658 // Take refcount of db before use.
659 // Abort if db is already being freed.
660 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
661 return NULL;
662 }
663
664 // This is an optimization for datagram sockets with only one single flow.
665 if (db->soflow_db_count == 1) {
666 if (db->soflow_db_only_entry != NULL &&
667 db->soflow_db_only_entry->soflow_feat_ctxt != NULL && db->soflow_db_only_entry->soflow_feat_ctxt_id == feature_context_id) {
668 SOFLOW_ENTRY_LOG(LOG_DEBUG, db->soflow_db_so, db->soflow_db_only_entry, db->soflow_db_only_entry->soflow_debug, "MATCHED only entry for featureID");
669 context = db->soflow_db_only_entry->soflow_feat_ctxt;
670 } else {
671 SOFLOW_LOG(LOG_DEBUG, db->soflow_db_so, db->soflow_db_debug, "MISMATCHED only entry for featureID %llu (entry %p - cfil %p id %llu)",
672 feature_context_id,
673 db->soflow_db_only_entry,
674 db->soflow_db_only_entry ? db->soflow_db_only_entry->soflow_feat_ctxt : NULL,
675 db->soflow_db_only_entry ? db->soflow_db_only_entry->soflow_feat_ctxt_id : 0);
676 }
677 } else {
678 hash_entry = soflow_db_lookup_by_feature_context_id(db, feature_context_id);
679 context = hash_entry != NULL ? hash_entry->soflow_feat_ctxt : NULL;
680 }
681
682 SOFLOW_DB_FREE(db);
683 return context;
684 }
685
686 u_int64_t
soflow_db_get_feature_context_id(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote)687 soflow_db_get_feature_context_id(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote)
688 {
689 struct soflow_hash_entry *hash_entry = NULL;
690 uint64_t context_id = 0;
691
692 if (db == NULL || db->soflow_db_so == NULL) {
693 return 0;
694 }
695
696 socket_lock_assert_owned(db->soflow_db_so);
697
698 // Take refcount of db before use.
699 // Abort if db is already being freed.
700 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
701 return 0;
702 }
703
704 hash_entry = soflow_db_lookup_entry(db, local, remote, false);
705 if (hash_entry == NULL) {
706 // No match with both local and remote, try match with remote only
707 hash_entry = soflow_db_lookup_entry(db, local, remote, true);
708 }
709 if (hash_entry != NULL && hash_entry->soflow_feat_ctxt != NULL) {
710 context_id = hash_entry->soflow_feat_ctxt_id;
711 }
712
713 SOFLOW_DB_FREE(db);
714
715 return context_id;
716 }
717
718 static struct soflow_hash_entry *
soflow_db_add_entry(struct soflow_db * db,struct sockaddr * local,struct sockaddr * remote)719 soflow_db_add_entry(struct soflow_db *db, struct sockaddr *local, struct sockaddr *remote)
720 {
721 struct soflow_hash_entry *entry = NULL;
722 struct inpcb *inp = db ? sotoinpcb(db->soflow_db_so) : NULL;
723 u_int32_t hashkey_faddr = 0, hashkey_laddr = 0;
724 int inp_hash_element = 0;
725 struct soflow_hash_head *flowhash = NULL;
726
727 if (db == NULL || inp == NULL) {
728 goto done;
729 }
730
731 entry = kalloc_type(struct soflow_hash_entry, Z_WAITOK | Z_ZERO | Z_NOFAIL);
732 os_ref_init(&entry->soflow_ref_count, &soflow_refgrp);
733
734 if (local != NULL) {
735 soflow_fill_hash_entry_from_address(entry, TRUE, local, FALSE);
736 } else {
737 soflow_fill_hash_entry_from_inp(entry, TRUE, inp, FALSE);
738 }
739 if (remote != NULL) {
740 soflow_fill_hash_entry_from_address(entry, FALSE, remote, FALSE);
741 } else {
742 soflow_fill_hash_entry_from_inp(entry, FALSE, inp, FALSE);
743 }
744 entry->soflow_lastused = net_uptime();
745 entry->soflow_db = db;
746 entry->soflow_debug = SOFLOW_ENABLE_DEBUG(db->soflow_db_so, entry);
747 microuptime(&entry->soflow_timestamp);
748
749 if (inp->inp_vflag & INP_IPV6) {
750 hashkey_faddr = entry->soflow_faddr.addr6.s6_addr32[3];
751 hashkey_laddr = entry->soflow_laddr.addr6.s6_addr32[3];
752 } else {
753 hashkey_faddr = entry->soflow_faddr.addr46.ia46_addr4.s_addr;
754 hashkey_laddr = entry->soflow_laddr.addr46.ia46_addr4.s_addr;
755 }
756 entry->soflow_flowhash = SOFLOW_HASH(hashkey_laddr, hashkey_faddr,
757 entry->soflow_lport, entry->soflow_fport);
758 inp_hash_element = entry->soflow_flowhash & db->soflow_db_hashmask;
759
760 socket_lock_assert_owned(db->soflow_db_so);
761
762 // Take refcount of db before use.
763 // Abort if db is already being freed.
764 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
765 return NULL;
766 }
767
768 flowhash = &db->soflow_db_hashbase[inp_hash_element];
769
770 LIST_INSERT_HEAD(flowhash, entry, soflow_entry_link);
771 db->soflow_db_count++;
772 db->soflow_db_only_entry = entry;
773 SOFLOW_LOG(LOG_INFO, db->soflow_db_so, db->soflow_db_debug, "total count %d", db->soflow_db_count);
774
775 SOFLOW_DB_FREE(db);
776
777 done:
778 return entry;
779 }
780
781 static boolean_t
soflow_udp_get_address_from_control(sa_family_t family,struct mbuf * control,uint8_t * __counted_by (* count)* address_ptr,int * count)782 soflow_udp_get_address_from_control(sa_family_t family, struct mbuf *control, uint8_t *__counted_by(*count) *address_ptr, int *count)
783 {
784 struct cmsghdr *cm;
785 struct in6_pktinfo *pi6;
786 struct socket *so = NULL;
787
788 if (control == NULL || address_ptr == NULL) {
789 return false;
790 }
791
792 for (; control != NULL; control = control->m_next) {
793 if (control->m_type != MT_CONTROL) {
794 continue;
795 }
796
797 for (cm = M_FIRST_CMSGHDR(control);
798 is_cmsg_valid(control, cm);
799 cm = M_NXT_CMSGHDR(control, cm)) {
800 SOFLOW_LOG(LOG_DEBUG, so, true, "Check control type %d", cm->cmsg_type);
801
802 switch (cm->cmsg_type) {
803 case IP_RECVDSTADDR:
804 if (family == AF_INET &&
805 cm->cmsg_level == IPPROTO_IP &&
806 cm->cmsg_len == CMSG_LEN(sizeof(struct in_addr))) {
807 *address_ptr = CMSG_DATA(cm);
808 *count = sizeof(struct in_addr);
809 return true;
810 }
811 break;
812 case IPV6_PKTINFO:
813 case IPV6_2292PKTINFO:
814 if (family == AF_INET6 &&
815 cm->cmsg_level == IPPROTO_IPV6 &&
816 cm->cmsg_len == CMSG_LEN(sizeof(struct in6_pktinfo))) {
817 pi6 = (struct in6_pktinfo *)(void *)CMSG_DATA(cm);
818 *address_ptr = (uint8_t *)&pi6->ipi6_addr;
819 *count = sizeof(struct in6_addr);
820 return true;
821 }
822 break;
823 default:
824 break;
825 }
826 }
827 }
828 return false;
829 }
830
831 static boolean_t
soflow_entry_local_address_needs_update(struct soflow_hash_entry * entry)832 soflow_entry_local_address_needs_update(struct soflow_hash_entry *entry)
833 {
834 if (entry->soflow_family == AF_INET6) {
835 return IN6_IS_ADDR_UNSPECIFIED(&entry->soflow_laddr.addr6);
836 } else if (entry->soflow_family == AF_INET) {
837 return entry->soflow_laddr.addr46.ia46_addr4.s_addr == INADDR_ANY;
838 }
839 return false;
840 }
841
842 static boolean_t
soflow_entry_local_port_needs_update(struct socket * so,struct soflow_hash_entry * entry)843 soflow_entry_local_port_needs_update(struct socket *so, struct soflow_hash_entry *entry)
844 {
845 if (SOFLOW_IS_UDP(so)) {
846 return entry->soflow_lport == 0;
847 }
848 return false;
849 }
850
851 static void
soflow_entry_update_local(struct soflow_db * db,struct soflow_hash_entry * entry,struct sockaddr * local,struct mbuf * control,u_short rcv_ifindex)852 soflow_entry_update_local(struct soflow_db *db, struct soflow_hash_entry *entry, struct sockaddr *local, struct mbuf *control, u_short rcv_ifindex)
853 {
854 struct inpcb *inp = sotoinpcb(db->soflow_db_so);
855 union sockaddr_in_4_6 address_buf = { };
856
857 if (inp == NULL || entry == NULL) {
858 return;
859 }
860
861 if (entry->soflow_outifindex == 0 && (inp->inp_last_outifp != NULL || rcv_ifindex != 0)) {
862 entry->soflow_outifindex = inp->inp_last_outifp ? inp->inp_last_outifp->if_index : rcv_ifindex;
863 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Updated outifp");
864 }
865
866 if (soflow_entry_local_address_needs_update(entry)) {
867 // Flow does not have a local address yet. Retrieve local address
868 // from control mbufs if present.
869 if (local == NULL && control != NULL) {
870 int size = 0;
871 uint8_t * __counted_by(size) addr_ptr = NULL;
872 boolean_t result = soflow_udp_get_address_from_control(entry->soflow_family, control, &addr_ptr, &size);
873
874 if (result && size && addr_ptr) {
875 switch (entry->soflow_family) {
876 case AF_INET:
877 if (size == sizeof(struct in_addr)) {
878 address_buf.sin.sin_port = 0;
879 address_buf.sin.sin_family = AF_INET;
880 address_buf.sin.sin_len = sizeof(struct sockaddr_in);
881 (void) memcpy(&address_buf.sin.sin_addr, addr_ptr, sizeof(struct in_addr));
882 local = sintosa(&address_buf.sin);
883 }
884 break;
885 case AF_INET6:
886 if (size == sizeof(struct in6_addr)) {
887 address_buf.sin6.sin6_port = 0;
888 address_buf.sin6.sin6_family = AF_INET6;
889 address_buf.sin6.sin6_len = sizeof(struct sockaddr_in6);
890 (void) memcpy(&address_buf.sin6.sin6_addr, addr_ptr, sizeof(struct in6_addr));
891 local = sin6tosa(&address_buf.sin6);
892 }
893 break;
894 default:
895 break;
896 }
897 }
898 }
899 if (local != NULL) {
900 soflow_fill_hash_entry_from_address(entry, TRUE, local, TRUE);
901 } else {
902 soflow_fill_hash_entry_from_inp(entry, TRUE, inp, TRUE);
903 }
904 if (entry->soflow_laddr_updated) {
905 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Updated address");
906 }
907 }
908
909 if (soflow_entry_local_port_needs_update(db->soflow_db_so, entry)) {
910 soflow_fill_hash_entry_from_inp(entry, TRUE, inp, TRUE);
911 if (entry->soflow_lport_updated) {
912 SOFLOW_ENTRY_LOG(LOG_INFO, db->soflow_db_so, entry, entry->soflow_debug, "Updated port");
913 }
914 }
915
916 return;
917 }
918
919 #if defined(NSTAT_EXTENSION_FILTER_DOMAIN_INFO)
920 static u_int32_t
ifnet_to_flags(struct ifnet * ifp,struct socket * so)921 ifnet_to_flags(struct ifnet *ifp, struct socket *so)
922 {
923 u_int32_t flags = 0;
924
925 if (ifp != NULL) {
926 flags = nstat_ifnet_to_flags(ifp);
927 if ((flags & NSTAT_IFNET_IS_WIFI) && ((flags & (NSTAT_IFNET_IS_AWDL | NSTAT_IFNET_IS_LLW)) == 0)) {
928 flags |= NSTAT_IFNET_IS_WIFI_INFRA;
929 }
930 } else {
931 flags = NSTAT_IFNET_IS_UNKNOWN_TYPE;
932 }
933
934 if (so != NULL && (so->so_flags1 & SOF1_CELLFALLBACK)) {
935 flags |= NSTAT_IFNET_VIA_CELLFALLBACK;
936 }
937 return flags;
938 }
939
940 static bool
soflow_nstat_provider_request_vals(nstat_provider_context ctx,u_int32_t * ifflagsp,nstat_counts * countsp,void * metadatap)941 soflow_nstat_provider_request_vals(nstat_provider_context ctx,
942 u_int32_t *ifflagsp,
943 nstat_counts *countsp,
944 void *metadatap)
945 {
946 struct soflow_hash_entry *hash_entry = (struct soflow_hash_entry *) ctx;
947 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
948 struct inpcb *inp = so ? sotoinpcb(so) : NULL;
949 char local[MAX_IPv6_STR_LEN + 6] = { 0 };
950 char remote[MAX_IPv6_STR_LEN + 6] = { 0 };
951 const void *addr = NULL;
952
953 if (hash_entry == NULL || so == NULL || inp == NULL) {
954 return false;
955 }
956
957 if (ifflagsp) {
958 if (hash_entry->soflow_outifindex) {
959 struct ifnet *ifp = ifindex2ifnet[hash_entry->soflow_outifindex];
960 *ifflagsp = ifnet_to_flags(ifp, so);
961 }
962 if ((countsp == NULL) && (metadatap == NULL)) {
963 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug, "ifflagsp set to 0x%X", *ifflagsp);
964 goto done;
965 }
966 }
967
968 if (countsp) {
969 bzero(countsp, sizeof(*countsp));
970 countsp->nstat_rxpackets = hash_entry->soflow_rxpackets;
971 countsp->nstat_rxbytes = hash_entry->soflow_rxbytes;
972 countsp->nstat_txpackets = hash_entry->soflow_txpackets;
973 countsp->nstat_txbytes = hash_entry->soflow_txbytes;
974
975 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug,
976 "Collected NSTAT counts: rxpackets %llu rxbytes %llu txpackets %llu txbytes %llu",
977 countsp->nstat_rxpackets, countsp->nstat_rxbytes, countsp->nstat_txpackets, countsp->nstat_txbytes);
978 }
979
980 if (metadatap) {
981 nstat_udp_descriptor *desc = (nstat_udp_descriptor *)metadatap;
982 bzero(desc, sizeof(*desc));
983
984 if (so->so_flags & SOF_DELEGATED) {
985 desc->eupid = so->e_upid;
986 desc->epid = so->e_pid;
987 uuid_copy(desc->euuid, so->e_uuid);
988 } else {
989 desc->eupid = so->last_upid;
990 desc->epid = so->last_pid;
991 uuid_copy(desc->euuid, so->last_uuid);
992 }
993
994 uuid_copy(desc->vuuid, so->so_vuuid);
995 uuid_copy(desc->fuuid, hash_entry->soflow_uuid);
996
997 if (hash_entry->soflow_family == AF_INET6) {
998 in6_ip6_to_sockaddr(&hash_entry->soflow_laddr.addr6, hash_entry->soflow_lport, hash_entry->soflow_laddr6_ifscope,
999 &desc->local.v6, sizeof(desc->local.v6));
1000 in6_ip6_to_sockaddr(&hash_entry->soflow_faddr.addr6, hash_entry->soflow_fport, hash_entry->soflow_faddr6_ifscope,
1001 &desc->remote.v6, sizeof(desc->remote.v6));
1002 } else if (hash_entry->soflow_family == AF_INET) {
1003 desc->local.v4.sin_family = AF_INET;
1004 desc->local.v4.sin_len = sizeof(struct sockaddr_in);
1005 desc->local.v4.sin_port = hash_entry->soflow_lport;
1006 desc->local.v4.sin_addr = hash_entry->soflow_laddr.addr46.ia46_addr4;
1007
1008 desc->remote.v4.sin_family = AF_INET;
1009 desc->remote.v4.sin_len = sizeof(struct sockaddr_in);
1010 desc->remote.v4.sin_port = hash_entry->soflow_fport;
1011 desc->remote.v4.sin_addr = hash_entry->soflow_faddr.addr46.ia46_addr4;
1012 }
1013
1014 desc->ifindex = hash_entry->soflow_outifindex;
1015 if (hash_entry->soflow_outifindex) {
1016 struct ifnet *ifp = ifindex2ifnet[hash_entry->soflow_outifindex];
1017 desc->ifnet_properties = (uint16_t)ifnet_to_flags(ifp, so);
1018 }
1019
1020 desc->rcvbufsize = so->so_rcv.sb_hiwat;
1021 desc->rcvbufused = so->so_rcv.sb_cc;
1022 desc->traffic_class = so->so_traffic_class;
1023 inp_get_activity_bitmap(inp, &desc->activity_bitmap);
1024
1025 if (hash_entry->soflow_debug) {
1026 switch (hash_entry->soflow_family) {
1027 case AF_INET6:
1028 addr = &desc->local.v6;
1029 inet_ntop(AF_INET6, addr, local, sizeof(local));
1030 addr = &desc->remote.v6;
1031 inet_ntop(AF_INET6, addr, remote, sizeof(local));
1032 break;
1033 case AF_INET:
1034 addr = &desc->local.v4.sin_addr;
1035 inet_ntop(AF_INET, addr, local, sizeof(local));
1036 addr = &desc->remote.v4.sin_addr;
1037 inet_ntop(AF_INET, addr, remote, sizeof(local));
1038 break;
1039 default:
1040 break;
1041 }
1042
1043 uint8_t *ptr = (uint8_t *)&desc->euuid;
1044
1045 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug,
1046 "Collected NSTAT metadata: eupid %llu epid %d euuid %x%x%x%x-%x%x%x%x-%x%x%x%x-%x%x%x%x "
1047 "outifp %d properties 0x%X lport %d fport %d laddr %s faddr %s "
1048 "rcvbufsize %u rcvbufused %u traffic_class %u",
1049 desc->eupid, desc->epid,
1050 ptr[0], ptr[1], ptr[2], ptr[3], ptr[4], ptr[5], ptr[6], ptr[7],
1051 ptr[8], ptr[9], ptr[10], ptr[11], ptr[12], ptr[13], ptr[14], ptr[15],
1052 desc->ifindex, desc->ifnet_properties,
1053 ntohs(desc->local.v4.sin_port), ntohs(desc->remote.v4.sin_port), local, remote,
1054 desc->rcvbufsize, desc->rcvbufused, desc->traffic_class);
1055 }
1056 }
1057 done:
1058 return true;
1059 }
1060
1061 static size_t
soflow_nstat_provider_request_extensions(nstat_provider_context ctx,int requested_extension,void * buf,size_t buf_size)1062 soflow_nstat_provider_request_extensions(nstat_provider_context ctx,
1063 int requested_extension,
1064 void *buf,
1065 size_t buf_size)
1066 {
1067 struct soflow_hash_entry *hash_entry = (struct soflow_hash_entry *) ctx;
1068 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
1069 struct inpcb *inp = so ? sotoinpcb(so) : NULL;
1070 struct nstat_domain_info *domain_info = NULL;
1071 size_t size = 0;
1072
1073 if (hash_entry == NULL || so == NULL || inp == NULL) {
1074 return 0;
1075 }
1076
1077 if (buf == NULL) {
1078 switch (requested_extension) {
1079 case NSTAT_EXTENDED_UPDATE_TYPE_DOMAIN:
1080 return sizeof(nstat_domain_info);
1081 default:
1082 return 0;
1083 }
1084 }
1085
1086 if (buf_size < sizeof(nstat_domain_info)) {
1087 return 0;
1088 }
1089
1090 switch (requested_extension) {
1091 case NSTAT_EXTENDED_UPDATE_TYPE_DOMAIN:
1092
1093 domain_info = (struct nstat_domain_info *)buf;
1094 necp_copy_inp_domain_info(inp, so, domain_info);
1095
1096 if (hash_entry->soflow_debug) {
1097 SOFLOW_LOG(LOG_DEBUG, so, hash_entry->soflow_debug, "Collected NSTAT domain_info:pid %d domain <%s> owner <%s> "
1098 "ctxt <%s> bundle id <%s> is_tracker %d is_non_app_initiated %d is_silent %d",
1099 so->so_flags & SOF_DELEGATED ? so->e_pid : so->last_pid,
1100 domain_info->domain_name,
1101 domain_info->domain_owner,
1102 domain_info->domain_tracker_ctxt,
1103 domain_info->domain_attributed_bundle_id,
1104 domain_info->is_tracker,
1105 domain_info->is_non_app_initiated,
1106 domain_info->is_silent);
1107 }
1108 size = sizeof(nstat_domain_info);
1109
1110 default:
1111 break;
1112 }
1113
1114 return size;
1115 }
1116 #endif
1117
1118 static void
soflow_update_flow_stats(struct soflow_hash_entry * hash_entry,size_t data_size,bool outgoing)1119 soflow_update_flow_stats(struct soflow_hash_entry *hash_entry, size_t data_size, bool outgoing)
1120 {
1121 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
1122
1123 if (hash_entry != NULL) {
1124 if (outgoing) {
1125 hash_entry->soflow_txbytes += data_size;
1126 hash_entry->soflow_txpackets++;
1127 SOFLOW_ENTRY_LOG(LOG_DEBUG, so, hash_entry, hash_entry->soflow_debug, "Stats update - Outgoing");
1128 } else {
1129 hash_entry->soflow_rxbytes += data_size;
1130 hash_entry->soflow_rxpackets++;
1131 SOFLOW_ENTRY_LOG(LOG_DEBUG, so, hash_entry, hash_entry->soflow_debug, "Stats update - Incoming");
1132 }
1133 }
1134 }
1135
1136 struct soflow_hash_entry *
soflow_get_flow(struct socket * so,struct sockaddr * local,struct sockaddr * remote,struct mbuf * control,size_t data_size,soflow_direction_t direction,uint16_t rcv_ifindex)1137 soflow_get_flow(struct socket *so, struct sockaddr *local, struct sockaddr *remote, struct mbuf *control,
1138 size_t data_size, soflow_direction_t direction, uint16_t rcv_ifindex)
1139 {
1140 struct soflow_hash_entry *hash_entry = NULL;
1141 struct inpcb *inp = sotoinpcb(so);
1142
1143 // Check if feature is disabled
1144 if (soflow_disable) {
1145 return NULL;
1146 }
1147
1148 socket_lock_assert_owned(so);
1149
1150 if (so->so_flow_db != NULL) {
1151 // Take refcount of db before use.
1152 // Abort if db is already being freed.
1153 if (os_ref_retain_try(&so->so_flow_db->soflow_db_ref_count) == false) {
1154 return NULL;
1155 }
1156
1157 // DB already exists, check if this is existing flow
1158 hash_entry = soflow_db_lookup_entry(so->so_flow_db, local, remote, false);
1159 if (hash_entry == NULL) {
1160 // No match with both local and remote, try match with remote only
1161 hash_entry = soflow_db_lookup_entry(so->so_flow_db, local, remote, true);
1162 }
1163 if (hash_entry != NULL) {
1164 // Take refcount of entry before use.
1165 // Abort if entry is already being freed.
1166 if (os_ref_retain_try(&hash_entry->soflow_ref_count) == false) {
1167 SOFLOW_DB_FREE(so->so_flow_db);
1168 return NULL;
1169 }
1170
1171 // Try to update flow info from socket and/or control mbufs if necessary
1172 if (hash_entry->soflow_outifindex == 0 ||
1173 soflow_entry_local_address_needs_update(hash_entry) || soflow_entry_local_port_needs_update(so, hash_entry)) {
1174 soflow_entry_update_local(so->so_flow_db, hash_entry, local, control, rcv_ifindex);
1175 }
1176 hash_entry->soflow_lastused = net_uptime();
1177 if (data_size > 0 && direction != SOFLOW_DIRECTION_UNKNOWN) {
1178 soflow_update_flow_stats(hash_entry, data_size, direction == SOFLOW_DIRECTION_OUTBOUND);
1179 }
1180
1181 SOFLOW_DB_FREE(so->so_flow_db);
1182 return hash_entry;
1183 }
1184
1185 SOFLOW_DB_FREE(so->so_flow_db);
1186 }
1187
1188 // No flow was found. Only add a new flow if the direction is known.
1189 if (direction == SOFLOW_DIRECTION_UNKNOWN) {
1190 return NULL;
1191 }
1192
1193 if (so->so_flow_db == NULL) {
1194 // If new socket, allocate cfil db
1195 if (soflow_db_init(so) != 0) {
1196 return NULL;
1197 }
1198 }
1199
1200 hash_entry = soflow_db_add_entry(so->so_flow_db, local, remote);
1201 if (hash_entry == NULL) {
1202 SOFLOW_LOG(LOG_ERR, so, true, "Failed to add entry");
1203 return NULL;
1204 }
1205
1206 // Take refcount of entry before use.
1207 // Abort if entry is already being freed.
1208 if (os_ref_retain_try(&hash_entry->soflow_ref_count) == false) {
1209 return NULL;
1210 }
1211
1212 if (inp && (inp->inp_last_outifp != NULL || rcv_ifindex != 0)) {
1213 hash_entry->soflow_outifindex = inp->inp_last_outifp ? inp->inp_last_outifp->if_index : rcv_ifindex;
1214 }
1215
1216 // Check if we can update the new flow's local address from control mbufs
1217 if (control != NULL) {
1218 soflow_entry_update_local(so->so_flow_db, hash_entry, local, control, rcv_ifindex);
1219 }
1220 hash_entry->soflow_outgoing = (direction == SOFLOW_DIRECTION_OUTBOUND);
1221 if (data_size > 0) {
1222 soflow_update_flow_stats(hash_entry, data_size, direction == SOFLOW_DIRECTION_OUTBOUND);
1223 }
1224
1225 // Only report flow to NSTAT if unconnected UDP
1226 if (!soflow_nstat_disable && SOFLOW_IS_UDP(so) && !(so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING))) {
1227 #if defined(NSTAT_EXTENSION_FILTER_DOMAIN_INFO)
1228 // Take refcount of entry before handing it to nstat. Abort if fail.
1229 if (os_ref_retain_try(&hash_entry->soflow_ref_count) == false) {
1230 return NULL;
1231 }
1232 uuid_generate_random(hash_entry->soflow_uuid);
1233 hash_entry->soflow_nstat_context = nstat_provider_stats_open((nstat_provider_context) hash_entry,
1234 NSTAT_PROVIDER_UDP_SUBFLOW, 0,
1235 soflow_nstat_provider_request_vals,
1236 soflow_nstat_provider_request_extensions);
1237 SOFLOW_LOG(LOG_INFO, so, hash_entry->soflow_debug, "<Open nstat> - context %lX", (unsigned long)hash_entry->soflow_nstat_context);
1238 #endif
1239 }
1240
1241 SOFLOW_LOCK_EXCLUSIVE;
1242 if (soflow_initialized == 0) {
1243 soflow_init();
1244 }
1245 TAILQ_INSERT_TAIL(&soflow_entry_head, hash_entry, soflow_entry_list_link);
1246 if (soflow_attached_count == 0) {
1247 thread_wakeup((caddr_t)&soflow_attached_count);
1248 }
1249 soflow_attached_count++;
1250 if (soflow_attached_high_water_mark < soflow_attached_count) {
1251 soflow_attached_high_water_mark = soflow_attached_count;
1252 }
1253 SOFLOW_UNLOCK_EXCLUSIVE;
1254
1255 SOFLOW_ENTRY_LOG(LOG_INFO, so, hash_entry, hash_entry->soflow_debug, "Added entry");
1256 return hash_entry;
1257 }
1258
1259 void
soflow_free_flow(struct soflow_hash_entry * entry)1260 soflow_free_flow(struct soflow_hash_entry *entry)
1261 {
1262 SOFLOW_ENTRY_FREE(entry);
1263 }
1264
1265 static bool
soflow_socket_safe_lock(struct inpcb * inp,struct inpcbinfo * pcbinfo)1266 soflow_socket_safe_lock(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1267 {
1268 struct socket *so = NULL;
1269
1270 VERIFY(pcbinfo != NULL);
1271
1272 if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) != WNT_STOPUSING) {
1273 // Safeguarded the inp state, unlock pcbinfo before locking socket.
1274 lck_rw_done(&pcbinfo->ipi_lock);
1275
1276 so = inp->inp_socket;
1277 socket_lock(so, 1);
1278 if (in_pcb_checkstate(inp, WNT_RELEASE, 1) != WNT_STOPUSING) {
1279 return true;
1280 }
1281 } else {
1282 // Failed to safeguarded the inp state, unlock pcbinfo and abort.
1283 lck_rw_done(&pcbinfo->ipi_lock);
1284 }
1285
1286 if (so) {
1287 socket_unlock(so, 1);
1288 }
1289 return false;
1290 }
1291
1292 static struct socket *
soflow_validate_dgram_socket(struct socket * so)1293 soflow_validate_dgram_socket(struct socket *so)
1294 {
1295 struct inpcb *inp = NULL;
1296 struct inpcbinfo *pcbinfo = NULL;
1297 struct socket *locked = NULL;
1298
1299 pcbinfo = &udbinfo;
1300 lck_rw_lock_shared(&pcbinfo->ipi_lock);
1301 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1302 if (inp->inp_state != INPCB_STATE_DEAD && inp->inp_socket == so) {
1303 if (soflow_socket_safe_lock(inp, pcbinfo)) {
1304 locked = inp->inp_socket;
1305 }
1306 /* pcbinfo is already unlocked, we are done. */
1307 goto done;
1308 }
1309 }
1310 lck_rw_done(&pcbinfo->ipi_lock);
1311 if (locked != NULL) {
1312 goto done;
1313 }
1314
1315 pcbinfo = &ripcbinfo;
1316 lck_rw_lock_shared(&pcbinfo->ipi_lock);
1317 LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1318 if (inp->inp_state != INPCB_STATE_DEAD && inp->inp_socket == so) {
1319 if (soflow_socket_safe_lock(inp, pcbinfo)) {
1320 locked = inp->inp_socket;
1321 }
1322 /* pcbinfo is already unlocked, we are done. */
1323 goto done;
1324 }
1325 }
1326 lck_rw_done(&pcbinfo->ipi_lock);
1327
1328 done:
1329 return locked;
1330 }
1331
1332 static void
soflow_gc_thread_sleep(bool forever)1333 soflow_gc_thread_sleep(bool forever)
1334 {
1335 if (forever) {
1336 (void) assert_wait((event_t) &soflow_attached_count,
1337 THREAD_INTERRUPTIBLE);
1338 } else {
1339 uint64_t deadline = 0;
1340 nanoseconds_to_absolutetime(SOFLOW_GC_RUN_INTERVAL_NSEC, &deadline);
1341 clock_absolutetime_interval_to_deadline(deadline, &deadline);
1342
1343 (void) assert_wait_deadline(&soflow_attached_count,
1344 THREAD_INTERRUPTIBLE, deadline);
1345 }
1346 }
1347
1348 static void
soflow_gc_thread_func(void * v,wait_result_t w)1349 soflow_gc_thread_func(void *v, wait_result_t w)
1350 {
1351 #pragma unused(v, w)
1352
1353 ASSERT(soflow_gc_thread == current_thread());
1354 thread_set_thread_name(current_thread(), "SOFLOW_GC");
1355
1356 // Kick off gc shortly
1357 soflow_gc_thread_sleep(false);
1358 thread_block_parameter((thread_continue_t) soflow_gc_expire, NULL);
1359 /* NOTREACHED */
1360 }
1361
1362 static bool
soflow_gc_idle_timed_out(struct soflow_hash_entry * hash_entry,int timeout,u_int64_t current_time)1363 soflow_gc_idle_timed_out(struct soflow_hash_entry *hash_entry, int timeout, u_int64_t current_time)
1364 {
1365 struct socket *so = (hash_entry && hash_entry->soflow_db) ? hash_entry->soflow_db->soflow_db_so : NULL;
1366
1367 if (hash_entry && (current_time - hash_entry->soflow_lastused >= (u_int64_t)timeout)) {
1368 SOFLOW_ENTRY_LOG(LOG_INFO, so, hash_entry, hash_entry->soflow_debug, "GC Idle Timeout detected");
1369 return true;
1370 }
1371 return false;
1372 }
1373
1374 static int
soflow_gc_cleanup(struct socket * so)1375 soflow_gc_cleanup(struct socket *so)
1376 {
1377 struct soflow_hash_entry *entry = NULL;
1378 struct soflow_hash_entry *temp_entry = NULL;
1379 struct soflow_hash_head *flowhash = NULL;
1380 struct soflow_db *db = NULL;
1381 int cleaned = 0;
1382
1383 if (so == NULL || so->so_flow_db == NULL) {
1384 return 0;
1385 }
1386 db = so->so_flow_db;
1387
1388 // Do not collect garbage for databases that have only one flow.
1389 if (db->soflow_db_count == 1 && db->soflow_db_only_entry != NULL) {
1390 return 0;
1391 }
1392
1393 socket_lock_assert_owned(so);
1394
1395 // Take refcount of db before use.
1396 // Abort if db is already being freed.
1397 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
1398 return 0;
1399 }
1400
1401 for (int i = 0; i < SOFLOW_HASH_SIZE; i++) {
1402 flowhash = &db->soflow_db_hashbase[i];
1403 LIST_FOREACH_SAFE(entry, flowhash, soflow_entry_link, temp_entry) {
1404 if (entry->soflow_gc || entry->soflow_feat_gc) {
1405 if (entry->soflow_feat_ctxt != NULL && soflow_feat_gc_perform_func_ptr != NULL) {
1406 soflow_feat_gc_perform_func_ptr(so, entry);
1407 entry->soflow_feat_ctxt = NULL;
1408 entry->soflow_feat_ctxt_id = 0;
1409 }
1410 entry->soflow_feat_gc = 0;
1411
1412 if (entry->soflow_gc) {
1413 SOFLOW_ENTRY_LOG(LOG_INFO, so, entry, entry->soflow_debug, "GC cleanup entry");
1414 entry->soflow_gc = 0;
1415 soflow_db_remove_entry(db, entry);
1416 cleaned++;
1417 }
1418 }
1419 }
1420 }
1421
1422 SOFLOW_DB_FREE(db);
1423 return cleaned;
1424 }
1425
1426 static void
soflow_gc_expire(void * v,wait_result_t w)1427 soflow_gc_expire(void *v, wait_result_t w)
1428 {
1429 #pragma unused(v, w)
1430
1431 static struct socket *socket_array[SOFLOW_GC_MAX_COUNT];
1432 struct soflow_hash_entry *hash_entry = NULL;
1433 struct socket *so = NULL;
1434 u_int64_t current_time = net_uptime();
1435 uint32_t socket_count = 0;
1436 uint32_t cleaned_count = 0;
1437 bool recorded = false;
1438
1439 // Collect a list of socket with expired flows
1440
1441 SOFLOW_LOCK_SHARED;
1442
1443 if (soflow_attached_count == 0) {
1444 SOFLOW_UNLOCK_SHARED;
1445 goto go_sleep;
1446 }
1447
1448 // Go thorough all flows in the flow list and record any socket with expired flows.
1449 TAILQ_FOREACH(hash_entry, &soflow_entry_head, soflow_entry_list_link) {
1450 if (socket_count >= SOFLOW_GC_MAX_COUNT) {
1451 break;
1452 }
1453 so = hash_entry->soflow_db ? hash_entry->soflow_db->soflow_db_so : NULL;
1454
1455 // Check if we need to perform cleanup due to idle time or feature specified rules
1456 hash_entry->soflow_gc = soflow_gc_idle_timed_out(hash_entry, SOFLOW_GC_IDLE_TO, current_time);
1457 hash_entry->soflow_feat_gc = (soflow_feat_gc_needed_func_ptr != NULL && soflow_feat_gc_needed_func_ptr(so, hash_entry, current_time));
1458
1459 if (hash_entry->soflow_gc || hash_entry->soflow_feat_gc) {
1460 if (so != NULL) {
1461 recorded = false;
1462 for (int i = 0; i < socket_count; i++) {
1463 if (socket_array[socket_count] == so) {
1464 recorded = true;
1465 break;
1466 }
1467 }
1468 if (recorded == false) {
1469 socket_array[socket_count] = so;
1470 socket_count++;
1471 }
1472 }
1473 }
1474 }
1475 SOFLOW_UNLOCK_SHARED;
1476
1477 if (socket_count == 0) {
1478 goto go_sleep;
1479 }
1480
1481 for (uint32_t i = 0; i < socket_count; i++) {
1482 // Validate socket and lock it
1483 so = soflow_validate_dgram_socket(socket_array[i]);
1484 if (so == NULL) {
1485 continue;
1486 }
1487 cleaned_count += soflow_gc_cleanup(so);
1488 socket_unlock(so, 1);
1489 }
1490
1491 so = NULL;
1492 SOFLOW_LOG(LOG_INFO, so, true, "<GC cleaned %d flows>", cleaned_count);
1493
1494 go_sleep:
1495
1496 // Sleep forever (until waken up) if no more UDP flow to clean
1497 SOFLOW_LOCK_SHARED;
1498 soflow_gc_thread_sleep(soflow_attached_count == 0 ? true : false);
1499 SOFLOW_UNLOCK_SHARED;
1500 thread_block_parameter((thread_continue_t)soflow_gc_expire, NULL);
1501 /* NOTREACHED */
1502 }
1503
1504 void
soflow_feat_set_functions(soflow_feat_gc_needed_func gc_needed_fn,soflow_feat_gc_perform_func gc_perform_fn,soflow_feat_detach_entry_func feat_detach_entry_fn,soflow_feat_detach_db_func feat_detach_db_fn)1505 soflow_feat_set_functions(soflow_feat_gc_needed_func gc_needed_fn,
1506 soflow_feat_gc_perform_func gc_perform_fn,
1507 soflow_feat_detach_entry_func feat_detach_entry_fn,
1508 soflow_feat_detach_db_func feat_detach_db_fn)
1509 {
1510 soflow_feat_gc_needed_func_ptr = gc_needed_fn;
1511 soflow_feat_gc_perform_func_ptr = gc_perform_fn;
1512 soflow_feat_detach_entry_func_ptr = feat_detach_entry_fn;
1513 soflow_feat_detach_db_func_ptr = feat_detach_db_fn;
1514 }
1515
1516 bool
soflow_db_apply(struct soflow_db * db,soflow_entry_apply_func entry_apply_fn,void * context)1517 soflow_db_apply(struct soflow_db *db, soflow_entry_apply_func entry_apply_fn, void *context)
1518 {
1519 struct soflow_hash_entry *entry = NULL;
1520 struct soflow_hash_entry *temp_entry = NULL;
1521 struct soflow_hash_head *flowhash = NULL;
1522
1523 if (db == NULL || db->soflow_db_so == NULL || entry_apply_fn == NULL) {
1524 return false;
1525 }
1526
1527 socket_lock_assert_owned(db->soflow_db_so);
1528
1529 // Take refcount of db before use.
1530 // Abort if db is already being freed.
1531 if (os_ref_retain_try(&db->soflow_db_ref_count) == false) {
1532 return false;
1533 }
1534
1535 for (int i = 0; i < SOFLOW_HASH_SIZE; i++) {
1536 flowhash = &db->soflow_db_hashbase[i];
1537 LIST_FOREACH_SAFE(entry, flowhash, soflow_entry_link, temp_entry) {
1538 if (entry_apply_fn(db->soflow_db_so, entry, context) == false) {
1539 goto done;
1540 }
1541 }
1542 }
1543
1544 done:
1545 SOFLOW_DB_FREE(db);
1546 return true;
1547 }
1548