1 /*
2 * Copyright (c) 2015-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* TCP-cache to store and retrieve TCP-related information */
30
31 #include <net/flowhash.h>
32 #include <net/route.h>
33 #include <net/necp.h>
34 #include <netinet/in_pcb.h>
35 #include <netinet/mptcp.h>
36 #include <netinet/mptcp_var.h>
37 #include <netinet/tcp_cache.h>
38 #include <netinet/tcp_seq.h>
39 #include <netinet/tcp_var.h>
40 #include <kern/locks.h>
41 #include <sys/queue.h>
42 #include <dev/random/randomdev.h>
43 #include <net/sockaddr_utils.h>
44
45 typedef union {
46 struct in_addr addr;
47 struct in6_addr addr6;
48 } in_4_6_addr;
49
50 struct tcp_heuristic_key {
51 union {
52 uint8_t thk_net_signature[IFNET_SIGNATURELEN];
53 in_4_6_addr thk_ip;
54 };
55 sa_family_t thk_family;
56 };
57
58 struct tcp_heuristic {
59 SLIST_ENTRY(tcp_heuristic) list;
60
61 uint32_t th_last_access;
62
63 struct tcp_heuristic_key th_key;
64
65 #define th_val_start th_tfo_data_loss
66 uint8_t th_tfo_data_loss; /* The number of times a SYN+data has been lost */
67 uint8_t th_tfo_req_loss; /* The number of times a SYN+cookie-req has been lost */
68 uint8_t th_tfo_data_rst; /* The number of times a SYN+data has received a RST */
69 uint8_t th_tfo_req_rst; /* The number of times a SYN+cookie-req has received a RST */
70 uint8_t th_mptcp_loss; /* The number of times a SYN+MP_CAPABLE has been lost */
71 uint8_t th_mptcp_success; /* The number of times MPTCP-negotiation has been successful */
72 uint8_t th_ecn_loss; /* The number of times a SYN+ecn has been lost */
73 uint8_t th_ecn_aggressive; /* The number of times we did an aggressive fallback */
74 uint8_t th_ecn_droprst; /* The number of times ECN connections received a RST after first data pkt */
75 uint8_t th_ecn_droprxmt; /* The number of times ECN connection is dropped after multiple retransmits */
76 uint8_t th_ecn_synrst; /* number of times RST was received in response to an ECN enabled SYN */
77 uint32_t th_tfo_enabled_time; /* The moment when we reenabled TFO after backing off */
78 uint32_t th_tfo_backoff_until; /* Time until when we should not try out TFO */
79 uint32_t th_tfo_backoff; /* Current backoff timer */
80 uint32_t th_mptcp_backoff; /* Time until when we should not try out MPTCP */
81 uint32_t th_ecn_backoff; /* Time until when we should not try out ECN */
82
83 uint8_t th_tfo_in_backoff:1, /* Are we avoiding TFO due to the backoff timer? */
84 th_mptcp_in_backoff:1, /* Are we avoiding MPTCP due to the backoff timer? */
85 th_mptcp_heuristic_disabled:1; /* Are heuristics disabled? */
86 // N.B.: we may sometimes erase ALL values from th_val_start to the end of the structure.
87 };
88
89 struct tcp_heuristics_head {
90 SLIST_HEAD(tcp_heur_bucket, tcp_heuristic) tcp_heuristics;
91
92 /* Per-hashbucket lock to avoid lock-contention */
93 lck_mtx_t thh_mtx;
94 };
95
96 struct tcp_cache_key {
97 sa_family_t tck_family;
98
99 struct tcp_heuristic_key tck_src;
100 in_4_6_addr tck_dst;
101 };
102
103 #define MPTCP_VERSION_SUPPORTED 1
104 #define MPTCP_VERSION_UNSUPPORTED -1
105 #define MPTCP_VERSION_SUPPORTED_UNKNOWN 0
106 struct tcp_cache {
107 SLIST_ENTRY(tcp_cache) list;
108
109 uint32_t tc_last_access;
110
111 struct tcp_cache_key tc_key;
112
113 uint8_t tc_tfo_cookie[TFO_COOKIE_LEN_MAX];
114 uint8_t tc_tfo_cookie_len;
115
116 uint8_t tc_mptcp_version_confirmed:1;
117 uint8_t tc_mptcp_version; /* version to use right now */
118 uint32_t tc_mptcp_next_version_try; /* Time, until we try preferred version again */
119 };
120
121 struct tcp_cache_head {
122 SLIST_HEAD(tcp_cache_bucket, tcp_cache) tcp_caches;
123
124 /* Per-hashbucket lock to avoid lock-contention */
125 lck_mtx_t tch_mtx;
126 };
127
128 struct tcp_cache_key_src {
129 struct ifnet *ifp;
130 in_4_6_addr laddr;
131 in_4_6_addr faddr;
132 int af;
133 };
134
135 static uint32_t tcp_cache_hash_seed;
136
137 static size_t tcp_cache_size;
138 static size_t tcp_heuristics_size;
139
140 /*
141 * The maximum depth of the hash-bucket. This way we limit the tcp_cache to
142 * TCP_CACHE_BUCKET_SIZE * tcp_cache_size and have "natural" garbage collection
143 */
144 #define TCP_CACHE_BUCKET_SIZE 5
145
146 static struct tcp_cache_head *__counted_by(tcp_cache_size) tcp_cache;
147
148 static LCK_ATTR_DECLARE(tcp_cache_mtx_attr, 0, 0);
149 static LCK_GRP_DECLARE(tcp_cache_mtx_grp, "tcpcache");
150
151 static struct tcp_heuristics_head *__counted_by(tcp_heuristics_size) tcp_heuristics;
152
153 static LCK_ATTR_DECLARE(tcp_heuristic_mtx_attr, 0, 0);
154 static LCK_GRP_DECLARE(tcp_heuristic_mtx_grp, "tcpheuristic");
155
156 static uint32_t tcp_backoff_maximum = 65536;
157
158 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, backoff_maximum, CTLFLAG_RW | CTLFLAG_LOCKED,
159 &tcp_backoff_maximum, 0, "Maximum time for which we won't try TFO");
160
161 static uint32_t tcp_ecn_timeout = 60;
162
163 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, ecn_timeout, CTLFLAG_RW | CTLFLAG_LOCKED,
164 &tcp_ecn_timeout, 60, "Initial minutes to wait before re-trying ECN");
165
166 static int disable_tcp_heuristics = 0;
167 SYSCTL_INT(_net_inet_tcp, OID_AUTO, disable_tcp_heuristics, CTLFLAG_RW | CTLFLAG_LOCKED,
168 &disable_tcp_heuristics, 0, "Set to 1, to disable all TCP heuristics (TFO, ECN, MPTCP)");
169
170 static uint32_t mptcp_version_timeout = 24 * 60;
171
172 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mptcp_version_timeout, CTLFLAG_RW | CTLFLAG_LOCKED,
173 &mptcp_version_timeout, 24 * 60, "Initial minutes to wait before re-trying MPTCP's preferred version");
174
175
176 static uint32_t
tcp_min_to_hz(uint32_t minutes)177 tcp_min_to_hz(uint32_t minutes)
178 {
179 if (minutes > 65536) {
180 return (uint32_t)65536 * 60 * TCP_RETRANSHZ;
181 }
182
183 return minutes * 60 * TCP_RETRANSHZ;
184 }
185
186 /*
187 * This number is coupled with tcp_ecn_timeout, because we want to prevent
188 * integer overflow. Need to find an unexpensive way to prevent integer overflow
189 * while still allowing a dynamic sysctl.
190 */
191 #define TCP_CACHE_OVERFLOW_PROTECT 9
192
193 /* Number of SYN-losses we accept */
194 #define TFO_MAX_COOKIE_LOSS 2
195 #define ECN_MAX_SYN_LOSS 2
196 #define MPTCP_MAX_SYN_LOSS 2
197 #define MPTCP_SUCCESS_TRIGGER 10
198 #define MPTCP_VERSION_MAX_FAIL 2
199 #define ECN_MAX_DROPRST 1
200 #define ECN_MAX_DROPRXMT 4
201 #define ECN_MAX_SYNRST 4
202
203 /* Flags for setting/unsetting loss-heuristics, limited to 4 bytes */
204 #define TCPCACHE_F_TFO_REQ 0x01
205 #define TCPCACHE_F_TFO_DATA 0x02
206 #define TCPCACHE_F_ECN 0x04
207 #define TCPCACHE_F_MPTCP 0x08
208 #define TCPCACHE_F_ECN_DROPRST 0x10
209 #define TCPCACHE_F_ECN_DROPRXMT 0x20
210 #define TCPCACHE_F_TFO_REQ_RST 0x40
211 #define TCPCACHE_F_TFO_DATA_RST 0x80
212 #define TCPCACHE_F_ECN_SYNRST 0x100
213
214 /* Always retry ECN after backing off to this level for some heuristics */
215 #define ECN_RETRY_LIMIT 9
216
217 #define TCP_CACHE_INC_IFNET_STAT(_ifp_, _af_, _stat_) { \
218 if ((_ifp_) != NULL) { \
219 if ((_af_) == AF_INET6) { \
220 (_ifp_)->if_ipv6_stat->_stat_++;\
221 } else { \
222 (_ifp_)->if_ipv4_stat->_stat_++;\
223 }\
224 }\
225 }
226
227 /*
228 * Round up to next higher power-of 2. See "Bit Twiddling Hacks".
229 *
230 * Might be worth moving this to a library so that others
231 * (e.g., scale_to_powerof2()) can use this as well instead of a while-loop.
232 */
233 static uint32_t
tcp_cache_roundup2(uint32_t a)234 tcp_cache_roundup2(uint32_t a)
235 {
236 a--;
237 a |= a >> 1;
238 a |= a >> 2;
239 a |= a >> 4;
240 a |= a >> 8;
241 a |= a >> 16;
242 a++;
243
244 return a;
245 }
246
247 static void
tcp_cache_hash_src(struct tcp_cache_key_src * tcks,struct tcp_heuristic_key * key)248 tcp_cache_hash_src(struct tcp_cache_key_src *tcks, struct tcp_heuristic_key *key)
249 {
250 struct ifnet *ifp = tcks->ifp;
251 uint8_t len = sizeof(key->thk_net_signature);
252 uint16_t flags;
253
254 if (tcks->af == AF_INET6) {
255 int ret;
256
257 key->thk_family = AF_INET6;
258 ret = ifnet_get_netsignature(ifp, AF_INET6, &len, &flags,
259 key->thk_net_signature);
260
261 /*
262 * ifnet_get_netsignature only returns EINVAL if ifn is NULL
263 * (we made sure that in the other cases it does not). So,
264 * in this case we should take the connection's address.
265 */
266 if (ret == ENOENT || ret == EINVAL) {
267 memcpy(&key->thk_ip.addr6, &tcks->laddr.addr6, sizeof(struct in6_addr));
268 }
269 } else {
270 int ret;
271
272 key->thk_family = AF_INET;
273 ret = ifnet_get_netsignature(ifp, AF_INET, &len, &flags,
274 key->thk_net_signature);
275
276 /*
277 * ifnet_get_netsignature only returns EINVAL if ifn is NULL
278 * (we made sure that in the other cases it does not). So,
279 * in this case we should take the connection's address.
280 */
281 if (ret == ENOENT || ret == EINVAL) {
282 memcpy(&key->thk_ip.addr, &tcks->laddr.addr, sizeof(struct in_addr));
283 }
284 }
285 }
286
287 static uint16_t
tcp_cache_hash(struct tcp_cache_key_src * tcks,struct tcp_cache_key * key)288 tcp_cache_hash(struct tcp_cache_key_src *tcks, struct tcp_cache_key *key)
289 {
290 uint32_t hash;
291
292 bzero(key, sizeof(struct tcp_cache_key));
293
294 tcp_cache_hash_src(tcks, &key->tck_src);
295
296 if (tcks->af == AF_INET6) {
297 key->tck_family = AF_INET6;
298 memcpy(&key->tck_dst.addr6, &tcks->faddr.addr6,
299 sizeof(struct in6_addr));
300 } else {
301 key->tck_family = AF_INET;
302 memcpy(&key->tck_dst.addr, &tcks->faddr.addr,
303 sizeof(struct in_addr));
304 }
305
306 hash = net_flowhash(key, sizeof(struct tcp_cache_key),
307 tcp_cache_hash_seed);
308
309 return (uint16_t)(hash & (tcp_cache_size - 1));
310 }
311
312 static void
tcp_cache_unlock(struct tcp_cache_head * head)313 tcp_cache_unlock(struct tcp_cache_head *head)
314 {
315 lck_mtx_unlock(&head->tch_mtx);
316 }
317
318 /*
319 * Make sure that everything that happens after tcp_getcache_with_lock()
320 * is short enough to justify that you hold the per-bucket lock!!!
321 *
322 * Otherwise, better build another lookup-function that does not hold the
323 * lock and you copy out the bits and bytes.
324 *
325 * That's why we provide the head as a "return"-pointer so that the caller
326 * can give it back to use for tcp_cache_unlock().
327 */
328 static struct tcp_cache *
tcp_getcache_with_lock(struct tcp_cache_key_src * tcks,int create,struct tcp_cache_head ** headarg)329 tcp_getcache_with_lock(struct tcp_cache_key_src *tcks,
330 int create, struct tcp_cache_head **headarg)
331 {
332 struct tcp_cache *__single tpcache = NULL;
333 struct tcp_cache_head *__single head;
334 struct tcp_cache_key key;
335 uint16_t hash;
336 int i = 0;
337
338 hash = tcp_cache_hash(tcks, &key);
339 head = &tcp_cache[hash];
340
341 lck_mtx_lock(&head->tch_mtx);
342
343 /*** First step: Look for the tcp_cache in our bucket ***/
344 SLIST_FOREACH(tpcache, &head->tcp_caches, list) {
345 if (memcmp(&tpcache->tc_key, &key, sizeof(key)) == 0) {
346 break;
347 }
348
349 i++;
350 }
351
352 /*** Second step: If it's not there, create/recycle it ***/
353 if ((tpcache == NULL) && create) {
354 if (i >= TCP_CACHE_BUCKET_SIZE) {
355 struct tcp_cache *oldest_cache = NULL;
356 uint32_t max_age = 0;
357
358 /* Look for the oldest tcp_cache in the bucket */
359 SLIST_FOREACH(tpcache, &head->tcp_caches, list) {
360 uint32_t age = tcp_now - tpcache->tc_last_access;
361 if (age >= max_age) {
362 max_age = age;
363 oldest_cache = tpcache;
364 }
365 }
366 VERIFY(oldest_cache != NULL);
367
368 tpcache = oldest_cache;
369
370 /* We recycle, thus let's indicate that there is no cookie */
371 tpcache->tc_tfo_cookie_len = 0;
372 } else {
373 /* Create a new cache and add it to the list */
374 tpcache = kalloc_type(struct tcp_cache, Z_NOPAGEWAIT | Z_ZERO);
375 if (tpcache == NULL) {
376 os_log_error(OS_LOG_DEFAULT, "%s could not allocate cache", __func__);
377 goto out_null;
378 }
379
380 tpcache->tc_mptcp_version = (uint8_t)mptcp_preferred_version;
381 tpcache->tc_mptcp_next_version_try = tcp_now;
382
383 SLIST_INSERT_HEAD(&head->tcp_caches, tpcache, list);
384 }
385
386 memcpy(&tpcache->tc_key, &key, sizeof(key));
387 }
388
389 if (tpcache == NULL) {
390 goto out_null;
391 }
392
393 /* Update timestamp for garbage collection purposes */
394 tpcache->tc_last_access = tcp_now;
395 *headarg = head;
396
397 return tpcache;
398
399 out_null:
400 tcp_cache_unlock(head);
401 return NULL;
402 }
403
404 static void
tcp_cache_key_src_create(struct tcpcb * tp,struct tcp_cache_key_src * tcks)405 tcp_cache_key_src_create(struct tcpcb *tp, struct tcp_cache_key_src *tcks)
406 {
407 struct inpcb *inp = tp->t_inpcb;
408 memset(tcks, 0, sizeof(*tcks));
409
410 tcks->ifp = inp->inp_last_outifp;
411
412 if (inp->inp_vflag & INP_IPV6) {
413 memcpy(&tcks->laddr.addr6, &inp->in6p_laddr, sizeof(struct in6_addr));
414 memcpy(&tcks->faddr.addr6, &inp->in6p_faddr, sizeof(struct in6_addr));
415 tcks->af = AF_INET6;
416 } else {
417 memcpy(&tcks->laddr.addr, &inp->inp_laddr, sizeof(struct in_addr));
418 memcpy(&tcks->faddr.addr, &inp->inp_faddr, sizeof(struct in_addr));
419 tcks->af = AF_INET;
420 }
421
422 return;
423 }
424
425 static void
mptcp_version_cache_key_src_init(struct sockaddr * dst,struct tcp_cache_key_src * tcks)426 mptcp_version_cache_key_src_init(struct sockaddr *dst, struct tcp_cache_key_src *tcks)
427 {
428 memset(tcks, 0, sizeof(*tcks));
429
430 if (dst->sa_family == AF_INET) {
431 memcpy(&tcks->faddr.addr, &SIN(dst)->sin_addr, sizeof(struct in_addr));
432 tcks->af = AF_INET;
433 } else {
434 memcpy(&tcks->faddr.addr6, &SIN6(dst)->sin6_addr, sizeof(struct in6_addr));
435 tcks->af = AF_INET6;
436 }
437
438 return;
439 }
440
441 static void
tcp_cache_set_cookie_common(struct tcp_cache_key_src * tcks,u_char * __counted_by (len)cookie,uint8_t len)442 tcp_cache_set_cookie_common(struct tcp_cache_key_src *tcks, u_char *__counted_by(len) cookie, uint8_t len)
443 {
444 struct tcp_cache_head *__single head;
445 struct tcp_cache *__single tpcache;
446
447 /* Call lookup/create function */
448 tpcache = tcp_getcache_with_lock(tcks, 1, &head);
449 if (tpcache == NULL) {
450 return;
451 }
452
453 tpcache->tc_tfo_cookie_len = len > TFO_COOKIE_LEN_MAX ?
454 TFO_COOKIE_LEN_MAX : len;
455 memcpy(tpcache->tc_tfo_cookie, cookie, tpcache->tc_tfo_cookie_len);
456
457 tcp_cache_unlock(head);
458 }
459
460 void
tcp_cache_set_cookie(struct tcpcb * tp,u_char * __counted_by (len)cookie,uint8_t len)461 tcp_cache_set_cookie(struct tcpcb *tp, u_char *__counted_by(len) cookie, uint8_t len)
462 {
463 struct tcp_cache_key_src tcks;
464
465 tcp_cache_key_src_create(tp, &tcks);
466 tcp_cache_set_cookie_common(&tcks, cookie, len);
467 }
468
469 static int
tcp_cache_get_cookie_common(struct tcp_cache_key_src * tcks,u_char * __counted_by (maxlen)cookie,uint8_t maxlen,uint8_t * len)470 tcp_cache_get_cookie_common(struct tcp_cache_key_src *tcks,
471 u_char *__counted_by(maxlen) cookie, uint8_t maxlen, uint8_t *len)
472 {
473 #pragma unused(maxlen)
474 struct tcp_cache_head *__single head;
475 struct tcp_cache *__single tpcache;
476
477 /* Call lookup/create function */
478 tpcache = tcp_getcache_with_lock(tcks, 1, &head);
479 if (tpcache == NULL) {
480 return 0;
481 }
482
483 if (tpcache->tc_tfo_cookie_len == 0) {
484 tcp_cache_unlock(head);
485 return 0;
486 }
487
488 /*
489 * Not enough space - this should never happen as it has been checked
490 * in tcp_tfo_check. So, fail here!
491 */
492 VERIFY(tpcache->tc_tfo_cookie_len <= *len);
493
494 memcpy(cookie, tpcache->tc_tfo_cookie, tpcache->tc_tfo_cookie_len);
495 *len = tpcache->tc_tfo_cookie_len;
496
497 tcp_cache_unlock(head);
498
499 return 1;
500 }
501
502 /*
503 * Get the cookie related to 'tp', and copy it into 'cookie', provided that len
504 * is big enough (len designates the available memory.
505 * Upon return, 'len' is set to the cookie's length.
506 *
507 * Returns 0 if we should request a cookie.
508 * Returns 1 if the cookie has been found and written.
509 */
510 int
tcp_cache_get_cookie(struct tcpcb * tp,u_char * __counted_by (maxlen)cookie,uint8_t maxlen,uint8_t * len)511 tcp_cache_get_cookie(struct tcpcb *tp, u_char *__counted_by(maxlen) cookie,
512 uint8_t maxlen, uint8_t *len)
513 {
514 struct tcp_cache_key_src tcks;
515
516 tcp_cache_key_src_create(tp, &tcks);
517 return tcp_cache_get_cookie_common(&tcks, cookie, maxlen, len);
518 }
519
520 static unsigned int
tcp_cache_get_cookie_len_common(struct tcp_cache_key_src * tcks)521 tcp_cache_get_cookie_len_common(struct tcp_cache_key_src *tcks)
522 {
523 struct tcp_cache_head *__single head;
524 struct tcp_cache *__single tpcache;
525 unsigned int cookie_len;
526
527 /* Call lookup/create function */
528 tpcache = tcp_getcache_with_lock(tcks, 1, &head);
529 if (tpcache == NULL) {
530 return 0;
531 }
532
533 cookie_len = tpcache->tc_tfo_cookie_len;
534
535 tcp_cache_unlock(head);
536
537 return cookie_len;
538 }
539
540 unsigned int
tcp_cache_get_cookie_len(struct tcpcb * tp)541 tcp_cache_get_cookie_len(struct tcpcb *tp)
542 {
543 struct tcp_cache_key_src tcks;
544
545 tcp_cache_key_src_create(tp, &tcks);
546 return tcp_cache_get_cookie_len_common(&tcks);
547 }
548
549 /*
550 * @return:
551 * 0 MPTCP_VERSION_0
552 * 1 MPTCP_VERSION_1
553 */
554 uint8_t
tcp_cache_get_mptcp_version(struct sockaddr * dst)555 tcp_cache_get_mptcp_version(struct sockaddr *dst)
556 {
557 struct tcp_cache_key_src tcks;
558 mptcp_version_cache_key_src_init(dst, &tcks);
559 uint8_t version = (uint8_t) mptcp_preferred_version;
560
561 struct tcp_cache_head *__single head;
562 struct tcp_cache *__single tpcache;
563
564 /* Call lookup/create function */
565 tpcache = tcp_getcache_with_lock(&tcks, 1, &head);
566 if (tpcache == NULL) {
567 return version;
568 }
569
570 version = tpcache->tc_mptcp_version;
571
572 /* Let's see if we should try the preferred version again */
573 if (!tpcache->tc_mptcp_version_confirmed &&
574 version != mptcp_preferred_version &&
575 TSTMP_GEQ(tcp_now, tpcache->tc_mptcp_next_version_try)) {
576 version = (uint8_t) mptcp_preferred_version;
577 }
578
579 tcp_cache_unlock(head);
580 return version;
581 }
582
583 void
tcp_cache_update_mptcp_version(struct tcpcb * tp,boolean_t succeeded)584 tcp_cache_update_mptcp_version(struct tcpcb *tp, boolean_t succeeded)
585 {
586 uint8_t version = tptomptp(tp)->mpt_version;
587 struct inpcb *inp = tp->t_inpcb;
588 struct tcp_cache_key_src tcks;
589 struct tcp_cache_head *__single head;
590 struct tcp_cache *__single tpcache;
591
592 if (inp->inp_vflag & INP_IPV6) {
593 struct sockaddr_in6 dst = {
594 .sin6_len = sizeof(struct sockaddr_in6),
595 .sin6_family = AF_INET6,
596 .sin6_addr = inp->in6p_faddr,
597 };
598 mptcp_version_cache_key_src_init(SA(&dst), &tcks);
599 } else {
600 struct sockaddr_in dst = {
601 .sin_len = sizeof(struct sockaddr_in),
602 .sin_family = AF_INET,
603 .sin_addr = inp->inp_faddr,
604 };
605 mptcp_version_cache_key_src_init(SA(&dst), &tcks);
606 }
607
608 /* Call lookup/create function */
609 tpcache = tcp_getcache_with_lock(&tcks, 1, &head);
610 if (tpcache == NULL) {
611 return;
612 }
613
614 /* We are still in probing phase */
615 if (tpcache->tc_mptcp_version_confirmed) {
616 goto exit;
617 }
618
619 if (succeeded) {
620 if (version == (uint8_t)mptcp_preferred_version) {
621 /* Preferred version succeeded - make it sticky */
622 tpcache->tc_mptcp_version_confirmed = true;
623 tpcache->tc_mptcp_version = version;
624 } else {
625 /* If we are past the next version try, set it
626 * so that we try preferred again in 24h
627 */
628 if (TSTMP_GEQ(tcp_now, tpcache->tc_mptcp_next_version_try)) {
629 tpcache->tc_mptcp_next_version_try = tcp_now + tcp_min_to_hz(mptcp_version_timeout);
630 }
631 }
632 } else {
633 if (version == (uint8_t)mptcp_preferred_version) {
634 /* Preferred version failed - try the other version */
635 tpcache->tc_mptcp_version = version == MPTCP_VERSION_0 ? MPTCP_VERSION_1 : MPTCP_VERSION_0;
636 }
637 /* Preferred version failed - make sure we give the preferred another
638 * shot in 24h.
639 */
640 if (TSTMP_GEQ(tcp_now, tpcache->tc_mptcp_next_version_try)) {
641 tpcache->tc_mptcp_next_version_try = tcp_now + tcp_min_to_hz(mptcp_version_timeout);
642 }
643 }
644
645 exit:
646 tcp_cache_unlock(head);
647 }
648
649 static uint16_t
tcp_heuristics_hash(struct tcp_cache_key_src * tcks,struct tcp_heuristic_key * key)650 tcp_heuristics_hash(struct tcp_cache_key_src *tcks, struct tcp_heuristic_key *key)
651 {
652 uint32_t hash;
653
654 bzero(key, sizeof(struct tcp_heuristic_key));
655
656 tcp_cache_hash_src(tcks, key);
657
658 hash = net_flowhash(key, sizeof(struct tcp_heuristic_key),
659 tcp_cache_hash_seed);
660
661 return (uint16_t)(hash & (tcp_cache_size - 1));
662 }
663
664 static void
tcp_heuristic_unlock(struct tcp_heuristics_head * head)665 tcp_heuristic_unlock(struct tcp_heuristics_head *head)
666 {
667 lck_mtx_unlock(&head->thh_mtx);
668 }
669
670 /*
671 * Make sure that everything that happens after tcp_getheuristic_with_lock()
672 * is short enough to justify that you hold the per-bucket lock!!!
673 *
674 * Otherwise, better build another lookup-function that does not hold the
675 * lock and you copy out the bits and bytes.
676 *
677 * That's why we provide the head as a "return"-pointer so that the caller
678 * can give it back to use for tcp_heur_unlock().
679 *
680 *
681 * ToDo - way too much code-duplication. We should create an interface to handle
682 * bucketized hashtables with recycling of the oldest element.
683 */
684 static struct tcp_heuristic *
tcp_getheuristic_with_lock(struct tcp_cache_key_src * tcks,int create,struct tcp_heuristics_head ** headarg)685 tcp_getheuristic_with_lock(struct tcp_cache_key_src *tcks,
686 int create, struct tcp_heuristics_head **headarg)
687 {
688 struct tcp_heuristic *__single tpheur = NULL;
689 struct tcp_heuristics_head *__single head;
690 struct tcp_heuristic_key key;
691 uint16_t hash;
692 int i = 0;
693
694 hash = tcp_heuristics_hash(tcks, &key);
695 head = &tcp_heuristics[hash];
696
697 lck_mtx_lock(&head->thh_mtx);
698
699 /*** First step: Look for the tcp_heur in our bucket ***/
700 SLIST_FOREACH(tpheur, &head->tcp_heuristics, list) {
701 if (memcmp(&tpheur->th_key, &key, sizeof(key)) == 0) {
702 break;
703 }
704
705 i++;
706 }
707
708 /*** Second step: If it's not there, create/recycle it ***/
709 if ((tpheur == NULL) && create) {
710 if (i >= TCP_CACHE_BUCKET_SIZE) {
711 struct tcp_heuristic *__single oldest_heur = NULL;
712 uint32_t max_age = 0;
713
714 /* Look for the oldest tcp_heur in the bucket */
715 SLIST_FOREACH(tpheur, &head->tcp_heuristics, list) {
716 uint32_t age = tcp_now - tpheur->th_last_access;
717 if (age >= max_age) {
718 max_age = age;
719 oldest_heur = tpheur;
720 }
721 }
722 VERIFY(oldest_heur != NULL);
723
724 tpheur = oldest_heur;
725
726 /* We recycle - set everything to 0 */
727 uint8_t *ptr = (uint8_t *)(struct tcp_heuristic *__indexable)tpheur;
728 const size_t preamble = offsetof(struct tcp_heuristic, th_val_start);
729 const size_t size = sizeof(struct tcp_heuristic) - preamble;
730 bzero(ptr + preamble, size);
731 } else {
732 /* Create a new heuristic and add it to the list */
733 tpheur = kalloc_type(struct tcp_heuristic, Z_NOPAGEWAIT | Z_ZERO);
734 if (tpheur == NULL) {
735 os_log_error(OS_LOG_DEFAULT, "%s could not allocate heuristic", __func__);
736 goto out_null;
737 }
738
739 SLIST_INSERT_HEAD(&head->tcp_heuristics, tpheur, list);
740 }
741
742 /*
743 * Set to tcp_now, to make sure it won't be > than tcp_now in the
744 * near future.
745 */
746 tpheur->th_ecn_backoff = tcp_now;
747 tpheur->th_tfo_backoff_until = tcp_now;
748 tpheur->th_mptcp_backoff = tcp_now;
749 tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);
750
751 memcpy(&tpheur->th_key, &key, sizeof(key));
752 }
753
754 if (tpheur == NULL) {
755 goto out_null;
756 }
757
758 /* Update timestamp for garbage collection purposes */
759 tpheur->th_last_access = tcp_now;
760 *headarg = head;
761
762 return tpheur;
763
764 out_null:
765 tcp_heuristic_unlock(head);
766 return NULL;
767 }
768
769 static void
tcp_heuristic_reset_counters(struct tcp_cache_key_src * tcks,uint8_t flags)770 tcp_heuristic_reset_counters(struct tcp_cache_key_src *tcks, uint8_t flags)
771 {
772 struct tcp_heuristics_head *__single head;
773 struct tcp_heuristic *__single tpheur;
774
775 /*
776 * Always create heuristics here because MPTCP needs to write success
777 * into it. Thus, we always end up creating them.
778 */
779 tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);
780 if (tpheur == NULL) {
781 return;
782 }
783
784 if (flags & TCPCACHE_F_TFO_DATA) {
785 if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) {
786 os_log(OS_LOG_DEFAULT, "%s: Resetting TFO-data loss to 0 from %u on heur %lx\n",
787 __func__, tpheur->th_tfo_data_loss, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
788 }
789 tpheur->th_tfo_data_loss = 0;
790 }
791
792 if (flags & TCPCACHE_F_TFO_REQ) {
793 if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) {
794 os_log(OS_LOG_DEFAULT, "%s: Resetting TFO-req loss to 0 from %u on heur %lx\n",
795 __func__, tpheur->th_tfo_req_loss, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
796 }
797 tpheur->th_tfo_req_loss = 0;
798 }
799
800 if (flags & TCPCACHE_F_TFO_DATA_RST) {
801 if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) {
802 os_log(OS_LOG_DEFAULT, "%s: Resetting TFO-data RST to 0 from %u on heur %lx\n",
803 __func__, tpheur->th_tfo_data_rst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
804 }
805 tpheur->th_tfo_data_rst = 0;
806 }
807
808 if (flags & TCPCACHE_F_TFO_REQ_RST) {
809 if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) {
810 os_log(OS_LOG_DEFAULT, "%s: Resetting TFO-req RST to 0 from %u on heur %lx\n",
811 __func__, tpheur->th_tfo_req_rst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
812 }
813 tpheur->th_tfo_req_rst = 0;
814 }
815
816 if (flags & TCPCACHE_F_ECN) {
817 if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS || tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) {
818 os_log(OS_LOG_DEFAULT, "%s: Resetting ECN-loss to 0 from %u and synrst from %u on heur %lx\n",
819 __func__, tpheur->th_ecn_loss, tpheur->th_ecn_synrst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
820 }
821 tpheur->th_ecn_loss = 0;
822 tpheur->th_ecn_synrst = 0;
823 }
824
825 if (flags & TCPCACHE_F_MPTCP) {
826 tpheur->th_mptcp_loss = 0;
827 if (tpheur->th_mptcp_success < MPTCP_SUCCESS_TRIGGER) {
828 tpheur->th_mptcp_success++;
829
830 if (tpheur->th_mptcp_success == MPTCP_SUCCESS_TRIGGER) {
831 os_log(mptcp_log_handle, "%s disabling heuristics for 12 hours", __func__);
832 tpheur->th_mptcp_heuristic_disabled = 1;
833 /* Disable heuristics for 12 hours */
834 tpheur->th_mptcp_backoff = tcp_now + tcp_min_to_hz(tcp_ecn_timeout * 12);
835 }
836 }
837 }
838
839 tcp_heuristic_unlock(head);
840 }
841
842 void
tcp_heuristic_tfo_success(struct tcpcb * tp)843 tcp_heuristic_tfo_success(struct tcpcb *tp)
844 {
845 struct tcp_cache_key_src tcks;
846 uint8_t flag = 0;
847
848 tcp_cache_key_src_create(tp, &tcks);
849
850 if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) {
851 flag = (TCPCACHE_F_TFO_DATA | TCPCACHE_F_TFO_REQ |
852 TCPCACHE_F_TFO_DATA_RST | TCPCACHE_F_TFO_REQ_RST);
853 }
854 if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) {
855 flag = (TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_REQ_RST);
856 }
857
858 tcp_heuristic_reset_counters(&tcks, flag);
859 }
860
861 void
tcp_heuristic_mptcp_success(struct tcpcb * tp)862 tcp_heuristic_mptcp_success(struct tcpcb *tp)
863 {
864 struct tcp_cache_key_src tcks;
865
866 tcp_cache_key_src_create(tp, &tcks);
867 tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_MPTCP);
868 }
869
870 void
tcp_heuristic_ecn_success(struct tcpcb * tp)871 tcp_heuristic_ecn_success(struct tcpcb *tp)
872 {
873 struct tcp_cache_key_src tcks;
874
875 tcp_cache_key_src_create(tp, &tcks);
876 tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_ECN);
877 }
878
879 static void
__tcp_heuristic_tfo_middlebox_common(struct tcp_heuristic * tpheur)880 __tcp_heuristic_tfo_middlebox_common(struct tcp_heuristic *tpheur)
881 {
882 if (tpheur->th_tfo_in_backoff) {
883 return;
884 }
885
886 tpheur->th_tfo_in_backoff = 1;
887
888 if (tpheur->th_tfo_enabled_time) {
889 uint32_t old_backoff = tpheur->th_tfo_backoff;
890
891 tpheur->th_tfo_backoff -= (tcp_now - tpheur->th_tfo_enabled_time);
892 if (tpheur->th_tfo_backoff > old_backoff) {
893 tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);
894 }
895 }
896
897 tpheur->th_tfo_backoff_until = tcp_now + tpheur->th_tfo_backoff;
898
899 /* Then, increase the backoff time */
900 tpheur->th_tfo_backoff *= 2;
901
902 if (tpheur->th_tfo_backoff > tcp_min_to_hz(tcp_backoff_maximum)) {
903 tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);
904 }
905
906 os_log(OS_LOG_DEFAULT, "%s disable TFO until %u now %u on %lx\n", __func__,
907 tpheur->th_tfo_backoff_until, tcp_now, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
908 }
909
910 static void
tcp_heuristic_tfo_middlebox_common(struct tcp_cache_key_src * tcks)911 tcp_heuristic_tfo_middlebox_common(struct tcp_cache_key_src *tcks)
912 {
913 struct tcp_heuristics_head *__single head;
914 struct tcp_heuristic *__single tpheur;
915
916 tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);
917 if (tpheur == NULL) {
918 return;
919 }
920
921 __tcp_heuristic_tfo_middlebox_common(tpheur);
922
923 tcp_heuristic_unlock(head);
924 }
925
926 static void
tcp_heuristic_inc_counters(struct tcp_cache_key_src * tcks,uint32_t flags)927 tcp_heuristic_inc_counters(struct tcp_cache_key_src *tcks,
928 uint32_t flags)
929 {
930 struct tcp_heuristics_head *__single head;
931 struct tcp_heuristic *__single tpheur;
932
933 tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);
934 if (tpheur == NULL) {
935 return;
936 }
937
938 /* Limit to prevent integer-overflow during exponential backoff */
939 if ((flags & TCPCACHE_F_TFO_DATA) && tpheur->th_tfo_data_loss < TCP_CACHE_OVERFLOW_PROTECT) {
940 tpheur->th_tfo_data_loss++;
941
942 if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) {
943 __tcp_heuristic_tfo_middlebox_common(tpheur);
944 }
945 }
946
947 if ((flags & TCPCACHE_F_TFO_REQ) && tpheur->th_tfo_req_loss < TCP_CACHE_OVERFLOW_PROTECT) {
948 tpheur->th_tfo_req_loss++;
949
950 if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) {
951 __tcp_heuristic_tfo_middlebox_common(tpheur);
952 }
953 }
954
955 if ((flags & TCPCACHE_F_TFO_DATA_RST) && tpheur->th_tfo_data_rst < TCP_CACHE_OVERFLOW_PROTECT) {
956 tpheur->th_tfo_data_rst++;
957
958 if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) {
959 __tcp_heuristic_tfo_middlebox_common(tpheur);
960 }
961 }
962
963 if ((flags & TCPCACHE_F_TFO_REQ_RST) && tpheur->th_tfo_req_rst < TCP_CACHE_OVERFLOW_PROTECT) {
964 tpheur->th_tfo_req_rst++;
965
966 if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) {
967 __tcp_heuristic_tfo_middlebox_common(tpheur);
968 }
969 }
970
971 if ((flags & TCPCACHE_F_ECN) &&
972 tpheur->th_ecn_loss < TCP_CACHE_OVERFLOW_PROTECT &&
973 TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {
974 tpheur->th_ecn_loss++;
975 if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS) {
976 tcpstat.tcps_ecn_fallback_synloss++;
977 TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, ecn_fallback_synloss);
978 tpheur->th_ecn_backoff = tcp_now +
979 (tcp_min_to_hz(tcp_ecn_timeout) <<
980 (tpheur->th_ecn_loss - ECN_MAX_SYN_LOSS));
981
982 os_log(OS_LOG_DEFAULT, "%s disable ECN until %u now %u on %lx for SYN-loss\n",
983 __func__, tpheur->th_ecn_backoff, tcp_now,
984 (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
985 }
986 }
987
988 if ((flags & TCPCACHE_F_MPTCP) &&
989 tpheur->th_mptcp_loss < TCP_CACHE_OVERFLOW_PROTECT &&
990 tpheur->th_mptcp_heuristic_disabled == 0) {
991 tpheur->th_mptcp_loss++;
992 if (tpheur->th_mptcp_loss >= MPTCP_MAX_SYN_LOSS) {
993 /*
994 * Yes, we take tcp_ecn_timeout, to avoid adding yet
995 * another sysctl that is just used for testing.
996 */
997 tpheur->th_mptcp_backoff = tcp_now +
998 (tcp_min_to_hz(tcp_ecn_timeout) <<
999 (tpheur->th_mptcp_loss - MPTCP_MAX_SYN_LOSS));
1000 tpheur->th_mptcp_in_backoff = 1;
1001
1002 os_log(OS_LOG_DEFAULT, "%s disable MPTCP until %u now %u on %lx\n",
1003 __func__, tpheur->th_mptcp_backoff, tcp_now,
1004 (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
1005 }
1006 }
1007
1008 if ((flags & TCPCACHE_F_ECN_DROPRST) &&
1009 tpheur->th_ecn_droprst < TCP_CACHE_OVERFLOW_PROTECT &&
1010 TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {
1011 tpheur->th_ecn_droprst++;
1012 if (tpheur->th_ecn_droprst >= ECN_MAX_DROPRST) {
1013 tcpstat.tcps_ecn_fallback_droprst++;
1014 TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af,
1015 ecn_fallback_droprst);
1016 tpheur->th_ecn_backoff = tcp_now +
1017 (tcp_min_to_hz(tcp_ecn_timeout) <<
1018 (tpheur->th_ecn_droprst - ECN_MAX_DROPRST));
1019
1020 os_log(OS_LOG_DEFAULT, "%s disable ECN until %u now %u on %lx for drop-RST\n",
1021 __func__, tpheur->th_ecn_backoff, tcp_now,
1022 (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
1023 }
1024 }
1025
1026 if ((flags & TCPCACHE_F_ECN_DROPRXMT) &&
1027 tpheur->th_ecn_droprxmt < TCP_CACHE_OVERFLOW_PROTECT &&
1028 TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {
1029 tpheur->th_ecn_droprxmt++;
1030 if (tpheur->th_ecn_droprxmt >= ECN_MAX_DROPRXMT) {
1031 tcpstat.tcps_ecn_fallback_droprxmt++;
1032 TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af,
1033 ecn_fallback_droprxmt);
1034 tpheur->th_ecn_backoff = tcp_now +
1035 (tcp_min_to_hz(tcp_ecn_timeout) <<
1036 (tpheur->th_ecn_droprxmt - ECN_MAX_DROPRXMT));
1037
1038 os_log(OS_LOG_DEFAULT, "%s disable ECN until %u now %u on %lx for drop-Rxmit\n",
1039 __func__, tpheur->th_ecn_backoff, tcp_now,
1040 (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
1041 }
1042 }
1043 if ((flags & TCPCACHE_F_ECN_SYNRST) &&
1044 tpheur->th_ecn_synrst < TCP_CACHE_OVERFLOW_PROTECT) {
1045 tpheur->th_ecn_synrst++;
1046 if (tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) {
1047 tcpstat.tcps_ecn_fallback_synrst++;
1048 TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af,
1049 ecn_fallback_synrst);
1050 tpheur->th_ecn_backoff = tcp_now +
1051 (tcp_min_to_hz(tcp_ecn_timeout) <<
1052 (tpheur->th_ecn_synrst - ECN_MAX_SYNRST));
1053
1054 os_log(OS_LOG_DEFAULT, "%s disable ECN until %u now %u on %lx for SYN-RST\n",
1055 __func__, tpheur->th_ecn_backoff, tcp_now,
1056 (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
1057 }
1058 }
1059 tcp_heuristic_unlock(head);
1060 }
1061
1062 void
tcp_heuristic_tfo_loss(struct tcpcb * tp)1063 tcp_heuristic_tfo_loss(struct tcpcb *tp)
1064 {
1065 struct tcp_cache_key_src tcks;
1066 uint32_t flag = 0;
1067
1068 if (symptoms_is_wifi_lossy() &&
1069 IFNET_IS_WIFI(tp->t_inpcb->inp_last_outifp)) {
1070 return;
1071 }
1072
1073 tcp_cache_key_src_create(tp, &tcks);
1074
1075 if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) {
1076 flag = (TCPCACHE_F_TFO_DATA | TCPCACHE_F_TFO_REQ);
1077 }
1078 if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) {
1079 flag = TCPCACHE_F_TFO_REQ;
1080 }
1081
1082 tcp_heuristic_inc_counters(&tcks, flag);
1083 }
1084
1085 void
tcp_heuristic_tfo_rst(struct tcpcb * tp)1086 tcp_heuristic_tfo_rst(struct tcpcb *tp)
1087 {
1088 struct tcp_cache_key_src tcks;
1089 uint32_t flag = 0;
1090
1091 tcp_cache_key_src_create(tp, &tcks);
1092
1093 if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) {
1094 flag = (TCPCACHE_F_TFO_DATA_RST | TCPCACHE_F_TFO_REQ_RST);
1095 }
1096 if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) {
1097 flag = TCPCACHE_F_TFO_REQ_RST;
1098 }
1099
1100 tcp_heuristic_inc_counters(&tcks, flag);
1101 }
1102
1103 void
tcp_heuristic_mptcp_loss(struct tcpcb * tp)1104 tcp_heuristic_mptcp_loss(struct tcpcb *tp)
1105 {
1106 struct tcp_cache_key_src tcks;
1107
1108 if (symptoms_is_wifi_lossy() &&
1109 IFNET_IS_WIFI(tp->t_inpcb->inp_last_outifp)) {
1110 return;
1111 }
1112
1113 tcp_cache_key_src_create(tp, &tcks);
1114
1115 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_MPTCP);
1116 }
1117
1118 void
tcp_heuristic_ecn_loss(struct tcpcb * tp)1119 tcp_heuristic_ecn_loss(struct tcpcb *tp)
1120 {
1121 struct tcp_cache_key_src tcks;
1122
1123 if (symptoms_is_wifi_lossy() &&
1124 IFNET_IS_WIFI(tp->t_inpcb->inp_last_outifp)) {
1125 return;
1126 }
1127
1128 tcp_cache_key_src_create(tp, &tcks);
1129
1130 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN);
1131 }
1132
1133 void
tcp_heuristic_ecn_droprst(struct tcpcb * tp)1134 tcp_heuristic_ecn_droprst(struct tcpcb *tp)
1135 {
1136 struct tcp_cache_key_src tcks;
1137
1138 tcp_cache_key_src_create(tp, &tcks);
1139
1140 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRST);
1141 }
1142
1143 void
tcp_heuristic_ecn_droprxmt(struct tcpcb * tp)1144 tcp_heuristic_ecn_droprxmt(struct tcpcb *tp)
1145 {
1146 struct tcp_cache_key_src tcks;
1147
1148 tcp_cache_key_src_create(tp, &tcks);
1149
1150 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRXMT);
1151 }
1152
1153 void
tcp_heuristic_ecn_synrst(struct tcpcb * tp)1154 tcp_heuristic_ecn_synrst(struct tcpcb *tp)
1155 {
1156 struct tcp_cache_key_src tcks;
1157
1158 tcp_cache_key_src_create(tp, &tcks);
1159
1160 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_SYNRST);
1161 }
1162
1163 void
tcp_heuristic_tfo_middlebox(struct tcpcb * tp)1164 tcp_heuristic_tfo_middlebox(struct tcpcb *tp)
1165 {
1166 struct tcp_cache_key_src tcks;
1167
1168 tp->t_tfo_flags |= TFO_F_HEURISTIC_DONE;
1169
1170 tcp_cache_key_src_create(tp, &tcks);
1171 tcp_heuristic_tfo_middlebox_common(&tcks);
1172 }
1173
1174 static void
tcp_heuristic_ecn_aggressive_common(struct tcp_cache_key_src * tcks)1175 tcp_heuristic_ecn_aggressive_common(struct tcp_cache_key_src *tcks)
1176 {
1177 struct tcp_heuristics_head *__single head;
1178 struct tcp_heuristic *__single tpheur;
1179
1180 tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);
1181 if (tpheur == NULL) {
1182 return;
1183 }
1184
1185 if (TSTMP_GT(tpheur->th_ecn_backoff, tcp_now)) {
1186 /* We are already in aggressive mode */
1187 tcp_heuristic_unlock(head);
1188 return;
1189 }
1190
1191 /* Must be done before, otherwise we will start off with expo-backoff */
1192 tpheur->th_ecn_backoff = tcp_now +
1193 (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_aggressive));
1194
1195 /*
1196 * Ugly way to prevent integer overflow... limit to prevent in
1197 * overflow during exp. backoff.
1198 */
1199 if (tpheur->th_ecn_aggressive < TCP_CACHE_OVERFLOW_PROTECT) {
1200 tpheur->th_ecn_aggressive++;
1201 }
1202
1203 tcp_heuristic_unlock(head);
1204
1205 os_log(OS_LOG_DEFAULT, "%s disable ECN until %u now %u on %lx\n", __func__,
1206 tpheur->th_ecn_backoff, tcp_now, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));
1207 }
1208
1209 void
tcp_heuristic_ecn_aggressive(struct tcpcb * tp)1210 tcp_heuristic_ecn_aggressive(struct tcpcb *tp)
1211 {
1212 struct tcp_cache_key_src tcks;
1213
1214 tcp_cache_key_src_create(tp, &tcks);
1215 tcp_heuristic_ecn_aggressive_common(&tcks);
1216 }
1217
1218 static boolean_t
tcp_heuristic_do_tfo_common(struct tcp_cache_key_src * tcks)1219 tcp_heuristic_do_tfo_common(struct tcp_cache_key_src *tcks)
1220 {
1221 struct tcp_heuristics_head *__single head;
1222 struct tcp_heuristic *__single tpheur;
1223
1224 if (disable_tcp_heuristics) {
1225 return TRUE;
1226 }
1227
1228 /* Get the tcp-heuristic. */
1229 tpheur = tcp_getheuristic_with_lock(tcks, 0, &head);
1230 if (tpheur == NULL) {
1231 return TRUE;
1232 }
1233
1234 if (tpheur->th_tfo_in_backoff == 0) {
1235 goto tfo_ok;
1236 }
1237
1238 if (TSTMP_GT(tcp_now, tpheur->th_tfo_backoff_until)) {
1239 tpheur->th_tfo_in_backoff = 0;
1240 tpheur->th_tfo_enabled_time = tcp_now;
1241
1242 goto tfo_ok;
1243 }
1244
1245 tcp_heuristic_unlock(head);
1246 return FALSE;
1247
1248 tfo_ok:
1249 tcp_heuristic_unlock(head);
1250 return TRUE;
1251 }
1252
1253 boolean_t
tcp_heuristic_do_tfo(struct tcpcb * tp)1254 tcp_heuristic_do_tfo(struct tcpcb *tp)
1255 {
1256 struct tcp_cache_key_src tcks;
1257
1258 tcp_cache_key_src_create(tp, &tcks);
1259 if (tcp_heuristic_do_tfo_common(&tcks)) {
1260 return TRUE;
1261 }
1262
1263 return FALSE;
1264 }
1265 /*
1266 * @return:
1267 * 0 Enable MPTCP (we are still discovering middleboxes)
1268 * -1 Enable MPTCP (heuristics have been temporarily disabled)
1269 * 1 Disable MPTCP
1270 */
1271 int
tcp_heuristic_do_mptcp(struct tcpcb * tp)1272 tcp_heuristic_do_mptcp(struct tcpcb *tp)
1273 {
1274 struct tcp_cache_key_src tcks;
1275 struct tcp_heuristics_head *__single head = NULL;
1276 struct tcp_heuristic *__single tpheur;
1277 int ret = 0;
1278
1279 if (disable_tcp_heuristics ||
1280 (tptomptp(tp)->mpt_mpte->mpte_flags & MPTE_FORCE_ENABLE)) {
1281 return 0;
1282 }
1283
1284 tcp_cache_key_src_create(tp, &tcks);
1285
1286 /* Get the tcp-heuristic. */
1287 tpheur = tcp_getheuristic_with_lock(&tcks, 0, &head);
1288 if (tpheur == NULL) {
1289 return 0;
1290 }
1291
1292 if (tpheur->th_mptcp_in_backoff == 0 ||
1293 tpheur->th_mptcp_heuristic_disabled == 1) {
1294 goto mptcp_ok;
1295 }
1296
1297 if (TSTMP_GT(tpheur->th_mptcp_backoff, tcp_now)) {
1298 goto fallback;
1299 }
1300
1301 tpheur->th_mptcp_in_backoff = 0;
1302
1303 mptcp_ok:
1304 if (tpheur->th_mptcp_heuristic_disabled) {
1305 ret = -1;
1306
1307 if (TSTMP_GT(tcp_now, tpheur->th_mptcp_backoff)) {
1308 tpheur->th_mptcp_heuristic_disabled = 0;
1309 tpheur->th_mptcp_success = 0;
1310 }
1311 }
1312
1313 tcp_heuristic_unlock(head);
1314 return ret;
1315
1316 fallback:
1317 if (head) {
1318 tcp_heuristic_unlock(head);
1319 }
1320
1321 if (tptomptp(tp)->mpt_mpte->mpte_flags & MPTE_FIRSTPARTY) {
1322 tcpstat.tcps_mptcp_fp_heuristic_fallback++;
1323 } else {
1324 tcpstat.tcps_mptcp_heuristic_fallback++;
1325 }
1326
1327 return 1;
1328 }
1329
1330 static boolean_t
tcp_heuristic_do_ecn_common(struct tcp_cache_key_src * tcks)1331 tcp_heuristic_do_ecn_common(struct tcp_cache_key_src *tcks)
1332 {
1333 struct tcp_heuristics_head *__single head;
1334 struct tcp_heuristic *__single tpheur;
1335 boolean_t ret = TRUE;
1336
1337 if (disable_tcp_heuristics) {
1338 return TRUE;
1339 }
1340
1341 /* Get the tcp-heuristic. */
1342 tpheur = tcp_getheuristic_with_lock(tcks, 0, &head);
1343 if (tpheur == NULL) {
1344 return ret;
1345 }
1346
1347 if (TSTMP_GT(tpheur->th_ecn_backoff, tcp_now)) {
1348 ret = FALSE;
1349 } else {
1350 /* Reset the following counters to start re-evaluating */
1351 if (tpheur->th_ecn_droprst >= ECN_RETRY_LIMIT) {
1352 tpheur->th_ecn_droprst = 0;
1353 }
1354 if (tpheur->th_ecn_droprxmt >= ECN_RETRY_LIMIT) {
1355 tpheur->th_ecn_droprxmt = 0;
1356 }
1357 if (tpheur->th_ecn_synrst >= ECN_RETRY_LIMIT) {
1358 tpheur->th_ecn_synrst = 0;
1359 }
1360
1361 /* Make sure it follows along */
1362 tpheur->th_ecn_backoff = tcp_now;
1363 }
1364
1365 tcp_heuristic_unlock(head);
1366
1367 return ret;
1368 }
1369
1370 boolean_t
tcp_heuristic_do_ecn(struct tcpcb * tp)1371 tcp_heuristic_do_ecn(struct tcpcb *tp)
1372 {
1373 struct tcp_cache_key_src tcks;
1374
1375 tcp_cache_key_src_create(tp, &tcks);
1376 return tcp_heuristic_do_ecn_common(&tcks);
1377 }
1378
1379 boolean_t
tcp_heuristic_do_ecn_with_address(struct ifnet * ifp,union sockaddr_in_4_6 * local_address)1380 tcp_heuristic_do_ecn_with_address(struct ifnet *ifp,
1381 union sockaddr_in_4_6 *local_address)
1382 {
1383 struct tcp_cache_key_src tcks;
1384
1385 memset(&tcks, 0, sizeof(tcks));
1386 tcks.ifp = ifp;
1387
1388 calculate_tcp_clock();
1389
1390 if (local_address->sa.sa_family == AF_INET6) {
1391 memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr));
1392 tcks.af = AF_INET6;
1393 } else if (local_address->sa.sa_family == AF_INET) {
1394 memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr));
1395 tcks.af = AF_INET;
1396 }
1397
1398 return tcp_heuristic_do_ecn_common(&tcks);
1399 }
1400
1401 void
tcp_heuristics_ecn_update(struct necp_tcp_ecn_cache * necp_buffer,struct ifnet * ifp,union sockaddr_in_4_6 * local_address)1402 tcp_heuristics_ecn_update(struct necp_tcp_ecn_cache *necp_buffer,
1403 struct ifnet *ifp, union sockaddr_in_4_6 *local_address)
1404 {
1405 struct tcp_cache_key_src tcks;
1406
1407 memset(&tcks, 0, sizeof(tcks));
1408 tcks.ifp = ifp;
1409
1410 calculate_tcp_clock();
1411
1412 if (local_address->sa.sa_family == AF_INET6) {
1413 memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr));
1414 tcks.af = AF_INET6;
1415 } else if (local_address->sa.sa_family == AF_INET) {
1416 memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr));
1417 tcks.af = AF_INET;
1418 }
1419
1420 if (necp_buffer->necp_tcp_ecn_heuristics_success) {
1421 tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_ECN);
1422 } else if (necp_buffer->necp_tcp_ecn_heuristics_loss) {
1423 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN);
1424 } else if (necp_buffer->necp_tcp_ecn_heuristics_drop_rst) {
1425 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRST);
1426 } else if (necp_buffer->necp_tcp_ecn_heuristics_drop_rxmt) {
1427 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRXMT);
1428 } else if (necp_buffer->necp_tcp_ecn_heuristics_syn_rst) {
1429 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_SYNRST);
1430 } else if (necp_buffer->necp_tcp_ecn_heuristics_aggressive) {
1431 tcp_heuristic_ecn_aggressive_common(&tcks);
1432 }
1433
1434 return;
1435 }
1436
1437 boolean_t
tcp_heuristic_do_tfo_with_address(struct ifnet * ifp,union sockaddr_in_4_6 * local_address,union sockaddr_in_4_6 * remote_address,uint8_t * __counted_by (maxlen)cookie,uint8_t maxlen,uint8_t * cookie_len)1438 tcp_heuristic_do_tfo_with_address(struct ifnet *ifp,
1439 union sockaddr_in_4_6 *local_address, union sockaddr_in_4_6 *remote_address,
1440 uint8_t *__counted_by(maxlen) cookie, uint8_t maxlen, uint8_t *cookie_len)
1441 {
1442 struct tcp_cache_key_src tcks;
1443
1444 memset(&tcks, 0, sizeof(tcks));
1445 tcks.ifp = ifp;
1446
1447 calculate_tcp_clock();
1448
1449 if (remote_address->sa.sa_family == AF_INET6) {
1450 memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr));
1451 memcpy(&tcks.faddr.addr6, &remote_address->sin6.sin6_addr, sizeof(struct in6_addr));
1452 tcks.af = AF_INET6;
1453 } else if (remote_address->sa.sa_family == AF_INET) {
1454 memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr));
1455 memcpy(&tcks.faddr.addr, &remote_address->sin.sin_addr, sizeof(struct in_addr));
1456 tcks.af = AF_INET;
1457 }
1458
1459 if (tcp_heuristic_do_tfo_common(&tcks)) {
1460 if (!tcp_cache_get_cookie_common(&tcks, cookie, maxlen, cookie_len)) {
1461 *cookie_len = 0;
1462 }
1463 return TRUE;
1464 }
1465
1466 return FALSE;
1467 }
1468
1469 void
tcp_heuristics_tfo_update(struct necp_tcp_tfo_cache * necp_buffer,struct ifnet * ifp,union sockaddr_in_4_6 * local_address,union sockaddr_in_4_6 * remote_address)1470 tcp_heuristics_tfo_update(struct necp_tcp_tfo_cache *necp_buffer,
1471 struct ifnet *ifp, union sockaddr_in_4_6 *local_address,
1472 union sockaddr_in_4_6 *remote_address)
1473 {
1474 struct tcp_cache_key_src tcks;
1475
1476 memset(&tcks, 0, sizeof(tcks));
1477 tcks.ifp = ifp;
1478
1479 calculate_tcp_clock();
1480
1481 if (remote_address->sa.sa_family == AF_INET6) {
1482 memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr));
1483 memcpy(&tcks.faddr.addr6, &remote_address->sin6.sin6_addr, sizeof(struct in6_addr));
1484 tcks.af = AF_INET6;
1485 } else if (remote_address->sa.sa_family == AF_INET) {
1486 memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr));
1487 memcpy(&tcks.faddr.addr, &remote_address->sin.sin_addr, sizeof(struct in_addr));
1488 tcks.af = AF_INET;
1489 }
1490
1491 if (necp_buffer->necp_tcp_tfo_heuristics_success) {
1492 tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_DATA |
1493 TCPCACHE_F_TFO_REQ_RST | TCPCACHE_F_TFO_DATA_RST);
1494 }
1495
1496 if (necp_buffer->necp_tcp_tfo_heuristics_success_req) {
1497 tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_REQ_RST);
1498 }
1499
1500 if (necp_buffer->necp_tcp_tfo_heuristics_loss) {
1501 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_DATA);
1502 }
1503
1504 if (necp_buffer->necp_tcp_tfo_heuristics_loss_req) {
1505 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ);
1506 }
1507
1508 if (necp_buffer->necp_tcp_tfo_heuristics_rst_data) {
1509 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ_RST | TCPCACHE_F_TFO_DATA_RST);
1510 }
1511
1512 if (necp_buffer->necp_tcp_tfo_heuristics_rst_req) {
1513 tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ_RST);
1514 }
1515
1516 if (necp_buffer->necp_tcp_tfo_heuristics_middlebox) {
1517 tcp_heuristic_tfo_middlebox_common(&tcks);
1518 }
1519
1520 if (necp_buffer->necp_tcp_tfo_cookie_len != 0) {
1521 tcp_cache_set_cookie_common(&tcks,
1522 necp_buffer->necp_tcp_tfo_cookie, necp_buffer->necp_tcp_tfo_cookie_len);
1523 }
1524
1525 return;
1526 }
1527
1528 #if (DEVELOPMENT || DEBUG)
1529 /*
1530 * This test sysctl forces the hash table to be full which will force us to
1531 * erase portions of it.
1532 */
1533 static int
1534 sysctl_fill_hashtable SYSCTL_HANDLER_ARGS
1535 {
1536 #pragma unused(arg1, arg2)
1537 int error = 0, val;
1538
1539 val = 0;
1540 error = sysctl_handle_int(oidp, &val, 0, req);
1541 if (error || !req->newptr) {
1542 return error;
1543 }
1544 if (val == 1) {
1545 struct necp_tcp_tfo_cache necp_buffer = {};
1546 union sockaddr_in_4_6 local_address = {}, remote_address = {};
1547
1548 necp_buffer.necp_tcp_tfo_heuristics_success = 1;
1549 necp_buffer.necp_tcp_tfo_heuristics_loss = 1;
1550 necp_buffer.necp_tcp_tfo_heuristics_middlebox = 1;
1551
1552 for (unsigned i = 0; i < 1024; i++) {
1553 local_address.sin.sin_family = AF_INET;
1554 local_address.sin.sin_len = sizeof(struct sockaddr_in);
1555 local_address.sin.sin_port = random() % UINT16_MAX;
1556 local_address.sin.sin_addr.s_addr = random();
1557
1558 remote_address.sin.sin_family = AF_INET;
1559 remote_address.sin.sin_len = sizeof(struct sockaddr_in);
1560 remote_address.sin.sin_port = random() % UINT16_MAX;
1561 remote_address.sin.sin_addr.s_addr = random();
1562
1563 tcp_heuristics_tfo_update(&necp_buffer, lo_ifp,
1564 &local_address,
1565 &remote_address);
1566 }
1567 }
1568
1569 return error;
1570 }
1571
1572 static int fill_hash_table = 0;
1573 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, test_cache, CTLTYPE_INT | CTLFLAG_RW |
1574 CTLFLAG_LOCKED, &fill_hash_table, 0, &sysctl_fill_hashtable, "I",
1575 "Tests the hash table erasing procedures");
1576 #endif /* DEVELOPMENT || DEBUG */
1577
1578 static void
sysctl_cleartfocache(void)1579 sysctl_cleartfocache(void)
1580 {
1581 int i;
1582
1583 for (i = 0; i < tcp_cache_size; i++) {
1584 struct tcp_cache_head *__single head = &tcp_cache[i];
1585 struct tcp_cache *__single tpcache, *__single tmp;
1586 struct tcp_heuristics_head *__single hhead = &tcp_heuristics[i];
1587 struct tcp_heuristic *__single tpheur, *__single htmp;
1588
1589 lck_mtx_lock(&head->tch_mtx);
1590 SLIST_FOREACH_SAFE(tpcache, &head->tcp_caches, list, tmp) {
1591 SLIST_REMOVE(&head->tcp_caches, tpcache, tcp_cache, list);
1592 kfree_type(struct tcp_cache, tpcache);
1593 }
1594 lck_mtx_unlock(&head->tch_mtx);
1595
1596 lck_mtx_lock(&hhead->thh_mtx);
1597 SLIST_FOREACH_SAFE(tpheur, &hhead->tcp_heuristics, list, htmp) {
1598 SLIST_REMOVE(&hhead->tcp_heuristics, tpheur, tcp_heuristic, list);
1599 kfree_type(struct tcp_heuristic, tpheur);
1600 }
1601 lck_mtx_unlock(&hhead->thh_mtx);
1602 }
1603 }
1604
1605 /* This sysctl is useful for testing purposes only */
1606 static int tcpcleartfo = 0;
1607
1608 static int sysctl_cleartfo SYSCTL_HANDLER_ARGS
1609 {
1610 #pragma unused(arg1, arg2)
1611 int error = 0, val, oldval = tcpcleartfo;
1612
1613 val = oldval;
1614 error = sysctl_handle_int(oidp, &val, 0, req);
1615 if (error || !req->newptr) {
1616 if (error) {
1617 os_log_error(OS_LOG_DEFAULT, "%s could not parse int: %d", __func__, error);
1618 }
1619 return error;
1620 }
1621
1622 /*
1623 * The actual value does not matter. If the value is set, it triggers
1624 * the clearing of the TFO cache. If a future implementation does not
1625 * use the route entry to hold the TFO cache, replace the route sysctl.
1626 */
1627
1628 if (val != oldval) {
1629 sysctl_cleartfocache();
1630 }
1631
1632 tcpcleartfo = val;
1633
1634 return error;
1635 }
1636
1637 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, clear_tfocache, CTLTYPE_INT | CTLFLAG_RW |
1638 CTLFLAG_LOCKED, &tcpcleartfo, 0, &sysctl_cleartfo, "I",
1639 "Toggle to clear the TFO destination based heuristic cache");
1640
1641 void
tcp_cache_init(void)1642 tcp_cache_init(void)
1643 {
1644 uint64_t sane_size_meg = sane_size / 1024 / 1024;
1645 size_t cache_size;
1646 /*
1647 * On machines with <100MB of memory this will result in a (full) cache-size
1648 * of 32 entries, thus 32 * 5 * 64bytes = 10KB. (about 0.01 %)
1649 * On machines with > 4GB of memory, we have a cache-size of 1024 entries,
1650 * thus about 327KB.
1651 *
1652 * Side-note: we convert to uint32_t. If sane_size is more than
1653 * 16000 TB, we loose precision. But, who cares? :)
1654 */
1655 cache_size = tcp_cache_roundup2((uint32_t)(sane_size_meg >> 2));
1656 if (cache_size < 32) {
1657 cache_size = 32;
1658 } else if (cache_size > 1024) {
1659 cache_size = 1024;
1660 }
1661
1662 tcp_cache = zalloc_permanent(sizeof(struct tcp_cache_head) * cache_size,
1663 ZALIGN(struct tcp_cache_head));
1664 tcp_cache_size = cache_size;
1665 tcp_heuristics = zalloc_permanent(sizeof(struct tcp_heuristics_head) * cache_size,
1666 ZALIGN(struct tcp_heuristics_head));
1667 tcp_heuristics_size = cache_size;
1668
1669 for (int i = 0; i < tcp_cache_size; i++) {
1670 lck_mtx_init(&tcp_cache[i].tch_mtx, &tcp_cache_mtx_grp,
1671 &tcp_cache_mtx_attr);
1672 SLIST_INIT(&tcp_cache[i].tcp_caches);
1673
1674 lck_mtx_init(&tcp_heuristics[i].thh_mtx, &tcp_heuristic_mtx_grp,
1675 &tcp_heuristic_mtx_attr);
1676 SLIST_INIT(&tcp_heuristics[i].tcp_heuristics);
1677 }
1678
1679 tcp_cache_hash_seed = RandomULong();
1680 }
1681