1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2019 Yandex LLC
5 * Copyright (c) 2015 Alexander V. Chernikov <[email protected]>
6 * Copyright (c) 2016-2019 Andrey V. Elsukov <[email protected]>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 *
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/counter.h>
34 #include <sys/ck.h>
35 #include <sys/epoch.h>
36 #include <sys/errno.h>
37 #include <sys/hash.h>
38 #include <sys/kernel.h>
39 #include <sys/lock.h>
40 #include <sys/malloc.h>
41 #include <sys/mbuf.h>
42 #include <sys/module.h>
43 #include <sys/rmlock.h>
44 #include <sys/socket.h>
45 #include <sys/syslog.h>
46 #include <sys/sysctl.h>
47
48 #include <net/if.h>
49 #include <net/if_var.h>
50 #include <net/if_pflog.h>
51 #include <net/pfil.h>
52
53 #include <netinet/in.h>
54 #include <netinet/ip.h>
55 #include <netinet/ip_var.h>
56 #include <netinet/ip_fw.h>
57 #include <netinet/ip6.h>
58 #include <netinet/icmp6.h>
59 #include <netinet/ip_icmp.h>
60 #include <netinet/tcp.h>
61 #include <netinet/udp.h>
62 #include <netinet6/in6_var.h>
63 #include <netinet6/ip6_var.h>
64 #include <netinet6/ip_fw_nat64.h>
65
66 #include <netpfil/ipfw/ip_fw_private.h>
67 #include <netpfil/pf/pf.h>
68
69 #include "nat64lsn.h"
70
71 MALLOC_DEFINE(M_NAT64LSN, "NAT64LSN", "NAT64LSN");
72
73 #define NAT64LSN_EPOCH_ENTER(et) NET_EPOCH_ENTER(et)
74 #define NAT64LSN_EPOCH_EXIT(et) NET_EPOCH_EXIT(et)
75 #define NAT64LSN_EPOCH_ASSERT() NET_EPOCH_ASSERT()
76 #define NAT64LSN_EPOCH_CALL(c, f) NET_EPOCH_CALL((f), (c))
77
78 static uma_zone_t nat64lsn_host_zone;
79 static uma_zone_t nat64lsn_pgchunk_zone;
80 static uma_zone_t nat64lsn_pg_zone;
81 static uma_zone_t nat64lsn_aliaslink_zone;
82 static uma_zone_t nat64lsn_state_zone;
83 static uma_zone_t nat64lsn_job_zone;
84
85 static void nat64lsn_periodic(void *data);
86 #define PERIODIC_DELAY 4
87 #define NAT64_LOOKUP(chain, cmd) \
88 (struct nat64lsn_cfg *)SRV_OBJECT((chain), (cmd)->arg1)
89 /*
90 * Delayed job queue, used to create new hosts
91 * and new portgroups
92 */
93 enum nat64lsn_jtype {
94 JTYPE_NEWHOST = 1,
95 JTYPE_NEWPORTGROUP,
96 JTYPE_DESTROY,
97 };
98
99 struct nat64lsn_job_item {
100 STAILQ_ENTRY(nat64lsn_job_item) entries;
101 enum nat64lsn_jtype jtype;
102
103 union {
104 struct { /* used by JTYPE_NEWHOST, JTYPE_NEWPORTGROUP */
105 struct mbuf *m;
106 struct nat64lsn_host *host;
107 struct nat64lsn_state *state;
108 uint32_t src6_hval;
109 uint32_t state_hval;
110 struct ipfw_flow_id f_id;
111 in_addr_t faddr;
112 uint16_t port;
113 uint8_t proto;
114 uint8_t done;
115 };
116 struct { /* used by JTYPE_DESTROY */
117 struct nat64lsn_hosts_slist hosts;
118 struct nat64lsn_pg_slist portgroups;
119 struct nat64lsn_pgchunk *pgchunk;
120 struct epoch_context epoch_ctx;
121 };
122 };
123 };
124
125 static struct mtx jmtx;
126 #define JQUEUE_LOCK_INIT() mtx_init(&jmtx, "qlock", NULL, MTX_DEF)
127 #define JQUEUE_LOCK_DESTROY() mtx_destroy(&jmtx)
128 #define JQUEUE_LOCK() mtx_lock(&jmtx)
129 #define JQUEUE_UNLOCK() mtx_unlock(&jmtx)
130
131 static int nat64lsn_alloc_host(struct nat64lsn_cfg *cfg,
132 struct nat64lsn_job_item *ji);
133 static int nat64lsn_alloc_pg(struct nat64lsn_cfg *cfg,
134 struct nat64lsn_job_item *ji);
135 static struct nat64lsn_job_item *nat64lsn_create_job(
136 struct nat64lsn_cfg *cfg, int jtype);
137 static void nat64lsn_enqueue_job(struct nat64lsn_cfg *cfg,
138 struct nat64lsn_job_item *ji);
139 static void nat64lsn_job_destroy(epoch_context_t ctx);
140 static void nat64lsn_destroy_host(struct nat64lsn_host *host);
141 static void nat64lsn_destroy_pg(struct nat64lsn_pg *pg);
142
143 static int nat64lsn_translate4(struct nat64lsn_cfg *cfg,
144 const struct ipfw_flow_id *f_id, struct mbuf **mp);
145 static int nat64lsn_translate6(struct nat64lsn_cfg *cfg,
146 struct ipfw_flow_id *f_id, struct mbuf **mp);
147 static int nat64lsn_translate6_internal(struct nat64lsn_cfg *cfg,
148 struct mbuf **mp, struct nat64lsn_state *state, uint8_t flags);
149
150 #define NAT64_BIT_TCP_FIN 0 /* FIN was seen */
151 #define NAT64_BIT_TCP_SYN 1 /* First syn in->out */
152 #define NAT64_BIT_TCP_ESTAB 2 /* Packet with Ack */
153 #define NAT64_BIT_READY_IPV4 6 /* state is ready for translate4 */
154 #define NAT64_BIT_STALE 7 /* state is going to be expired */
155
156 #define NAT64_FLAG_FIN (1 << NAT64_BIT_TCP_FIN)
157 #define NAT64_FLAG_SYN (1 << NAT64_BIT_TCP_SYN)
158 #define NAT64_FLAG_ESTAB (1 << NAT64_BIT_TCP_ESTAB)
159 #define NAT64_FLAGS_TCP (NAT64_FLAG_SYN|NAT64_FLAG_ESTAB|NAT64_FLAG_FIN)
160
161 #define NAT64_FLAG_READY (1 << NAT64_BIT_READY_IPV4)
162 #define NAT64_FLAG_STALE (1 << NAT64_BIT_STALE)
163
164 static inline uint8_t
convert_tcp_flags(uint8_t flags)165 convert_tcp_flags(uint8_t flags)
166 {
167 uint8_t result;
168
169 result = flags & (TH_FIN|TH_SYN);
170 result |= (flags & TH_RST) >> 2; /* Treat RST as FIN */
171 result |= (flags & TH_ACK) >> 2; /* Treat ACK as estab */
172
173 return (result);
174 }
175
176 static void
nat64lsn_log(struct pfloghdr * plog,struct mbuf * m,sa_family_t family,struct nat64lsn_state * state)177 nat64lsn_log(struct pfloghdr *plog, struct mbuf *m, sa_family_t family,
178 struct nat64lsn_state *state)
179 {
180
181 memset(plog, 0, sizeof(*plog));
182 plog->length = PFLOG_HDRLEN;
183 plog->af = family;
184 plog->action = PF_NAT;
185 plog->dir = PF_IN;
186 plog->rulenr = htonl(state->ip_src);
187 plog->subrulenr = htonl((uint32_t)(state->aport << 16) |
188 (state->proto << 8) | (state->ip_dst & 0xff));
189 plog->ruleset[0] = '\0';
190 strlcpy(plog->ifname, "NAT64LSN", sizeof(plog->ifname));
191 ipfw_bpf_mtap2(plog, PFLOG_HDRLEN, m);
192 }
193
194 #define HVAL(p, n, s) jenkins_hash32((const uint32_t *)(p), (n), (s))
195 #define HOST_HVAL(c, a) HVAL((a),\
196 sizeof(struct in6_addr) / sizeof(uint32_t), (c)->hash_seed)
197 #define HOSTS(c, v) ((c)->hosts_hash[(v) & ((c)->hosts_hashsize - 1)])
198
199 #define ALIASLINK_HVAL(c, f) HVAL(&(f)->dst_ip6,\
200 sizeof(struct in6_addr) * 2 / sizeof(uint32_t), (c)->hash_seed)
201 #define ALIAS_BYHASH(c, v) \
202 ((c)->aliases[(v) & ((1 << (32 - (c)->plen4)) - 1)])
203 static struct nat64lsn_aliaslink*
nat64lsn_get_aliaslink(struct nat64lsn_cfg * cfg __unused,struct nat64lsn_host * host,const struct ipfw_flow_id * f_id __unused)204 nat64lsn_get_aliaslink(struct nat64lsn_cfg *cfg __unused,
205 struct nat64lsn_host *host, const struct ipfw_flow_id *f_id __unused)
206 {
207
208 /*
209 * We can implement some different algorithms how
210 * select an alias address.
211 * XXX: for now we use first available.
212 */
213 return (CK_SLIST_FIRST(&host->aliases));
214 }
215
216 #define STATE_HVAL(c, d) HVAL((d), 2, (c)->hash_seed)
217 #define STATE_HASH(h, v) \
218 ((h)->states_hash[(v) & ((h)->states_hashsize - 1)])
219 #define STATES_CHUNK(p, v) \
220 ((p)->chunks_count == 1 ? (p)->states : \
221 ((p)->states_chunk[CHUNK_BY_FADDR(p, v)]))
222
223 #ifdef __LP64__
224 #define FREEMASK_FFSLL(pg, faddr) \
225 ffsll(*FREEMASK_CHUNK((pg), (faddr)))
226 #define FREEMASK_BTR(pg, faddr, bit) \
227 ck_pr_btr_64(FREEMASK_CHUNK((pg), (faddr)), (bit))
228 #define FREEMASK_BTS(pg, faddr, bit) \
229 ck_pr_bts_64(FREEMASK_CHUNK((pg), (faddr)), (bit))
230 #define FREEMASK_ISSET(pg, faddr, bit) \
231 ISSET64(*FREEMASK_CHUNK((pg), (faddr)), (bit))
232 #define FREEMASK_COPY(pg, n, out) \
233 (out) = ck_pr_load_64(FREEMASK_CHUNK((pg), (n)))
234 #else
235 static inline int
freemask_ffsll(uint32_t * freemask)236 freemask_ffsll(uint32_t *freemask)
237 {
238 int i;
239
240 if ((i = ffsl(freemask[0])) != 0)
241 return (i);
242 if ((i = ffsl(freemask[1])) != 0)
243 return (i + 32);
244 return (0);
245 }
246 #define FREEMASK_FFSLL(pg, faddr) \
247 freemask_ffsll(FREEMASK_CHUNK((pg), (faddr)))
248 #define FREEMASK_BTR(pg, faddr, bit) \
249 ck_pr_btr_32(FREEMASK_CHUNK((pg), (faddr)) + (bit) / 32, (bit) % 32)
250 #define FREEMASK_BTS(pg, faddr, bit) \
251 ck_pr_bts_32(FREEMASK_CHUNK((pg), (faddr)) + (bit) / 32, (bit) % 32)
252 #define FREEMASK_ISSET(pg, faddr, bit) \
253 ISSET32(*(FREEMASK_CHUNK((pg), (faddr)) + (bit) / 32), (bit) % 32)
254 #define FREEMASK_COPY(pg, n, out) \
255 (out) = ck_pr_load_32(FREEMASK_CHUNK((pg), (n))) | \
256 ((uint64_t)ck_pr_load_32(FREEMASK_CHUNK((pg), (n)) + 1) << 32)
257 #endif /* !__LP64__ */
258
259 #define NAT64LSN_TRY_PGCNT 32
260 static struct nat64lsn_pg*
nat64lsn_get_pg(uint32_t * chunkmask,uint32_t * pgmask,struct nat64lsn_pgchunk ** chunks,struct nat64lsn_pg ** pgptr,uint32_t * pgidx,in_addr_t faddr)261 nat64lsn_get_pg(uint32_t *chunkmask, uint32_t *pgmask,
262 struct nat64lsn_pgchunk **chunks, struct nat64lsn_pg **pgptr,
263 uint32_t *pgidx, in_addr_t faddr)
264 {
265 struct nat64lsn_pg *pg, *oldpg;
266 uint32_t idx, oldidx;
267 int cnt;
268
269 cnt = 0;
270 /* First try last used PG */
271 oldpg = pg = ck_pr_load_ptr(pgptr);
272 idx = oldidx = ck_pr_load_32(pgidx);
273 /* If pgidx is out of range, reset it to the first pgchunk */
274 if (!ISSET32(*chunkmask, idx / 32))
275 idx = 0;
276 do {
277 ck_pr_fence_load();
278 if (pg != NULL && FREEMASK_BITCOUNT(pg, faddr) > 0) {
279 /*
280 * If last used PG has not free states,
281 * try to update pointer.
282 * NOTE: it can be already updated by jobs handler,
283 * thus we use CAS operation.
284 */
285 if (cnt > 0)
286 ck_pr_cas_ptr(pgptr, oldpg, pg);
287 return (pg);
288 }
289 /* Stop if idx is out of range */
290 if (!ISSET32(*chunkmask, idx / 32))
291 break;
292
293 if (ISSET32(pgmask[idx / 32], idx % 32))
294 pg = ck_pr_load_ptr(
295 &chunks[idx / 32]->pgptr[idx % 32]);
296 else
297 pg = NULL;
298
299 idx++;
300 } while (++cnt < NAT64LSN_TRY_PGCNT);
301
302 /* If pgidx is out of range, reset it to the first pgchunk */
303 if (!ISSET32(*chunkmask, idx / 32))
304 idx = 0;
305 ck_pr_cas_32(pgidx, oldidx, idx);
306 return (NULL);
307 }
308
309 static struct nat64lsn_state*
nat64lsn_get_state6to4(struct nat64lsn_cfg * cfg,struct nat64lsn_host * host,const struct ipfw_flow_id * f_id,uint32_t hval,in_addr_t faddr,uint16_t port,uint8_t proto)310 nat64lsn_get_state6to4(struct nat64lsn_cfg *cfg, struct nat64lsn_host *host,
311 const struct ipfw_flow_id *f_id, uint32_t hval, in_addr_t faddr,
312 uint16_t port, uint8_t proto)
313 {
314 struct nat64lsn_aliaslink *link;
315 struct nat64lsn_state *state;
316 struct nat64lsn_pg *pg;
317 int i, offset;
318
319 NAT64LSN_EPOCH_ASSERT();
320
321 /* Check that we already have state for given arguments */
322 CK_SLIST_FOREACH(state, &STATE_HASH(host, hval), entries) {
323 if (state->proto == proto && state->ip_dst == faddr &&
324 state->sport == port && state->dport == f_id->dst_port)
325 return (state);
326 }
327
328 link = nat64lsn_get_aliaslink(cfg, host, f_id);
329 if (link == NULL)
330 return (NULL);
331
332 switch (proto) {
333 case IPPROTO_TCP:
334 pg = nat64lsn_get_pg(
335 &link->alias->tcp_chunkmask, link->alias->tcp_pgmask,
336 link->alias->tcp, &link->alias->tcp_pg,
337 &link->alias->tcp_pgidx, faddr);
338 break;
339 case IPPROTO_UDP:
340 pg = nat64lsn_get_pg(
341 &link->alias->udp_chunkmask, link->alias->udp_pgmask,
342 link->alias->udp, &link->alias->udp_pg,
343 &link->alias->udp_pgidx, faddr);
344 break;
345 case IPPROTO_ICMP:
346 pg = nat64lsn_get_pg(
347 &link->alias->icmp_chunkmask, link->alias->icmp_pgmask,
348 link->alias->icmp, &link->alias->icmp_pg,
349 &link->alias->icmp_pgidx, faddr);
350 break;
351 default:
352 panic("%s: wrong proto %d", __func__, proto);
353 }
354 if (pg == NULL)
355 return (NULL);
356
357 /* Check that PG has some free states */
358 state = NULL;
359 i = FREEMASK_BITCOUNT(pg, faddr);
360 while (i-- > 0) {
361 offset = FREEMASK_FFSLL(pg, faddr);
362 if (offset == 0) {
363 /*
364 * We lost the race.
365 * No more free states in this PG.
366 */
367 break;
368 }
369
370 /* Lets try to atomically grab the state */
371 if (FREEMASK_BTR(pg, faddr, offset - 1)) {
372 state = &STATES_CHUNK(pg, faddr)->state[offset - 1];
373 /* Initialize */
374 state->flags = proto != IPPROTO_TCP ? 0 :
375 convert_tcp_flags(f_id->_flags);
376 state->proto = proto;
377 state->aport = pg->base_port + offset - 1;
378 state->dport = f_id->dst_port;
379 state->sport = port;
380 state->ip6_dst = f_id->dst_ip6;
381 state->ip_dst = faddr;
382 state->ip_src = link->alias->addr;
383 state->hval = hval;
384 state->host = host;
385 SET_AGE(state->timestamp);
386
387 /* Insert new state into host's hash table */
388 HOST_LOCK(host);
389 CK_SLIST_INSERT_HEAD(&STATE_HASH(host, hval),
390 state, entries);
391 host->states_count++;
392 /*
393 * XXX: In case if host is going to be expired,
394 * reset NAT64LSN_DEADHOST flag.
395 */
396 host->flags &= ~NAT64LSN_DEADHOST;
397 HOST_UNLOCK(host);
398 NAT64STAT_INC(&cfg->base.stats, screated);
399 /* Mark the state as ready for translate4 */
400 ck_pr_fence_store();
401 ck_pr_bts_32(&state->flags, NAT64_BIT_READY_IPV4);
402 break;
403 }
404 }
405 return (state);
406 }
407
408 /*
409 * Inspects icmp packets to see if the message contains different
410 * packet header so we need to alter @addr and @port.
411 */
412 static int
inspect_icmp_mbuf(struct mbuf ** mp,uint8_t * proto,uint32_t * addr,uint16_t * port)413 inspect_icmp_mbuf(struct mbuf **mp, uint8_t *proto, uint32_t *addr,
414 uint16_t *port)
415 {
416 struct icmp *icmp;
417 struct ip *ip;
418 int off;
419 uint8_t inner_proto;
420
421 ip = mtod(*mp, struct ip *); /* Outer IP header */
422 off = (ip->ip_hl << 2) + ICMP_MINLEN;
423 if ((*mp)->m_len < off)
424 *mp = m_pullup(*mp, off);
425 if (*mp == NULL)
426 return (ENOMEM);
427
428 ip = mtod(*mp, struct ip *); /* Outer IP header */
429 icmp = L3HDR(ip, struct icmp *);
430 switch (icmp->icmp_type) {
431 case ICMP_ECHO:
432 case ICMP_ECHOREPLY:
433 /* Use icmp ID as distinguisher */
434 *port = ntohs(icmp->icmp_id);
435 return (0);
436 case ICMP_UNREACH:
437 case ICMP_TIMXCEED:
438 break;
439 default:
440 return (EOPNOTSUPP);
441 }
442 /*
443 * ICMP_UNREACH and ICMP_TIMXCEED contains IP header + 64 bits
444 * of ULP header.
445 */
446 if ((*mp)->m_pkthdr.len < off + sizeof(struct ip) + ICMP_MINLEN)
447 return (EINVAL);
448 if ((*mp)->m_len < off + sizeof(struct ip) + ICMP_MINLEN)
449 *mp = m_pullup(*mp, off + sizeof(struct ip) + ICMP_MINLEN);
450 if (*mp == NULL)
451 return (ENOMEM);
452 ip = mtodo(*mp, off); /* Inner IP header */
453 inner_proto = ip->ip_p;
454 off += ip->ip_hl << 2; /* Skip inner IP header */
455 *addr = ntohl(ip->ip_src.s_addr);
456 if ((*mp)->m_len < off + ICMP_MINLEN)
457 *mp = m_pullup(*mp, off + ICMP_MINLEN);
458 if (*mp == NULL)
459 return (ENOMEM);
460 switch (inner_proto) {
461 case IPPROTO_TCP:
462 case IPPROTO_UDP:
463 /* Copy source port from the header */
464 *port = ntohs(*((uint16_t *)mtodo(*mp, off)));
465 *proto = inner_proto;
466 return (0);
467 case IPPROTO_ICMP:
468 /*
469 * We will translate only ICMP errors for our ICMP
470 * echo requests.
471 */
472 icmp = mtodo(*mp, off);
473 if (icmp->icmp_type != ICMP_ECHO)
474 return (EOPNOTSUPP);
475 *port = ntohs(icmp->icmp_id);
476 return (0);
477 };
478 return (EOPNOTSUPP);
479 }
480
481 static struct nat64lsn_state*
nat64lsn_get_state4to6(struct nat64lsn_cfg * cfg,struct nat64lsn_alias * alias,in_addr_t faddr,uint16_t port,uint8_t proto)482 nat64lsn_get_state4to6(struct nat64lsn_cfg *cfg, struct nat64lsn_alias *alias,
483 in_addr_t faddr, uint16_t port, uint8_t proto)
484 {
485 struct nat64lsn_state *state;
486 struct nat64lsn_pg *pg;
487 int chunk_idx, pg_idx, state_idx;
488
489 NAT64LSN_EPOCH_ASSERT();
490
491 if (port < NAT64_MIN_PORT)
492 return (NULL);
493 /*
494 * Alias keeps 32 pgchunks for each protocol.
495 * Each pgchunk has 32 pointers to portgroup.
496 * Each portgroup has 64 states for ports.
497 */
498 port -= NAT64_MIN_PORT;
499 chunk_idx = port / 2048;
500
501 port -= chunk_idx * 2048;
502 pg_idx = port / 64;
503 state_idx = port % 64;
504
505 /*
506 * First check in proto_chunkmask that we have allocated PG chunk.
507 * Then check in proto_pgmask that we have valid PG pointer.
508 */
509 pg = NULL;
510 switch (proto) {
511 case IPPROTO_TCP:
512 if (ISSET32(alias->tcp_chunkmask, chunk_idx) &&
513 ISSET32(alias->tcp_pgmask[chunk_idx], pg_idx)) {
514 pg = alias->tcp[chunk_idx]->pgptr[pg_idx];
515 break;
516 }
517 return (NULL);
518 case IPPROTO_UDP:
519 if (ISSET32(alias->udp_chunkmask, chunk_idx) &&
520 ISSET32(alias->udp_pgmask[chunk_idx], pg_idx)) {
521 pg = alias->udp[chunk_idx]->pgptr[pg_idx];
522 break;
523 }
524 return (NULL);
525 case IPPROTO_ICMP:
526 if (ISSET32(alias->icmp_chunkmask, chunk_idx) &&
527 ISSET32(alias->icmp_pgmask[chunk_idx], pg_idx)) {
528 pg = alias->icmp[chunk_idx]->pgptr[pg_idx];
529 break;
530 }
531 return (NULL);
532 default:
533 panic("%s: wrong proto %d", __func__, proto);
534 }
535 if (pg == NULL)
536 return (NULL);
537
538 if (FREEMASK_ISSET(pg, faddr, state_idx))
539 return (NULL);
540
541 state = &STATES_CHUNK(pg, faddr)->state[state_idx];
542 ck_pr_fence_load();
543 if (ck_pr_load_32(&state->flags) & NAT64_FLAG_READY)
544 return (state);
545 return (NULL);
546 }
547
548 /*
549 * Reassemble IPv4 fragments, make PULLUP if needed, get some ULP fields
550 * that might be unknown until reassembling is completed.
551 */
552 static struct mbuf*
nat64lsn_reassemble4(struct nat64lsn_cfg * cfg,struct mbuf * m,uint16_t * port)553 nat64lsn_reassemble4(struct nat64lsn_cfg *cfg, struct mbuf *m,
554 uint16_t *port)
555 {
556 struct ip *ip;
557 int len;
558
559 m = ip_reass(m);
560 if (m == NULL)
561 return (NULL);
562 /* IP header must be contigious after ip_reass() */
563 ip = mtod(m, struct ip *);
564 len = ip->ip_hl << 2;
565 switch (ip->ip_p) {
566 case IPPROTO_ICMP:
567 len += ICMP_MINLEN; /* Enough to get icmp_id */
568 break;
569 case IPPROTO_TCP:
570 len += sizeof(struct tcphdr);
571 break;
572 case IPPROTO_UDP:
573 len += sizeof(struct udphdr);
574 break;
575 default:
576 m_freem(m);
577 NAT64STAT_INC(&cfg->base.stats, noproto);
578 return (NULL);
579 }
580 if (m->m_len < len) {
581 m = m_pullup(m, len);
582 if (m == NULL) {
583 NAT64STAT_INC(&cfg->base.stats, nomem);
584 return (NULL);
585 }
586 ip = mtod(m, struct ip *);
587 }
588 switch (ip->ip_p) {
589 case IPPROTO_TCP:
590 *port = ntohs(L3HDR(ip, struct tcphdr *)->th_dport);
591 break;
592 case IPPROTO_UDP:
593 *port = ntohs(L3HDR(ip, struct udphdr *)->uh_dport);
594 break;
595 }
596 return (m);
597 }
598
599 static int
nat64lsn_translate4(struct nat64lsn_cfg * cfg,const struct ipfw_flow_id * f_id,struct mbuf ** mp)600 nat64lsn_translate4(struct nat64lsn_cfg *cfg,
601 const struct ipfw_flow_id *f_id, struct mbuf **mp)
602 {
603 struct pfloghdr loghdr, *logdata;
604 struct in6_addr src6;
605 struct nat64lsn_state *state;
606 struct nat64lsn_alias *alias;
607 uint32_t addr, flags;
608 uint16_t port, ts;
609 int ret;
610 uint8_t proto;
611
612 addr = f_id->dst_ip;
613 port = f_id->dst_port;
614 proto = f_id->proto;
615 if (addr < cfg->prefix4 || addr > cfg->pmask4) {
616 NAT64STAT_INC(&cfg->base.stats, nomatch4);
617 return (cfg->nomatch_verdict);
618 }
619
620 /* Reassemble fragments if needed */
621 ret = ntohs(mtod(*mp, struct ip *)->ip_off);
622 if ((ret & (IP_MF | IP_OFFMASK)) != 0) {
623 *mp = nat64lsn_reassemble4(cfg, *mp, &port);
624 if (*mp == NULL)
625 return (IP_FW_DENY);
626 }
627
628 /* Check if protocol is supported */
629 switch (proto) {
630 case IPPROTO_ICMP:
631 ret = inspect_icmp_mbuf(mp, &proto, &addr, &port);
632 if (ret != 0) {
633 if (ret == ENOMEM) {
634 NAT64STAT_INC(&cfg->base.stats, nomem);
635 return (IP_FW_DENY);
636 }
637 NAT64STAT_INC(&cfg->base.stats, noproto);
638 return (cfg->nomatch_verdict);
639 }
640 if (addr < cfg->prefix4 || addr > cfg->pmask4) {
641 NAT64STAT_INC(&cfg->base.stats, nomatch4);
642 return (cfg->nomatch_verdict);
643 }
644 /* FALLTHROUGH */
645 case IPPROTO_TCP:
646 case IPPROTO_UDP:
647 break;
648 default:
649 NAT64STAT_INC(&cfg->base.stats, noproto);
650 return (cfg->nomatch_verdict);
651 }
652
653 alias = &ALIAS_BYHASH(cfg, addr);
654 MPASS(addr == alias->addr);
655
656 /* Check that we have state for this port */
657 state = nat64lsn_get_state4to6(cfg, alias, f_id->src_ip,
658 port, proto);
659 if (state == NULL) {
660 NAT64STAT_INC(&cfg->base.stats, nomatch4);
661 return (cfg->nomatch_verdict);
662 }
663
664 /* TODO: Check flags to see if we need to do some static mapping */
665
666 /* Update some state fields if need */
667 SET_AGE(ts);
668 if (f_id->proto == IPPROTO_TCP)
669 flags = convert_tcp_flags(f_id->_flags);
670 else
671 flags = 0;
672 if (state->timestamp != ts)
673 state->timestamp = ts;
674 if ((state->flags & flags) != flags)
675 state->flags |= flags;
676
677 port = htons(state->sport);
678 src6 = state->ip6_dst;
679
680 if (cfg->base.flags & NAT64_LOG) {
681 logdata = &loghdr;
682 nat64lsn_log(logdata, *mp, AF_INET, state);
683 } else
684 logdata = NULL;
685
686 /*
687 * We already have src6 with embedded address, but it is possible,
688 * that src_ip is different than state->ip_dst, this is why we
689 * do embedding again.
690 */
691 nat64_embed_ip4(&src6, cfg->base.plat_plen, htonl(f_id->src_ip));
692 ret = nat64_do_handle_ip4(*mp, &src6, &state->host->addr, port,
693 &cfg->base, logdata);
694 if (ret == NAT64SKIP)
695 return (cfg->nomatch_verdict);
696 if (ret == NAT64RETURN)
697 *mp = NULL;
698 return (IP_FW_DENY);
699 }
700
701 /*
702 * Check if particular state is stale and should be deleted.
703 * Return 1 if true, 0 otherwise.
704 */
705 static int
nat64lsn_check_state(struct nat64lsn_cfg * cfg,struct nat64lsn_state * state)706 nat64lsn_check_state(struct nat64lsn_cfg *cfg, struct nat64lsn_state *state)
707 {
708 int age, ttl;
709
710 /* State was marked as stale in previous pass. */
711 if (ISSET32(state->flags, NAT64_BIT_STALE))
712 return (1);
713
714 /* State is not yet initialized, it is going to be READY */
715 if (!ISSET32(state->flags, NAT64_BIT_READY_IPV4))
716 return (0);
717
718 age = GET_AGE(state->timestamp);
719 switch (state->proto) {
720 case IPPROTO_TCP:
721 if (ISSET32(state->flags, NAT64_BIT_TCP_FIN))
722 ttl = cfg->st_close_ttl;
723 else if (ISSET32(state->flags, NAT64_BIT_TCP_ESTAB))
724 ttl = cfg->st_estab_ttl;
725 else if (ISSET32(state->flags, NAT64_BIT_TCP_SYN))
726 ttl = cfg->st_syn_ttl;
727 else
728 ttl = cfg->st_syn_ttl;
729 if (age > ttl)
730 return (1);
731 break;
732 case IPPROTO_UDP:
733 if (age > cfg->st_udp_ttl)
734 return (1);
735 break;
736 case IPPROTO_ICMP:
737 if (age > cfg->st_icmp_ttl)
738 return (1);
739 break;
740 }
741 return (0);
742 }
743
744 static int
nat64lsn_maintain_pg(struct nat64lsn_cfg * cfg,struct nat64lsn_pg * pg)745 nat64lsn_maintain_pg(struct nat64lsn_cfg *cfg, struct nat64lsn_pg *pg)
746 {
747 struct nat64lsn_state *state;
748 struct nat64lsn_host *host;
749 uint64_t freemask;
750 int c, i, update_age;
751
752 update_age = 0;
753 for (c = 0; c < pg->chunks_count; c++) {
754 FREEMASK_COPY(pg, c, freemask);
755 for (i = 0; i < 64; i++) {
756 if (ISSET64(freemask, i))
757 continue;
758 state = &STATES_CHUNK(pg, c)->state[i];
759 if (nat64lsn_check_state(cfg, state) == 0) {
760 update_age = 1;
761 continue;
762 }
763 /*
764 * Expire state:
765 * 1. Mark as STALE and unlink from host's hash.
766 * 2. Set bit in freemask.
767 */
768 if (ISSET32(state->flags, NAT64_BIT_STALE)) {
769 /*
770 * State was marked as STALE in previous
771 * pass. Now it is safe to release it.
772 */
773 state->flags = 0;
774 ck_pr_fence_store();
775 FREEMASK_BTS(pg, c, i);
776 NAT64STAT_INC(&cfg->base.stats, sdeleted);
777 continue;
778 }
779 MPASS(state->flags & NAT64_FLAG_READY);
780
781 host = state->host;
782 HOST_LOCK(host);
783 CK_SLIST_REMOVE(&STATE_HASH(host, state->hval),
784 state, nat64lsn_state, entries);
785 host->states_count--;
786 HOST_UNLOCK(host);
787
788 /* Reset READY flag */
789 ck_pr_btr_32(&state->flags, NAT64_BIT_READY_IPV4);
790 /* And set STALE flag */
791 ck_pr_bts_32(&state->flags, NAT64_BIT_STALE);
792 ck_pr_fence_store();
793 /*
794 * Now translate6 will not use this state, wait
795 * until it become safe for translate4, then mark
796 * state as free.
797 */
798 }
799 }
800
801 /*
802 * We have some alive states, update timestamp.
803 */
804 if (update_age)
805 SET_AGE(pg->timestamp);
806
807 if (GET_AGE(pg->timestamp) < cfg->pg_delete_delay)
808 return (0);
809
810 return (1);
811 }
812
813 static void
nat64lsn_expire_portgroups(struct nat64lsn_cfg * cfg,struct nat64lsn_pg_slist * portgroups)814 nat64lsn_expire_portgroups(struct nat64lsn_cfg *cfg,
815 struct nat64lsn_pg_slist *portgroups)
816 {
817 struct nat64lsn_alias *alias;
818 struct nat64lsn_pg *pg, *tpg, *firstpg, **pgptr;
819 uint32_t *pgmask, *pgidx;
820 int i, idx;
821
822 for (i = 0; i < 1 << (32 - cfg->plen4); i++) {
823 alias = &cfg->aliases[i];
824 CK_SLIST_FOREACH_SAFE(pg, &alias->portgroups, entries, tpg) {
825 if (nat64lsn_maintain_pg(cfg, pg) == 0)
826 continue;
827 /* Always keep first PG */
828 if (pg->base_port == NAT64_MIN_PORT)
829 continue;
830 /*
831 * PG is expired, unlink it and schedule for
832 * deferred destroying.
833 */
834 idx = (pg->base_port - NAT64_MIN_PORT) / 64;
835 switch (pg->proto) {
836 case IPPROTO_TCP:
837 pgmask = alias->tcp_pgmask;
838 pgptr = &alias->tcp_pg;
839 pgidx = &alias->tcp_pgidx;
840 firstpg = alias->tcp[0]->pgptr[0];
841 break;
842 case IPPROTO_UDP:
843 pgmask = alias->udp_pgmask;
844 pgptr = &alias->udp_pg;
845 pgidx = &alias->udp_pgidx;
846 firstpg = alias->udp[0]->pgptr[0];
847 break;
848 case IPPROTO_ICMP:
849 pgmask = alias->icmp_pgmask;
850 pgptr = &alias->icmp_pg;
851 pgidx = &alias->icmp_pgidx;
852 firstpg = alias->icmp[0]->pgptr[0];
853 break;
854 }
855 /* Reset the corresponding bit in pgmask array. */
856 ck_pr_btr_32(&pgmask[idx / 32], idx % 32);
857 ck_pr_fence_store();
858 /* If last used PG points to this PG, reset it. */
859 ck_pr_cas_ptr(pgptr, pg, firstpg);
860 ck_pr_cas_32(pgidx, idx, 0);
861 /* Unlink PG from alias's chain */
862 ALIAS_LOCK(alias);
863 CK_SLIST_REMOVE(&alias->portgroups, pg,
864 nat64lsn_pg, entries);
865 alias->portgroups_count--;
866 ALIAS_UNLOCK(alias);
867 /* And link to job's chain for deferred destroying */
868 NAT64STAT_INC(&cfg->base.stats, spgdeleted);
869 CK_SLIST_INSERT_HEAD(portgroups, pg, entries);
870 }
871 }
872 }
873
874 static void
nat64lsn_expire_hosts(struct nat64lsn_cfg * cfg,struct nat64lsn_hosts_slist * hosts)875 nat64lsn_expire_hosts(struct nat64lsn_cfg *cfg,
876 struct nat64lsn_hosts_slist *hosts)
877 {
878 struct nat64lsn_host *host, *tmp;
879 int i;
880
881 for (i = 0; i < cfg->hosts_hashsize; i++) {
882 CK_SLIST_FOREACH_SAFE(host, &cfg->hosts_hash[i],
883 entries, tmp) {
884 /* Is host was marked in previous call? */
885 if (host->flags & NAT64LSN_DEADHOST) {
886 if (host->states_count > 0) {
887 host->flags &= ~NAT64LSN_DEADHOST;
888 continue;
889 }
890 /*
891 * Unlink host from hash table and schedule
892 * it for deferred destroying.
893 */
894 CFG_LOCK(cfg);
895 CK_SLIST_REMOVE(&cfg->hosts_hash[i], host,
896 nat64lsn_host, entries);
897 cfg->hosts_count--;
898 CFG_UNLOCK(cfg);
899 CK_SLIST_INSERT_HEAD(hosts, host, entries);
900 continue;
901 }
902 if (GET_AGE(host->timestamp) < cfg->host_delete_delay)
903 continue;
904 if (host->states_count > 0)
905 continue;
906 /* Mark host as going to be expired in next pass */
907 host->flags |= NAT64LSN_DEADHOST;
908 ck_pr_fence_store();
909 }
910 }
911 }
912
913 static struct nat64lsn_pgchunk*
nat64lsn_expire_pgchunk(struct nat64lsn_cfg * cfg)914 nat64lsn_expire_pgchunk(struct nat64lsn_cfg *cfg)
915 {
916 #if 0
917 struct nat64lsn_alias *alias;
918 struct nat64lsn_pgchunk *chunk;
919 uint32_t pgmask;
920 int i, c;
921
922 for (i = 0; i < 1 << (32 - cfg->plen4); i++) {
923 alias = &cfg->aliases[i];
924 if (GET_AGE(alias->timestamp) < cfg->pgchunk_delete_delay)
925 continue;
926 /* Always keep single chunk allocated */
927 for (c = 1; c < 32; c++) {
928 if ((alias->tcp_chunkmask & (1 << c)) == 0)
929 break;
930 chunk = ck_pr_load_ptr(&alias->tcp[c]);
931 if (ck_pr_load_32(&alias->tcp_pgmask[c]) != 0)
932 continue;
933 ck_pr_btr_32(&alias->tcp_chunkmask, c);
934 ck_pr_fence_load();
935 if (ck_pr_load_32(&alias->tcp_pgmask[c]) != 0)
936 continue;
937 }
938 }
939 #endif
940 return (NULL);
941 }
942
943 #if 0
944 static void
945 nat64lsn_maintain_hosts(struct nat64lsn_cfg *cfg)
946 {
947 struct nat64lsn_host *h;
948 struct nat64lsn_states_slist *hash;
949 int i, j, hsize;
950
951 for (i = 0; i < cfg->hosts_hashsize; i++) {
952 CK_SLIST_FOREACH(h, &cfg->hosts_hash[i], entries) {
953 if (h->states_count / 2 < h->states_hashsize ||
954 h->states_hashsize >= NAT64LSN_MAX_HSIZE)
955 continue;
956 hsize = h->states_hashsize * 2;
957 hash = malloc(sizeof(*hash)* hsize, M_NOWAIT);
958 if (hash == NULL)
959 continue;
960 for (j = 0; j < hsize; j++)
961 CK_SLIST_INIT(&hash[i]);
962
963 ck_pr_bts_32(&h->flags, NAT64LSN_GROWHASH);
964 }
965 }
966 }
967 #endif
968
969 /*
970 * This procedure is used to perform various maintenance
971 * on dynamic hash list. Currently it is called every 4 seconds.
972 */
973 static void
nat64lsn_periodic(void * data)974 nat64lsn_periodic(void *data)
975 {
976 struct nat64lsn_job_item *ji;
977 struct nat64lsn_cfg *cfg;
978
979 cfg = (struct nat64lsn_cfg *) data;
980 CURVNET_SET(cfg->vp);
981 if (cfg->hosts_count > 0) {
982 ji = uma_zalloc(nat64lsn_job_zone, M_NOWAIT);
983 if (ji != NULL) {
984 ji->jtype = JTYPE_DESTROY;
985 CK_SLIST_INIT(&ji->hosts);
986 CK_SLIST_INIT(&ji->portgroups);
987 nat64lsn_expire_hosts(cfg, &ji->hosts);
988 nat64lsn_expire_portgroups(cfg, &ji->portgroups);
989 ji->pgchunk = nat64lsn_expire_pgchunk(cfg);
990 NAT64LSN_EPOCH_CALL(&ji->epoch_ctx,
991 nat64lsn_job_destroy);
992 } else
993 NAT64STAT_INC(&cfg->base.stats, jnomem);
994 }
995 callout_schedule(&cfg->periodic, hz * PERIODIC_DELAY);
996 CURVNET_RESTORE();
997 }
998
999 #define ALLOC_ERROR(stage, type) ((stage) ? 10 * (type) + (stage): 0)
1000 #define HOST_ERROR(stage) ALLOC_ERROR(stage, 1)
1001 #define PG_ERROR(stage) ALLOC_ERROR(stage, 2)
1002 static int
nat64lsn_alloc_host(struct nat64lsn_cfg * cfg,struct nat64lsn_job_item * ji)1003 nat64lsn_alloc_host(struct nat64lsn_cfg *cfg, struct nat64lsn_job_item *ji)
1004 {
1005 char a[INET6_ADDRSTRLEN];
1006 struct nat64lsn_aliaslink *link;
1007 struct nat64lsn_host *host;
1008 struct nat64lsn_state *state;
1009 uint32_t hval, data[2];
1010 int i;
1011
1012 /* Check that host was not yet added. */
1013 NAT64LSN_EPOCH_ASSERT();
1014 CK_SLIST_FOREACH(host, &HOSTS(cfg, ji->src6_hval), entries) {
1015 if (IN6_ARE_ADDR_EQUAL(&ji->f_id.src_ip6, &host->addr)) {
1016 /* The host was allocated in previous call. */
1017 ji->host = host;
1018 goto get_state;
1019 }
1020 }
1021
1022 host = ji->host = uma_zalloc(nat64lsn_host_zone, M_NOWAIT);
1023 if (ji->host == NULL)
1024 return (HOST_ERROR(1));
1025
1026 host->states_hashsize = NAT64LSN_HSIZE;
1027 host->states_hash = malloc(sizeof(struct nat64lsn_states_slist) *
1028 host->states_hashsize, M_NAT64LSN, M_NOWAIT);
1029 if (host->states_hash == NULL) {
1030 uma_zfree(nat64lsn_host_zone, host);
1031 return (HOST_ERROR(2));
1032 }
1033
1034 link = uma_zalloc(nat64lsn_aliaslink_zone, M_NOWAIT);
1035 if (link == NULL) {
1036 free(host->states_hash, M_NAT64LSN);
1037 uma_zfree(nat64lsn_host_zone, host);
1038 return (HOST_ERROR(3));
1039 }
1040
1041 /* Initialize */
1042 HOST_LOCK_INIT(host);
1043 SET_AGE(host->timestamp);
1044 host->addr = ji->f_id.src_ip6;
1045 host->hval = ji->src6_hval;
1046 host->flags = 0;
1047 host->states_count = 0;
1048 host->states_hashsize = NAT64LSN_HSIZE;
1049 CK_SLIST_INIT(&host->aliases);
1050 for (i = 0; i < host->states_hashsize; i++)
1051 CK_SLIST_INIT(&host->states_hash[i]);
1052
1053 /* Determine alias from flow hash. */
1054 hval = ALIASLINK_HVAL(cfg, &ji->f_id);
1055 link->alias = &ALIAS_BYHASH(cfg, hval);
1056 CK_SLIST_INSERT_HEAD(&host->aliases, link, host_entries);
1057
1058 ALIAS_LOCK(link->alias);
1059 CK_SLIST_INSERT_HEAD(&link->alias->hosts, link, alias_entries);
1060 link->alias->hosts_count++;
1061 ALIAS_UNLOCK(link->alias);
1062
1063 CFG_LOCK(cfg);
1064 CK_SLIST_INSERT_HEAD(&HOSTS(cfg, ji->src6_hval), host, entries);
1065 cfg->hosts_count++;
1066 CFG_UNLOCK(cfg);
1067
1068 get_state:
1069 data[0] = ji->faddr;
1070 data[1] = (ji->f_id.dst_port << 16) | ji->port;
1071 ji->state_hval = hval = STATE_HVAL(cfg, data);
1072 state = nat64lsn_get_state6to4(cfg, host, &ji->f_id, hval,
1073 ji->faddr, ji->port, ji->proto);
1074 /*
1075 * We failed to obtain new state, used alias needs new PG.
1076 * XXX: or another alias should be used.
1077 */
1078 if (state == NULL) {
1079 /* Try to allocate new PG */
1080 if (nat64lsn_alloc_pg(cfg, ji) != PG_ERROR(0))
1081 return (HOST_ERROR(4));
1082 /* We assume that nat64lsn_alloc_pg() got state */
1083 } else
1084 ji->state = state;
1085
1086 ji->done = 1;
1087 DPRINTF(DP_OBJ, "ALLOC HOST %s %p",
1088 inet_ntop(AF_INET6, &host->addr, a, sizeof(a)), host);
1089 return (HOST_ERROR(0));
1090 }
1091
1092 static int
nat64lsn_find_pg_place(uint32_t * data)1093 nat64lsn_find_pg_place(uint32_t *data)
1094 {
1095 int i;
1096
1097 for (i = 0; i < 32; i++) {
1098 if (~data[i] == 0)
1099 continue;
1100 return (i * 32 + ffs(~data[i]) - 1);
1101 }
1102 return (-1);
1103 }
1104
1105 static int
nat64lsn_alloc_proto_pg(struct nat64lsn_cfg * cfg,struct nat64lsn_alias * alias,uint32_t * chunkmask,uint32_t * pgmask,struct nat64lsn_pgchunk ** chunks,struct nat64lsn_pg ** pgptr,uint8_t proto)1106 nat64lsn_alloc_proto_pg(struct nat64lsn_cfg *cfg,
1107 struct nat64lsn_alias *alias, uint32_t *chunkmask,
1108 uint32_t *pgmask, struct nat64lsn_pgchunk **chunks,
1109 struct nat64lsn_pg **pgptr, uint8_t proto)
1110 {
1111 struct nat64lsn_pg *pg;
1112 int i, pg_idx, chunk_idx;
1113
1114 /* Find place in pgchunk where PG can be added */
1115 pg_idx = nat64lsn_find_pg_place(pgmask);
1116 if (pg_idx < 0) /* no more PGs */
1117 return (PG_ERROR(1));
1118 /* Check that we have allocated pgchunk for given PG index */
1119 chunk_idx = pg_idx / 32;
1120 if (!ISSET32(*chunkmask, chunk_idx)) {
1121 chunks[chunk_idx] = uma_zalloc(nat64lsn_pgchunk_zone,
1122 M_NOWAIT);
1123 if (chunks[chunk_idx] == NULL)
1124 return (PG_ERROR(2));
1125 ck_pr_bts_32(chunkmask, chunk_idx);
1126 ck_pr_fence_store();
1127 }
1128 /* Allocate PG and states chunks */
1129 pg = uma_zalloc(nat64lsn_pg_zone, M_NOWAIT);
1130 if (pg == NULL)
1131 return (PG_ERROR(3));
1132 pg->chunks_count = cfg->states_chunks;
1133 if (pg->chunks_count > 1) {
1134 pg->freemask_chunk = malloc(pg->chunks_count *
1135 sizeof(uint64_t), M_NAT64LSN, M_NOWAIT);
1136 if (pg->freemask_chunk == NULL) {
1137 uma_zfree(nat64lsn_pg_zone, pg);
1138 return (PG_ERROR(4));
1139 }
1140 pg->states_chunk = malloc(pg->chunks_count *
1141 sizeof(struct nat64lsn_states_chunk *), M_NAT64LSN,
1142 M_NOWAIT | M_ZERO);
1143 if (pg->states_chunk == NULL) {
1144 free(pg->freemask_chunk, M_NAT64LSN);
1145 uma_zfree(nat64lsn_pg_zone, pg);
1146 return (PG_ERROR(5));
1147 }
1148 for (i = 0; i < pg->chunks_count; i++) {
1149 pg->states_chunk[i] = uma_zalloc(
1150 nat64lsn_state_zone, M_NOWAIT);
1151 if (pg->states_chunk[i] == NULL)
1152 goto states_failed;
1153 }
1154 memset(pg->freemask_chunk, 0xff,
1155 sizeof(uint64_t) * pg->chunks_count);
1156 } else {
1157 pg->states = uma_zalloc(nat64lsn_state_zone, M_NOWAIT);
1158 if (pg->states == NULL) {
1159 uma_zfree(nat64lsn_pg_zone, pg);
1160 return (PG_ERROR(6));
1161 }
1162 memset(&pg->freemask64, 0xff, sizeof(uint64_t));
1163 }
1164
1165 /* Initialize PG and hook it to pgchunk */
1166 SET_AGE(pg->timestamp);
1167 pg->proto = proto;
1168 pg->base_port = NAT64_MIN_PORT + 64 * pg_idx;
1169 ck_pr_store_ptr(&chunks[chunk_idx]->pgptr[pg_idx % 32], pg);
1170 ck_pr_fence_store();
1171 ck_pr_bts_32(&pgmask[pg_idx / 32], pg_idx % 32);
1172 ck_pr_store_ptr(pgptr, pg);
1173
1174 ALIAS_LOCK(alias);
1175 CK_SLIST_INSERT_HEAD(&alias->portgroups, pg, entries);
1176 SET_AGE(alias->timestamp);
1177 alias->portgroups_count++;
1178 ALIAS_UNLOCK(alias);
1179 NAT64STAT_INC(&cfg->base.stats, spgcreated);
1180 return (PG_ERROR(0));
1181
1182 states_failed:
1183 for (i = 0; i < pg->chunks_count; i++)
1184 uma_zfree(nat64lsn_state_zone, pg->states_chunk[i]);
1185 free(pg->freemask_chunk, M_NAT64LSN);
1186 free(pg->states_chunk, M_NAT64LSN);
1187 uma_zfree(nat64lsn_pg_zone, pg);
1188 return (PG_ERROR(7));
1189 }
1190
1191 static int
nat64lsn_alloc_pg(struct nat64lsn_cfg * cfg,struct nat64lsn_job_item * ji)1192 nat64lsn_alloc_pg(struct nat64lsn_cfg *cfg, struct nat64lsn_job_item *ji)
1193 {
1194 struct nat64lsn_aliaslink *link;
1195 struct nat64lsn_alias *alias;
1196 int ret;
1197
1198 link = nat64lsn_get_aliaslink(cfg, ji->host, &ji->f_id);
1199 if (link == NULL)
1200 return (PG_ERROR(1));
1201
1202 /*
1203 * TODO: check that we did not already allocated PG in
1204 * previous call.
1205 */
1206
1207 ret = 0;
1208 alias = link->alias;
1209 /* Find place in pgchunk where PG can be added */
1210 switch (ji->proto) {
1211 case IPPROTO_TCP:
1212 ret = nat64lsn_alloc_proto_pg(cfg, alias,
1213 &alias->tcp_chunkmask, alias->tcp_pgmask,
1214 alias->tcp, &alias->tcp_pg, ji->proto);
1215 break;
1216 case IPPROTO_UDP:
1217 ret = nat64lsn_alloc_proto_pg(cfg, alias,
1218 &alias->udp_chunkmask, alias->udp_pgmask,
1219 alias->udp, &alias->udp_pg, ji->proto);
1220 break;
1221 case IPPROTO_ICMP:
1222 ret = nat64lsn_alloc_proto_pg(cfg, alias,
1223 &alias->icmp_chunkmask, alias->icmp_pgmask,
1224 alias->icmp, &alias->icmp_pg, ji->proto);
1225 break;
1226 default:
1227 panic("%s: wrong proto %d", __func__, ji->proto);
1228 }
1229 if (ret == PG_ERROR(1)) {
1230 /*
1231 * PG_ERROR(1) means that alias lacks free PGs
1232 * XXX: try next alias.
1233 */
1234 printf("NAT64LSN: %s: failed to obtain PG\n",
1235 __func__);
1236 return (ret);
1237 }
1238 if (ret == PG_ERROR(0)) {
1239 ji->state = nat64lsn_get_state6to4(cfg, ji->host, &ji->f_id,
1240 ji->state_hval, ji->faddr, ji->port, ji->proto);
1241 if (ji->state == NULL)
1242 ret = PG_ERROR(8);
1243 else
1244 ji->done = 1;
1245 }
1246 return (ret);
1247 }
1248
1249 static void
nat64lsn_do_request(void * data)1250 nat64lsn_do_request(void *data)
1251 {
1252 struct epoch_tracker et;
1253 struct nat64lsn_job_head jhead;
1254 struct nat64lsn_job_item *ji, *ji2;
1255 struct nat64lsn_cfg *cfg;
1256 int jcount;
1257 uint8_t flags;
1258
1259 cfg = (struct nat64lsn_cfg *)data;
1260 if (cfg->jlen == 0)
1261 return;
1262
1263 CURVNET_SET(cfg->vp);
1264 STAILQ_INIT(&jhead);
1265
1266 /* Grab queue */
1267 JQUEUE_LOCK();
1268 STAILQ_SWAP(&jhead, &cfg->jhead, nat64lsn_job_item);
1269 jcount = cfg->jlen;
1270 cfg->jlen = 0;
1271 JQUEUE_UNLOCK();
1272
1273 /* TODO: check if we need to resize hash */
1274
1275 NAT64STAT_INC(&cfg->base.stats, jcalls);
1276 DPRINTF(DP_JQUEUE, "count=%d", jcount);
1277
1278 /*
1279 * TODO:
1280 * What we should do here is to build a hash
1281 * to ensure we don't have lots of duplicate requests.
1282 * Skip this for now.
1283 *
1284 * TODO: Limit per-call number of items
1285 */
1286
1287 NAT64LSN_EPOCH_ENTER(et);
1288 STAILQ_FOREACH(ji, &jhead, entries) {
1289 switch (ji->jtype) {
1290 case JTYPE_NEWHOST:
1291 if (nat64lsn_alloc_host(cfg, ji) != HOST_ERROR(0))
1292 NAT64STAT_INC(&cfg->base.stats, jhostfails);
1293 break;
1294 case JTYPE_NEWPORTGROUP:
1295 if (nat64lsn_alloc_pg(cfg, ji) != PG_ERROR(0))
1296 NAT64STAT_INC(&cfg->base.stats, jportfails);
1297 break;
1298 default:
1299 continue;
1300 }
1301 if (ji->done != 0) {
1302 flags = ji->proto != IPPROTO_TCP ? 0 :
1303 convert_tcp_flags(ji->f_id._flags);
1304 nat64lsn_translate6_internal(cfg, &ji->m,
1305 ji->state, flags);
1306 NAT64STAT_INC(&cfg->base.stats, jreinjected);
1307 }
1308 }
1309 NAT64LSN_EPOCH_EXIT(et);
1310
1311 ji = STAILQ_FIRST(&jhead);
1312 while (ji != NULL) {
1313 ji2 = STAILQ_NEXT(ji, entries);
1314 /*
1315 * In any case we must free mbuf if
1316 * translator did not consumed it.
1317 */
1318 m_freem(ji->m);
1319 uma_zfree(nat64lsn_job_zone, ji);
1320 ji = ji2;
1321 }
1322 CURVNET_RESTORE();
1323 }
1324
1325 static struct nat64lsn_job_item *
nat64lsn_create_job(struct nat64lsn_cfg * cfg,int jtype)1326 nat64lsn_create_job(struct nat64lsn_cfg *cfg, int jtype)
1327 {
1328 struct nat64lsn_job_item *ji;
1329
1330 /*
1331 * Do not try to lock possibly contested mutex if we're near the
1332 * limit. Drop packet instead.
1333 */
1334 ji = NULL;
1335 if (cfg->jlen >= cfg->jmaxlen)
1336 NAT64STAT_INC(&cfg->base.stats, jmaxlen);
1337 else {
1338 ji = uma_zalloc(nat64lsn_job_zone, M_NOWAIT);
1339 if (ji == NULL)
1340 NAT64STAT_INC(&cfg->base.stats, jnomem);
1341 }
1342 if (ji == NULL) {
1343 NAT64STAT_INC(&cfg->base.stats, dropped);
1344 DPRINTF(DP_DROPS, "failed to create job");
1345 } else {
1346 ji->jtype = jtype;
1347 ji->done = 0;
1348 }
1349 return (ji);
1350 }
1351
1352 static void
nat64lsn_enqueue_job(struct nat64lsn_cfg * cfg,struct nat64lsn_job_item * ji)1353 nat64lsn_enqueue_job(struct nat64lsn_cfg *cfg, struct nat64lsn_job_item *ji)
1354 {
1355
1356 JQUEUE_LOCK();
1357 STAILQ_INSERT_TAIL(&cfg->jhead, ji, entries);
1358 NAT64STAT_INC(&cfg->base.stats, jrequests);
1359 cfg->jlen++;
1360
1361 if (callout_pending(&cfg->jcallout) == 0)
1362 callout_reset(&cfg->jcallout, 1, nat64lsn_do_request, cfg);
1363 JQUEUE_UNLOCK();
1364 }
1365
1366 static void
nat64lsn_job_destroy(epoch_context_t ctx)1367 nat64lsn_job_destroy(epoch_context_t ctx)
1368 {
1369 struct nat64lsn_job_item *ji;
1370 struct nat64lsn_host *host;
1371 struct nat64lsn_pg *pg;
1372 int i;
1373
1374 ji = __containerof(ctx, struct nat64lsn_job_item, epoch_ctx);
1375 MPASS(ji->jtype == JTYPE_DESTROY);
1376 while (!CK_SLIST_EMPTY(&ji->hosts)) {
1377 host = CK_SLIST_FIRST(&ji->hosts);
1378 CK_SLIST_REMOVE_HEAD(&ji->hosts, entries);
1379 if (host->states_count > 0) {
1380 /*
1381 * XXX: The state has been created
1382 * during host deletion.
1383 */
1384 printf("NAT64LSN: %s: destroying host with %d "
1385 "states\n", __func__, host->states_count);
1386 }
1387 nat64lsn_destroy_host(host);
1388 }
1389 while (!CK_SLIST_EMPTY(&ji->portgroups)) {
1390 pg = CK_SLIST_FIRST(&ji->portgroups);
1391 CK_SLIST_REMOVE_HEAD(&ji->portgroups, entries);
1392 for (i = 0; i < pg->chunks_count; i++) {
1393 if (FREEMASK_BITCOUNT(pg, i) != 64) {
1394 /*
1395 * XXX: The state has been created during
1396 * PG deletion.
1397 */
1398 printf("NAT64LSN: %s: destroying PG %p "
1399 "with non-empty chunk %d\n", __func__,
1400 pg, i);
1401 }
1402 }
1403 nat64lsn_destroy_pg(pg);
1404 }
1405 uma_zfree(nat64lsn_pgchunk_zone, ji->pgchunk);
1406 uma_zfree(nat64lsn_job_zone, ji);
1407 }
1408
1409 static int
nat64lsn_request_host(struct nat64lsn_cfg * cfg,const struct ipfw_flow_id * f_id,struct mbuf ** mp,uint32_t hval,in_addr_t faddr,uint16_t port,uint8_t proto)1410 nat64lsn_request_host(struct nat64lsn_cfg *cfg,
1411 const struct ipfw_flow_id *f_id, struct mbuf **mp, uint32_t hval,
1412 in_addr_t faddr, uint16_t port, uint8_t proto)
1413 {
1414 struct nat64lsn_job_item *ji;
1415
1416 ji = nat64lsn_create_job(cfg, JTYPE_NEWHOST);
1417 if (ji != NULL) {
1418 ji->m = *mp;
1419 ji->f_id = *f_id;
1420 ji->faddr = faddr;
1421 ji->port = port;
1422 ji->proto = proto;
1423 ji->src6_hval = hval;
1424
1425 nat64lsn_enqueue_job(cfg, ji);
1426 NAT64STAT_INC(&cfg->base.stats, jhostsreq);
1427 *mp = NULL;
1428 }
1429 return (IP_FW_DENY);
1430 }
1431
1432 static int
nat64lsn_request_pg(struct nat64lsn_cfg * cfg,struct nat64lsn_host * host,const struct ipfw_flow_id * f_id,struct mbuf ** mp,uint32_t hval,in_addr_t faddr,uint16_t port,uint8_t proto)1433 nat64lsn_request_pg(struct nat64lsn_cfg *cfg, struct nat64lsn_host *host,
1434 const struct ipfw_flow_id *f_id, struct mbuf **mp, uint32_t hval,
1435 in_addr_t faddr, uint16_t port, uint8_t proto)
1436 {
1437 struct nat64lsn_job_item *ji;
1438
1439 ji = nat64lsn_create_job(cfg, JTYPE_NEWPORTGROUP);
1440 if (ji != NULL) {
1441 ji->m = *mp;
1442 ji->f_id = *f_id;
1443 ji->faddr = faddr;
1444 ji->port = port;
1445 ji->proto = proto;
1446 ji->state_hval = hval;
1447 ji->host = host;
1448
1449 nat64lsn_enqueue_job(cfg, ji);
1450 NAT64STAT_INC(&cfg->base.stats, jportreq);
1451 *mp = NULL;
1452 }
1453 return (IP_FW_DENY);
1454 }
1455
1456 static int
nat64lsn_translate6_internal(struct nat64lsn_cfg * cfg,struct mbuf ** mp,struct nat64lsn_state * state,uint8_t flags)1457 nat64lsn_translate6_internal(struct nat64lsn_cfg *cfg, struct mbuf **mp,
1458 struct nat64lsn_state *state, uint8_t flags)
1459 {
1460 struct pfloghdr loghdr, *logdata;
1461 int ret;
1462 uint16_t ts;
1463
1464 /* Update timestamp and flags if needed */
1465 SET_AGE(ts);
1466 if (state->timestamp != ts)
1467 state->timestamp = ts;
1468 if ((state->flags & flags) != 0)
1469 state->flags |= flags;
1470
1471 if (cfg->base.flags & NAT64_LOG) {
1472 logdata = &loghdr;
1473 nat64lsn_log(logdata, *mp, AF_INET6, state);
1474 } else
1475 logdata = NULL;
1476
1477 ret = nat64_do_handle_ip6(*mp, htonl(state->ip_src),
1478 htons(state->aport), &cfg->base, logdata);
1479 if (ret == NAT64SKIP)
1480 return (cfg->nomatch_verdict);
1481 if (ret == NAT64RETURN)
1482 *mp = NULL;
1483 return (IP_FW_DENY);
1484 }
1485
1486 static int
nat64lsn_translate6(struct nat64lsn_cfg * cfg,struct ipfw_flow_id * f_id,struct mbuf ** mp)1487 nat64lsn_translate6(struct nat64lsn_cfg *cfg, struct ipfw_flow_id *f_id,
1488 struct mbuf **mp)
1489 {
1490 struct nat64lsn_state *state;
1491 struct nat64lsn_host *host;
1492 struct icmp6_hdr *icmp6;
1493 uint32_t addr, hval, data[2];
1494 int offset, proto;
1495 uint16_t port;
1496 uint8_t flags;
1497
1498 /* Check if protocol is supported */
1499 port = f_id->src_port;
1500 proto = f_id->proto;
1501 switch (f_id->proto) {
1502 case IPPROTO_ICMPV6:
1503 /*
1504 * For ICMPv6 echo reply/request we use icmp6_id as
1505 * local port.
1506 */
1507 offset = 0;
1508 proto = nat64_getlasthdr(*mp, &offset);
1509 if (proto < 0) {
1510 NAT64STAT_INC(&cfg->base.stats, dropped);
1511 DPRINTF(DP_DROPS, "mbuf isn't contigious");
1512 return (IP_FW_DENY);
1513 }
1514 if (proto == IPPROTO_ICMPV6) {
1515 icmp6 = mtodo(*mp, offset);
1516 if (icmp6->icmp6_type == ICMP6_ECHO_REQUEST ||
1517 icmp6->icmp6_type == ICMP6_ECHO_REPLY)
1518 port = ntohs(icmp6->icmp6_id);
1519 }
1520 proto = IPPROTO_ICMP;
1521 /* FALLTHROUGH */
1522 case IPPROTO_TCP:
1523 case IPPROTO_UDP:
1524 break;
1525 default:
1526 NAT64STAT_INC(&cfg->base.stats, noproto);
1527 return (cfg->nomatch_verdict);
1528 }
1529
1530 /* Extract IPv4 from destination IPv6 address */
1531 addr = nat64_extract_ip4(&f_id->dst_ip6, cfg->base.plat_plen);
1532 if (addr == 0 || nat64_check_private_ip4(&cfg->base, addr) != 0) {
1533 char a[INET_ADDRSTRLEN];
1534
1535 NAT64STAT_INC(&cfg->base.stats, dropped);
1536 DPRINTF(DP_DROPS, "dropped due to embedded IPv4 address %s",
1537 inet_ntop(AF_INET, &addr, a, sizeof(a)));
1538 return (IP_FW_DENY); /* XXX: add extra stats? */
1539 }
1540
1541 /* Try to find host */
1542 hval = HOST_HVAL(cfg, &f_id->src_ip6);
1543 CK_SLIST_FOREACH(host, &HOSTS(cfg, hval), entries) {
1544 if (IN6_ARE_ADDR_EQUAL(&f_id->src_ip6, &host->addr))
1545 break;
1546 }
1547 /* We use IPv4 address in host byte order */
1548 addr = ntohl(addr);
1549 if (host == NULL)
1550 return (nat64lsn_request_host(cfg, f_id, mp,
1551 hval, addr, port, proto));
1552
1553 flags = proto != IPPROTO_TCP ? 0 : convert_tcp_flags(f_id->_flags);
1554
1555 data[0] = addr;
1556 data[1] = (f_id->dst_port << 16) | port;
1557 hval = STATE_HVAL(cfg, data);
1558 state = nat64lsn_get_state6to4(cfg, host, f_id, hval, addr,
1559 port, proto);
1560 if (state == NULL)
1561 return (nat64lsn_request_pg(cfg, host, f_id, mp, hval, addr,
1562 port, proto));
1563 return (nat64lsn_translate6_internal(cfg, mp, state, flags));
1564 }
1565
1566 /*
1567 * Main dataplane entry point.
1568 */
1569 int
ipfw_nat64lsn(struct ip_fw_chain * ch,struct ip_fw_args * args,ipfw_insn * cmd,int * done)1570 ipfw_nat64lsn(struct ip_fw_chain *ch, struct ip_fw_args *args,
1571 ipfw_insn *cmd, int *done)
1572 {
1573 struct nat64lsn_cfg *cfg;
1574 ipfw_insn *icmd;
1575 int ret;
1576
1577 IPFW_RLOCK_ASSERT(ch);
1578
1579 *done = 0; /* continue the search in case of failure */
1580 icmd = cmd + 1;
1581 if (cmd->opcode != O_EXTERNAL_ACTION ||
1582 cmd->arg1 != V_nat64lsn_eid ||
1583 icmd->opcode != O_EXTERNAL_INSTANCE ||
1584 (cfg = NAT64_LOOKUP(ch, icmd)) == NULL)
1585 return (IP_FW_DENY);
1586
1587 *done = 1; /* terminate the search */
1588
1589 switch (args->f_id.addr_type) {
1590 case 4:
1591 ret = nat64lsn_translate4(cfg, &args->f_id, &args->m);
1592 break;
1593 case 6:
1594 /*
1595 * Check that destination IPv6 address matches our prefix6.
1596 */
1597 if ((cfg->base.flags & NAT64LSN_ANYPREFIX) == 0 &&
1598 memcmp(&args->f_id.dst_ip6, &cfg->base.plat_prefix,
1599 cfg->base.plat_plen / 8) != 0) {
1600 ret = cfg->nomatch_verdict;
1601 break;
1602 }
1603 ret = nat64lsn_translate6(cfg, &args->f_id, &args->m);
1604 break;
1605 default:
1606 ret = cfg->nomatch_verdict;
1607 }
1608
1609 if (ret != IP_FW_PASS && args->m != NULL) {
1610 m_freem(args->m);
1611 args->m = NULL;
1612 }
1613 return (ret);
1614 }
1615
1616 static int
nat64lsn_state_ctor(void * mem,int size,void * arg,int flags)1617 nat64lsn_state_ctor(void *mem, int size, void *arg, int flags)
1618 {
1619 struct nat64lsn_states_chunk *chunk;
1620 int i;
1621
1622 chunk = (struct nat64lsn_states_chunk *)mem;
1623 for (i = 0; i < 64; i++)
1624 chunk->state[i].flags = 0;
1625 return (0);
1626 }
1627
1628 void
nat64lsn_init_internal(void)1629 nat64lsn_init_internal(void)
1630 {
1631
1632 nat64lsn_host_zone = uma_zcreate("NAT64LSN hosts",
1633 sizeof(struct nat64lsn_host), NULL, NULL, NULL, NULL,
1634 UMA_ALIGN_PTR, 0);
1635 nat64lsn_pgchunk_zone = uma_zcreate("NAT64LSN portgroup chunks",
1636 sizeof(struct nat64lsn_pgchunk), NULL, NULL, NULL, NULL,
1637 UMA_ALIGN_PTR, 0);
1638 nat64lsn_pg_zone = uma_zcreate("NAT64LSN portgroups",
1639 sizeof(struct nat64lsn_pg), NULL, NULL, NULL, NULL,
1640 UMA_ALIGN_PTR, 0);
1641 nat64lsn_aliaslink_zone = uma_zcreate("NAT64LSN links",
1642 sizeof(struct nat64lsn_aliaslink), NULL, NULL, NULL, NULL,
1643 UMA_ALIGN_PTR, 0);
1644 nat64lsn_state_zone = uma_zcreate("NAT64LSN states",
1645 sizeof(struct nat64lsn_states_chunk), nat64lsn_state_ctor,
1646 NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1647 nat64lsn_job_zone = uma_zcreate("NAT64LSN jobs",
1648 sizeof(struct nat64lsn_job_item), NULL, NULL, NULL, NULL,
1649 UMA_ALIGN_PTR, 0);
1650 JQUEUE_LOCK_INIT();
1651 }
1652
1653 void
nat64lsn_uninit_internal(void)1654 nat64lsn_uninit_internal(void)
1655 {
1656
1657 /* XXX: epoch_task drain */
1658 JQUEUE_LOCK_DESTROY();
1659 uma_zdestroy(nat64lsn_host_zone);
1660 uma_zdestroy(nat64lsn_pgchunk_zone);
1661 uma_zdestroy(nat64lsn_pg_zone);
1662 uma_zdestroy(nat64lsn_aliaslink_zone);
1663 uma_zdestroy(nat64lsn_state_zone);
1664 uma_zdestroy(nat64lsn_job_zone);
1665 }
1666
1667 void
nat64lsn_start_instance(struct nat64lsn_cfg * cfg)1668 nat64lsn_start_instance(struct nat64lsn_cfg *cfg)
1669 {
1670
1671 CALLOUT_LOCK(cfg);
1672 callout_reset(&cfg->periodic, hz * PERIODIC_DELAY,
1673 nat64lsn_periodic, cfg);
1674 CALLOUT_UNLOCK(cfg);
1675 }
1676
1677 struct nat64lsn_cfg *
nat64lsn_init_instance(struct ip_fw_chain * ch,in_addr_t prefix,int plen)1678 nat64lsn_init_instance(struct ip_fw_chain *ch, in_addr_t prefix, int plen)
1679 {
1680 struct nat64lsn_cfg *cfg;
1681 struct nat64lsn_alias *alias;
1682 int i, naddr;
1683
1684 cfg = malloc(sizeof(struct nat64lsn_cfg), M_NAT64LSN,
1685 M_WAITOK | M_ZERO);
1686
1687 CFG_LOCK_INIT(cfg);
1688 CALLOUT_LOCK_INIT(cfg);
1689 STAILQ_INIT(&cfg->jhead);
1690 cfg->vp = curvnet;
1691 COUNTER_ARRAY_ALLOC(cfg->base.stats.cnt, NAT64STATS, M_WAITOK);
1692
1693 cfg->hash_seed = arc4random();
1694 cfg->hosts_hashsize = NAT64LSN_HOSTS_HSIZE;
1695 cfg->hosts_hash = malloc(sizeof(struct nat64lsn_hosts_slist) *
1696 cfg->hosts_hashsize, M_NAT64LSN, M_WAITOK | M_ZERO);
1697 for (i = 0; i < cfg->hosts_hashsize; i++)
1698 CK_SLIST_INIT(&cfg->hosts_hash[i]);
1699
1700 naddr = 1 << (32 - plen);
1701 cfg->prefix4 = prefix;
1702 cfg->pmask4 = prefix | (naddr - 1);
1703 cfg->plen4 = plen;
1704 cfg->aliases = malloc(sizeof(struct nat64lsn_alias) * naddr,
1705 M_NAT64LSN, M_WAITOK | M_ZERO);
1706 for (i = 0; i < naddr; i++) {
1707 alias = &cfg->aliases[i];
1708 alias->addr = prefix + i; /* host byte order */
1709 CK_SLIST_INIT(&alias->hosts);
1710 ALIAS_LOCK_INIT(alias);
1711 }
1712
1713 callout_init_mtx(&cfg->periodic, &cfg->periodic_lock, 0);
1714 callout_init(&cfg->jcallout, CALLOUT_MPSAFE);
1715
1716 return (cfg);
1717 }
1718
1719 static void
nat64lsn_destroy_pg(struct nat64lsn_pg * pg)1720 nat64lsn_destroy_pg(struct nat64lsn_pg *pg)
1721 {
1722 int i;
1723
1724 if (pg->chunks_count == 1) {
1725 uma_zfree(nat64lsn_state_zone, pg->states);
1726 } else {
1727 for (i = 0; i < pg->chunks_count; i++)
1728 uma_zfree(nat64lsn_state_zone, pg->states_chunk[i]);
1729 free(pg->states_chunk, M_NAT64LSN);
1730 free(pg->freemask_chunk, M_NAT64LSN);
1731 }
1732 uma_zfree(nat64lsn_pg_zone, pg);
1733 }
1734
1735 static void
nat64lsn_destroy_alias(struct nat64lsn_cfg * cfg,struct nat64lsn_alias * alias)1736 nat64lsn_destroy_alias(struct nat64lsn_cfg *cfg,
1737 struct nat64lsn_alias *alias)
1738 {
1739 struct nat64lsn_pg *pg;
1740 int i;
1741
1742 while (!CK_SLIST_EMPTY(&alias->portgroups)) {
1743 pg = CK_SLIST_FIRST(&alias->portgroups);
1744 CK_SLIST_REMOVE_HEAD(&alias->portgroups, entries);
1745 nat64lsn_destroy_pg(pg);
1746 }
1747 for (i = 0; i < 32; i++) {
1748 if (ISSET32(alias->tcp_chunkmask, i))
1749 uma_zfree(nat64lsn_pgchunk_zone, alias->tcp[i]);
1750 if (ISSET32(alias->udp_chunkmask, i))
1751 uma_zfree(nat64lsn_pgchunk_zone, alias->udp[i]);
1752 if (ISSET32(alias->icmp_chunkmask, i))
1753 uma_zfree(nat64lsn_pgchunk_zone, alias->icmp[i]);
1754 }
1755 ALIAS_LOCK_DESTROY(alias);
1756 }
1757
1758 static void
nat64lsn_destroy_host(struct nat64lsn_host * host)1759 nat64lsn_destroy_host(struct nat64lsn_host *host)
1760 {
1761 struct nat64lsn_aliaslink *link;
1762
1763 while (!CK_SLIST_EMPTY(&host->aliases)) {
1764 link = CK_SLIST_FIRST(&host->aliases);
1765 CK_SLIST_REMOVE_HEAD(&host->aliases, host_entries);
1766
1767 ALIAS_LOCK(link->alias);
1768 CK_SLIST_REMOVE(&link->alias->hosts, link,
1769 nat64lsn_aliaslink, alias_entries);
1770 link->alias->hosts_count--;
1771 ALIAS_UNLOCK(link->alias);
1772
1773 uma_zfree(nat64lsn_aliaslink_zone, link);
1774 }
1775 HOST_LOCK_DESTROY(host);
1776 free(host->states_hash, M_NAT64LSN);
1777 uma_zfree(nat64lsn_host_zone, host);
1778 }
1779
1780 void
nat64lsn_destroy_instance(struct nat64lsn_cfg * cfg)1781 nat64lsn_destroy_instance(struct nat64lsn_cfg *cfg)
1782 {
1783 struct nat64lsn_host *host;
1784 int i;
1785
1786 CALLOUT_LOCK(cfg);
1787 callout_drain(&cfg->periodic);
1788 CALLOUT_UNLOCK(cfg);
1789 callout_drain(&cfg->jcallout);
1790
1791 for (i = 0; i < cfg->hosts_hashsize; i++) {
1792 while (!CK_SLIST_EMPTY(&cfg->hosts_hash[i])) {
1793 host = CK_SLIST_FIRST(&cfg->hosts_hash[i]);
1794 CK_SLIST_REMOVE_HEAD(&cfg->hosts_hash[i], entries);
1795 nat64lsn_destroy_host(host);
1796 }
1797 }
1798
1799 for (i = 0; i < (1 << (32 - cfg->plen4)); i++)
1800 nat64lsn_destroy_alias(cfg, &cfg->aliases[i]);
1801
1802 CALLOUT_LOCK_DESTROY(cfg);
1803 CFG_LOCK_DESTROY(cfg);
1804 COUNTER_ARRAY_FREE(cfg->base.stats.cnt, NAT64STATS);
1805 free(cfg->hosts_hash, M_NAT64LSN);
1806 free(cfg->aliases, M_NAT64LSN);
1807 free(cfg, M_NAT64LSN);
1808 }
1809