1 /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3 * memcached - memory caching daemon
4 *
5 * https://www.memcached.org/
6 *
7 * Copyright 2003 Danga Interactive, Inc. All rights reserved.
8 *
9 * Use and distribution licensed under the BSD license. See
10 * the LICENSE file for full text.
11 *
12 * Authors:
13 * Anatoly Vorobey <[email protected]>
14 * Brad Fitzpatrick <[email protected]>
15 */
16 #include "memcached.h"
17 #include "storage.h"
18 #include "authfile.h"
19 #include "restart.h"
20 #include <sys/stat.h>
21 #include <sys/socket.h>
22 #include <sys/un.h>
23 #include <signal.h>
24 #include <sys/param.h>
25 #include <sys/resource.h>
26 #include <sys/uio.h>
27 #include <ctype.h>
28 #include <stdarg.h>
29
30 /* some POSIX systems need the following definition
31 * to get mlockall flags out of sys/mman.h. */
32 #ifndef _P1003_1B_VISIBLE
33 #define _P1003_1B_VISIBLE
34 #endif
35 #include <pwd.h>
36 #include <sys/mman.h>
37 #include <fcntl.h>
38 #include <netinet/tcp.h>
39 #include <arpa/inet.h>
40 #include <errno.h>
41 #include <stdlib.h>
42 #include <stdio.h>
43 #include <string.h>
44 #include <time.h>
45 #include <assert.h>
46 #include <sysexits.h>
47 #include <stddef.h>
48
49 #ifdef HAVE_GETOPT_LONG
50 #include <getopt.h>
51 #endif
52
53 #include "tls.h"
54
55 #include "proto_text.h"
56 #include "proto_bin.h"
57 #include "proto_proxy.h"
58
59 #if defined(__FreeBSD__)
60 #include <sys/sysctl.h>
61 #endif
62
63 /*
64 * forward declarations
65 */
66 static void drive_machine(conn *c);
67 static int new_socket(struct addrinfo *ai);
68 static ssize_t tcp_read(conn *arg, void *buf, size_t count);
69 static ssize_t tcp_sendmsg(conn *arg, struct msghdr *msg, int flags);
70 static ssize_t tcp_write(conn *arg, void *buf, size_t count);
71
72 enum try_read_result {
73 READ_DATA_RECEIVED,
74 READ_NO_DATA_RECEIVED,
75 READ_ERROR, /** an error occurred (on the socket) (or client closed connection) */
76 READ_MEMORY_ERROR /** failed to allocate more memory */
77 };
78
79 static int try_read_command_negotiate(conn *c);
80 static int try_read_command_udp(conn *c);
81
82 static enum try_read_result try_read_network(conn *c);
83 static enum try_read_result try_read_udp(conn *c);
84
85 static int start_conn_timeout_thread(void);
86
87 /* stats */
88 static void stats_init(void);
89 static void conn_to_str(const conn *c, char *addr, char *svr_addr);
90
91 /* defaults */
92 static void settings_init(void);
93
94 /* event handling, network IO */
95 static void event_handler(const evutil_socket_t fd, const short which, void *arg);
96 static void conn_close(conn *c);
97 static void conn_init(void);
98 static bool update_event(conn *c, const int new_flags);
99 static void complete_nread(conn *c);
100
101 static void conn_free(conn *c);
102
103 /** exported globals **/
104 struct stats stats;
105 struct stats_state stats_state;
106 struct settings settings;
107 time_t process_started; /* when the process was started */
108 conn **conns;
109
110 struct slab_rebalance slab_rebal;
111 volatile int slab_rebalance_signal;
112 #ifdef EXTSTORE
113 /* hoping this is temporary; I'd prefer to cut globals, but will complete this
114 * battle another day.
115 */
116 void *ext_storage = NULL;
117 #endif
118 /** file scope variables **/
119 static conn *listen_conn = NULL;
120 static int max_fds;
121 static struct event_base *main_base;
122
123 enum transmit_result {
124 TRANSMIT_COMPLETE, /** All done writing. */
125 TRANSMIT_INCOMPLETE, /** More data remaining to write. */
126 TRANSMIT_SOFT_ERROR, /** Can't write any more right now. */
127 TRANSMIT_HARD_ERROR /** Can't write (c->state is set to conn_closing) */
128 };
129
130 /* Default methods to read from/ write to a socket */
tcp_read(conn * c,void * buf,size_t count)131 ssize_t tcp_read(conn *c, void *buf, size_t count) {
132 assert (c != NULL);
133 return read(c->sfd, buf, count);
134 }
135
tcp_sendmsg(conn * c,struct msghdr * msg,int flags)136 ssize_t tcp_sendmsg(conn *c, struct msghdr *msg, int flags) {
137 assert (c != NULL);
138 return sendmsg(c->sfd, msg, flags);
139 }
140
tcp_write(conn * c,void * buf,size_t count)141 ssize_t tcp_write(conn *c, void *buf, size_t count) {
142 assert (c != NULL);
143 return write(c->sfd, buf, count);
144 }
145
146 static enum transmit_result transmit(conn *c);
147
148 /* This reduces the latency without adding lots of extra wiring to be able to
149 * notify the listener thread of when to listen again.
150 * Also, the clock timer could be broken out into its own thread and we
151 * can block the listener via a condition.
152 */
153 static volatile bool allow_new_conns = true;
154 static int stop_main_loop = NOT_STOP;
155 static struct event maxconnsevent;
maxconns_handler(const evutil_socket_t fd,const short which,void * arg)156 static void maxconns_handler(const evutil_socket_t fd, const short which, void *arg) {
157 struct timeval t = {.tv_sec = 0, .tv_usec = 10000};
158
159 if (fd == -42 || allow_new_conns == false) {
160 /* reschedule in 10ms if we need to keep polling */
161 evtimer_set(&maxconnsevent, maxconns_handler, 0);
162 event_base_set(main_base, &maxconnsevent);
163 evtimer_add(&maxconnsevent, &t);
164 } else {
165 evtimer_del(&maxconnsevent);
166 accept_new_conns(true);
167 }
168 }
169
170 /*
171 * given time value that's either unix time or delta from current unix time, return
172 * unix time. Use the fact that delta can't exceed one month (and real time value can't
173 * be that low).
174 */
realtime(const time_t exptime)175 rel_time_t realtime(const time_t exptime) {
176 /* no. of seconds in 30 days - largest possible delta exptime */
177
178 if (exptime == 0) return 0; /* 0 means never expire */
179
180 if (exptime > REALTIME_MAXDELTA) {
181 /* if item expiration is at/before the server started, give it an
182 expiration time of 1 second after the server started.
183 (because 0 means don't expire). without this, we'd
184 underflow and wrap around to some large value way in the
185 future, effectively making items expiring in the past
186 really expiring never */
187 if (exptime <= process_started)
188 return (rel_time_t)1;
189 return (rel_time_t)(exptime - process_started);
190 } else {
191 return (rel_time_t)(exptime + current_time);
192 }
193 }
194
stats_init(void)195 static void stats_init(void) {
196 memset(&stats, 0, sizeof(struct stats));
197 memset(&stats_state, 0, sizeof(struct stats_state));
198 stats_state.accepting_conns = true; /* assuming we start in this state. */
199
200 /* make the time we started always be 2 seconds before we really
201 did, so time(0) - time.started is never zero. if so, things
202 like 'settings.oldest_live' which act as booleans as well as
203 values are now false in boolean context... */
204 process_started = time(0) - ITEM_UPDATE_INTERVAL - 2;
205 stats_prefix_init(settings.prefix_delimiter);
206 }
207
stats_reset(void)208 void stats_reset(void) {
209 STATS_LOCK();
210 memset(&stats, 0, sizeof(struct stats));
211 stats_prefix_clear();
212 STATS_UNLOCK();
213 threadlocal_stats_reset();
214 item_stats_reset();
215 }
216
settings_init(void)217 static void settings_init(void) {
218 settings.use_cas = true;
219 settings.access = 0700;
220 settings.port = 11211;
221 settings.udpport = 0;
222 ssl_init_settings();
223 /* By default this string should be NULL for getaddrinfo() */
224 settings.inter = NULL;
225 settings.maxbytes = 64 * 1024 * 1024; /* default is 64MB */
226 settings.maxconns = 1024; /* to limit connections-related memory to about 5MB */
227 settings.verbose = 0;
228 settings.oldest_live = 0;
229 settings.evict_to_free = 1; /* push old items out of cache when memory runs out */
230 settings.socketpath = NULL; /* by default, not using a unix socket */
231 settings.auth_file = NULL; /* by default, not using ASCII authentication tokens */
232 settings.factor = 1.25;
233 settings.chunk_size = 48; /* space for a modest key and value */
234 settings.num_threads = 4; /* N workers */
235 settings.num_threads_per_udp = 0;
236 settings.prefix_delimiter = ':';
237 settings.detail_enabled = 0;
238 settings.reqs_per_event = 20;
239 settings.backlog = 1024;
240 settings.binding_protocol = negotiating_prot;
241 settings.item_size_max = 1024 * 1024; /* The famous 1MB upper limit. */
242 settings.slab_page_size = 1024 * 1024; /* chunks are split from 1MB pages. */
243 settings.slab_chunk_size_max = settings.slab_page_size / 2;
244 settings.sasl = false;
245 settings.maxconns_fast = true;
246 settings.lru_crawler = false;
247 settings.lru_crawler_sleep = 100;
248 settings.lru_crawler_tocrawl = 0;
249 settings.lru_maintainer_thread = false;
250 settings.lru_segmented = true;
251 settings.hot_lru_pct = 20;
252 settings.warm_lru_pct = 40;
253 settings.hot_max_factor = 0.2;
254 settings.warm_max_factor = 2.0;
255 settings.temp_lru = false;
256 settings.temporary_ttl = 61;
257 settings.idle_timeout = 0; /* disabled */
258 settings.hashpower_init = 0;
259 settings.slab_reassign = true;
260 settings.slab_automove = 1;
261 settings.slab_automove_ratio = 0.8;
262 settings.slab_automove_window = 30;
263 settings.shutdown_command = false;
264 settings.tail_repair_time = TAIL_REPAIR_TIME_DEFAULT;
265 settings.flush_enabled = true;
266 settings.dump_enabled = true;
267 settings.crawls_persleep = 1000;
268 settings.logger_watcher_buf_size = LOGGER_WATCHER_BUF_SIZE;
269 settings.logger_buf_size = LOGGER_BUF_SIZE;
270 settings.drop_privileges = false;
271 settings.watch_enabled = true;
272 settings.read_buf_mem_limit = 0;
273 #ifdef MEMCACHED_DEBUG
274 settings.relaxed_privileges = false;
275 #endif
276 settings.num_napi_ids = 0;
277 settings.memory_file = NULL;
278 #ifdef SOCK_COOKIE_ID
279 settings.sock_cookie_id = 0;
280 #endif
281 }
282
283 extern pthread_mutex_t conn_lock;
284
285 /* Connection timeout thread bits */
286 static pthread_t conn_timeout_tid;
287 static int do_run_conn_timeout_thread;
288 static pthread_cond_t conn_timeout_cond = PTHREAD_COND_INITIALIZER;
289 static pthread_mutex_t conn_timeout_lock = PTHREAD_MUTEX_INITIALIZER;
290
291 #define CONNS_PER_SLICE 100
conn_timeout_thread(void * arg)292 static void *conn_timeout_thread(void *arg) {
293 int i;
294 conn *c;
295 rel_time_t oldest_last_cmd;
296 int sleep_time;
297 int sleep_slice = max_fds / CONNS_PER_SLICE;
298 if (sleep_slice == 0)
299 sleep_slice = CONNS_PER_SLICE;
300
301 useconds_t timeslice = 1000000 / sleep_slice;
302
303 mutex_lock(&conn_timeout_lock);
304 while(do_run_conn_timeout_thread) {
305 if (settings.verbose > 2)
306 fprintf(stderr, "idle timeout thread at top of connection list\n");
307
308 oldest_last_cmd = current_time;
309
310 for (i = 0; i < max_fds; i++) {
311 if ((i % CONNS_PER_SLICE) == 0) {
312 if (settings.verbose > 2)
313 fprintf(stderr, "idle timeout thread sleeping for %ulus\n",
314 (unsigned int)timeslice);
315 usleep(timeslice);
316 }
317
318 if (!conns[i])
319 continue;
320
321 c = conns[i];
322
323 if (!IS_TCP(c->transport))
324 continue;
325
326 if (c->state != conn_new_cmd && c->state != conn_read)
327 continue;
328
329 if ((current_time - c->last_cmd_time) > settings.idle_timeout) {
330 timeout_conn(c);
331 } else {
332 if (c->last_cmd_time < oldest_last_cmd)
333 oldest_last_cmd = c->last_cmd_time;
334 }
335 }
336
337 /* This is the soonest we could have another connection time out */
338 sleep_time = settings.idle_timeout - (current_time - oldest_last_cmd) + 1;
339 if (sleep_time <= 0)
340 sleep_time = 1;
341
342 if (settings.verbose > 2)
343 fprintf(stderr,
344 "idle timeout thread finished pass, sleeping for %ds\n",
345 sleep_time);
346
347 struct timeval now;
348 struct timespec to_sleep;
349 gettimeofday(&now, NULL);
350 to_sleep.tv_sec = now.tv_sec + sleep_time;
351 to_sleep.tv_nsec = 0;
352
353 pthread_cond_timedwait(&conn_timeout_cond, &conn_timeout_lock, &to_sleep);
354 }
355
356 mutex_unlock(&conn_timeout_lock);
357 return NULL;
358 }
359
start_conn_timeout_thread(void)360 static int start_conn_timeout_thread(void) {
361 int ret;
362
363 if (settings.idle_timeout == 0)
364 return -1;
365
366 do_run_conn_timeout_thread = 1;
367 if ((ret = pthread_create(&conn_timeout_tid, NULL,
368 conn_timeout_thread, NULL)) != 0) {
369 fprintf(stderr, "Can't create idle connection timeout thread: %s\n",
370 strerror(ret));
371 return -1;
372 }
373 thread_setname(conn_timeout_tid, "mc-idletimeout");
374
375 return 0;
376 }
377
stop_conn_timeout_thread(void)378 int stop_conn_timeout_thread(void) {
379 if (!do_run_conn_timeout_thread)
380 return -1;
381 mutex_lock(&conn_timeout_lock);
382 do_run_conn_timeout_thread = 0;
383 pthread_cond_signal(&conn_timeout_cond);
384 mutex_unlock(&conn_timeout_lock);
385 pthread_join(conn_timeout_tid, NULL);
386 return 0;
387 }
388
389 /*
390 * read buffer cache helper functions
391 */
rbuf_release(conn * c)392 static void rbuf_release(conn *c) {
393 if (c->rbuf != NULL && c->rbytes == 0 && !IS_UDP(c->transport)) {
394 if (c->rbuf_malloced) {
395 free(c->rbuf);
396 c->rbuf_malloced = false;
397 } else {
398 do_cache_free(c->thread->rbuf_cache, c->rbuf);
399 }
400 c->rsize = 0;
401 c->rbuf = NULL;
402 c->rcurr = NULL;
403 }
404 }
405
rbuf_alloc(conn * c)406 static bool rbuf_alloc(conn *c) {
407 if (c->rbuf == NULL) {
408 c->rbuf = do_cache_alloc(c->thread->rbuf_cache);
409 if (!c->rbuf) {
410 THR_STATS_LOCK(c->thread);
411 c->thread->stats.read_buf_oom++;
412 THR_STATS_UNLOCK(c->thread);
413 return false;
414 }
415 c->rsize = READ_BUFFER_SIZE;
416 c->rcurr = c->rbuf;
417 }
418 return true;
419 }
420
421 // Just for handling huge ASCII multigets.
422 // The previous system was essentially the same; realloc'ing until big enough,
423 // then realloc'ing back down after the request finished.
rbuf_switch_to_malloc(conn * c)424 bool rbuf_switch_to_malloc(conn *c) {
425 // Might as well start with x2 and work from there.
426 size_t size = c->rsize * 2;
427 char *tmp = malloc(size);
428 if (!tmp)
429 return false;
430
431 memcpy(tmp, c->rcurr, c->rbytes);
432 do_cache_free(c->thread->rbuf_cache, c->rbuf);
433
434 c->rcurr = c->rbuf = tmp;
435 c->rsize = size;
436 c->rbuf_malloced = true;
437 return true;
438 }
439
440 /*
441 * Initializes the connections array. We don't actually allocate connection
442 * structures until they're needed, so as to avoid wasting memory when the
443 * maximum connection count is much higher than the actual number of
444 * connections.
445 *
446 * This does end up wasting a few pointers' worth of memory for FDs that are
447 * used for things other than connections, but that's worth it in exchange for
448 * being able to directly index the conns array by FD.
449 */
conn_init(void)450 static void conn_init(void) {
451 /* We're unlikely to see an FD much higher than maxconns. */
452 int next_fd = dup(1);
453 if (next_fd < 0) {
454 perror("Failed to duplicate file descriptor\n");
455 exit(1);
456 }
457 int headroom = 10; /* account for extra unexpected open FDs */
458 struct rlimit rl;
459
460 max_fds = settings.maxconns + headroom + next_fd;
461
462 /* But if possible, get the actual highest FD we can possibly ever see. */
463 if (getrlimit(RLIMIT_NOFILE, &rl) == 0) {
464 max_fds = rl.rlim_max;
465 } else {
466 fprintf(stderr, "Failed to query maximum file descriptor; "
467 "falling back to maxconns\n");
468 }
469
470 close(next_fd);
471
472 if ((conns = calloc(max_fds, sizeof(conn *))) == NULL) {
473 fprintf(stderr, "Failed to allocate connection structures\n");
474 /* This is unrecoverable so bail out early. */
475 exit(1);
476 }
477 }
478
prot_text(enum protocol prot)479 static const char *prot_text(enum protocol prot) {
480 char *rv = "unknown";
481 switch(prot) {
482 case ascii_prot:
483 rv = "ascii";
484 break;
485 case binary_prot:
486 rv = "binary";
487 break;
488 case negotiating_prot:
489 rv = "auto-negotiate";
490 break;
491 #ifdef PROXY
492 case proxy_prot:
493 rv = "proxy";
494 break;
495 #endif
496 }
497 return rv;
498 }
499
conn_close_idle(conn * c)500 void conn_close_idle(conn *c) {
501 if (settings.idle_timeout > 0 &&
502 (current_time - c->last_cmd_time) > settings.idle_timeout) {
503 if (c->state != conn_new_cmd && c->state != conn_read) {
504 if (settings.verbose > 1)
505 fprintf(stderr,
506 "fd %d wants to timeout, but isn't in read state", c->sfd);
507 return;
508 }
509
510 if (settings.verbose > 1)
511 fprintf(stderr, "Closing idle fd %d\n", c->sfd);
512
513 pthread_mutex_lock(&c->thread->stats.mutex);
514 c->thread->stats.idle_kicks++;
515 pthread_mutex_unlock(&c->thread->stats.mutex);
516
517 c->close_reason = IDLE_TIMEOUT_CLOSE;
518
519 conn_set_state(c, conn_closing);
520 drive_machine(c);
521 }
522 }
523
_conn_event_readd(conn * c)524 static void _conn_event_readd(conn *c) {
525 c->ev_flags = EV_READ | EV_PERSIST;
526 event_set(&c->event, c->sfd, c->ev_flags, event_handler, (void *)c);
527 event_base_set(c->thread->base, &c->event);
528
529 // TODO: call conn_cleanup/fail/etc
530 if (event_add(&c->event, 0) == -1) {
531 perror("event_add");
532 }
533 }
534
535 /* bring conn back from a sidethread. could have had its event base moved. */
conn_worker_readd(conn * c)536 void conn_worker_readd(conn *c) {
537 if (c->io_queues_submitted) { // TODO: ensure this is safe?
538 c->io_queues_submitted--;
539 // If we're still waiting for other queues to return, don't re-add the
540 // connection yet.
541 if (c->io_queues_submitted != 0) {
542 return;
543 }
544 }
545
546 switch (c->state) {
547 case conn_closing:
548 drive_machine(c);
549 break;
550 case conn_io_pending:
551 // The event listener was removed as more data showed up while
552 // waiting for the async response.
553 _conn_event_readd(c);
554 // Explicit fall-through.
555 case conn_io_queue:
556 conn_set_state(c, conn_io_resume);
557 // machine will know how to return based on secondary state.
558 drive_machine(c);
559 break;
560 default:
561 event_del(&c->event);
562 _conn_event_readd(c);
563 conn_set_state(c, conn_new_cmd);
564 }
565
566 }
567
thread_io_queue_add(LIBEVENT_THREAD * t,int type,void * ctx,io_queue_stack_cb cb)568 void thread_io_queue_add(LIBEVENT_THREAD *t, int type, void *ctx, io_queue_stack_cb cb) {
569 io_queue_cb_t *q = t->io_queues;
570 while (q->type != IO_QUEUE_NONE) {
571 q++;
572 }
573 q->type = type;
574 q->ctx = ctx;
575 q->submit_cb = cb;
576 return;
577 }
578
conn_io_queue_setup(conn * c)579 void conn_io_queue_setup(conn *c) {
580 io_queue_cb_t *qcb = c->thread->io_queues;
581 io_queue_t *q = c->io_queues;
582 while (qcb->type != IO_QUEUE_NONE) {
583 q->type = qcb->type;
584 q->ctx = qcb->ctx;
585 q->stack_ctx = NULL;
586 q->count = 0;
587 qcb++;
588 q++;
589 }
590 }
591
592 // To be called from conn_release_items to ensure the stack ptrs are reset.
conn_io_queue_reset(conn * c)593 static void conn_io_queue_reset(conn *c) {
594 for (io_queue_t *q = c->io_queues; q->type != IO_QUEUE_NONE; q++) {
595 assert(q->count == 0);
596 q->stack_ctx = NULL;
597 }
598 }
599
thread_io_queue_get(LIBEVENT_THREAD * t,int type)600 io_queue_cb_t *thread_io_queue_get(LIBEVENT_THREAD *t, int type) {
601 io_queue_cb_t *q = t->io_queues;
602 while (q->type != IO_QUEUE_NONE) {
603 if (q->type == type) {
604 return q;
605 }
606 q++;
607 }
608 return NULL;
609 }
610
conn_io_queue_get(conn * c,int type)611 io_queue_t *conn_io_queue_get(conn *c, int type) {
612 io_queue_t *q = c->io_queues;
613 while (q->type != IO_QUEUE_NONE) {
614 if (q->type == type) {
615 return q;
616 }
617 q++;
618 }
619 return NULL;
620 }
621
622 // called to return a single IO object to the original worker thread.
conn_io_queue_return(io_pending_t * io)623 void conn_io_queue_return(io_pending_t *io) {
624 io->return_cb(io);
625 }
626
conn_new(const int sfd,enum conn_states init_state,const int event_flags,const int read_buffer_size,enum network_transport transport,struct event_base * base,void * ssl,uint64_t conntag,enum protocol bproto)627 conn *conn_new(const int sfd, enum conn_states init_state,
628 const int event_flags,
629 const int read_buffer_size, enum network_transport transport,
630 struct event_base *base, void *ssl, uint64_t conntag,
631 enum protocol bproto) {
632 conn *c;
633
634 assert(sfd >= 0 && sfd < max_fds);
635 c = conns[sfd];
636
637 if (NULL == c) {
638 if (!(c = (conn *)calloc(1, sizeof(conn)))) {
639 STATS_LOCK();
640 stats.malloc_fails++;
641 STATS_UNLOCK();
642 fprintf(stderr, "Failed to allocate connection object\n");
643 return NULL;
644 }
645 MEMCACHED_CONN_CREATE(c);
646 c->read = NULL;
647 c->sendmsg = NULL;
648 c->write = NULL;
649 c->rbuf = NULL;
650
651 c->rsize = read_buffer_size;
652
653 // UDP connections use a persistent static buffer.
654 if (c->rsize) {
655 c->rbuf = (char *)malloc((size_t)c->rsize);
656 }
657
658 if (c->rsize && c->rbuf == NULL) {
659 conn_free(c);
660 STATS_LOCK();
661 stats.malloc_fails++;
662 STATS_UNLOCK();
663 fprintf(stderr, "Failed to allocate buffers for connection\n");
664 return NULL;
665 }
666
667
668 STATS_LOCK();
669 stats_state.conn_structs++;
670 STATS_UNLOCK();
671
672 c->sfd = sfd;
673 conns[sfd] = c;
674 }
675
676 c->transport = transport;
677 c->protocol = bproto;
678 c->tag = conntag;
679
680 /* unix socket mode doesn't need this, so zeroed out. but why
681 * is this done for every command? presumably for UDP
682 * mode. */
683 if (!settings.socketpath) {
684 c->request_addr_size = sizeof(c->request_addr);
685 } else {
686 c->request_addr_size = 0;
687 }
688
689 if (transport == tcp_transport && init_state == conn_new_cmd) {
690 if (getpeername(sfd, (struct sockaddr *) &c->request_addr,
691 &c->request_addr_size)) {
692 perror("getpeername");
693 memset(&c->request_addr, 0, sizeof(c->request_addr));
694 }
695 }
696
697 if (init_state == conn_new_cmd) {
698 LOGGER_LOG(NULL, LOG_CONNEVENTS, LOGGER_CONNECTION_NEW, NULL,
699 &c->request_addr, c->request_addr_size, c->transport, 0, sfd);
700 }
701
702 if (settings.verbose > 1) {
703 if (init_state == conn_listening) {
704 fprintf(stderr, "<%d server listening (%s)\n", sfd,
705 prot_text(c->protocol));
706 } else if (IS_UDP(transport)) {
707 fprintf(stderr, "<%d server listening (udp)\n", sfd);
708 } else if (c->protocol == negotiating_prot) {
709 fprintf(stderr, "<%d new auto-negotiating client connection\n",
710 sfd);
711 } else if (c->protocol == ascii_prot) {
712 fprintf(stderr, "<%d new ascii client connection.\n", sfd);
713 } else if (c->protocol == binary_prot) {
714 fprintf(stderr, "<%d new binary client connection.\n", sfd);
715 #ifdef PROXY
716 } else if (c->protocol == proxy_prot) {
717 fprintf(stderr, "<%d new proxy client connection.\n", sfd);
718 #endif
719 } else {
720 fprintf(stderr, "<%d new unknown (%d) client connection\n",
721 sfd, c->protocol);
722 assert(false);
723 }
724 }
725
726 c->state = init_state;
727 c->rlbytes = 0;
728 c->cmd = -1;
729 c->rbytes = 0;
730 c->rcurr = c->rbuf;
731 c->ritem = 0;
732 c->rbuf_malloced = false;
733 c->item_malloced = false;
734 c->sasl_started = false;
735 c->set_stale = false;
736 c->mset_res = false;
737 c->close_after_write = false;
738 c->last_cmd_time = current_time; /* initialize for idle kicker */
739 // wipe all queues.
740 memset(c->io_queues, 0, sizeof(c->io_queues));
741 c->io_queues_submitted = 0;
742
743 c->item = 0;
744 c->ssl = NULL;
745 #ifdef TLS
746 c->ssl_wbuf = NULL;
747 #endif
748
749 c->noreply = false;
750
751 if (ssl) {
752 // musn't get here without ssl enabled.
753 assert(settings.ssl_enabled);
754 ssl_init_conn(c, ssl);
755 c->ssl_enabled = true;
756 } else {
757 c->read = tcp_read;
758 c->sendmsg = tcp_sendmsg;
759 c->write = tcp_write;
760 c->ssl_enabled = false;
761 }
762
763 if (IS_UDP(transport)) {
764 c->try_read_command = try_read_command_udp;
765 } else {
766 switch (c->protocol) {
767 case ascii_prot:
768 if (settings.auth_file == NULL) {
769 c->authenticated = true;
770 c->try_read_command = try_read_command_ascii;
771 } else {
772 c->authenticated = false;
773 c->try_read_command = try_read_command_asciiauth;
774 }
775 break;
776 case binary_prot:
777 // binprot handles its own authentication via SASL parsing.
778 c->authenticated = false;
779 c->try_read_command = try_read_command_binary;
780 break;
781 case negotiating_prot:
782 c->try_read_command = try_read_command_negotiate;
783 break;
784 #ifdef PROXY
785 case proxy_prot:
786 c->try_read_command = try_read_command_proxy;
787 break;
788 #endif
789 }
790 }
791
792 event_set(&c->event, sfd, event_flags, event_handler, (void *)c);
793 event_base_set(base, &c->event);
794 c->ev_flags = event_flags;
795
796 if (event_add(&c->event, 0) == -1) {
797 perror("event_add");
798 return NULL;
799 }
800
801 STATS_LOCK();
802 stats_state.curr_conns++;
803 stats.total_conns++;
804 STATS_UNLOCK();
805
806 MEMCACHED_CONN_ALLOCATE(c->sfd);
807
808 return c;
809 }
810
conn_release_items(conn * c)811 void conn_release_items(conn *c) {
812 assert(c != NULL);
813
814 if (c->item) {
815 if (c->item_malloced) {
816 free(c->item);
817 c->item_malloced = false;
818 } else {
819 item_remove(c->item);
820 }
821 c->item = 0;
822 }
823
824 // Cull any unsent responses.
825 if (c->resp_head) {
826 mc_resp *resp = c->resp_head;
827 // r_f() handles the chain maintenance.
828 while (resp) {
829 // temporary by default. hide behind a debug flag in the future:
830 // double free detection. Transmit loops can drop out early, but
831 // here we could infinite loop.
832 if (resp->free) {
833 fprintf(stderr, "ERROR: double free detected during conn_release_items(): [%d] [%s]\n",
834 c->sfd, c->protocol == binary_prot ? "binary" : "ascii");
835 // Since this is a critical failure, just leak the memory.
836 // If these errors are seen, an abort() can be used instead.
837 c->resp_head = NULL;
838 c->resp = NULL;
839 break;
840 }
841 resp = resp_finish(c, resp);
842 }
843 conn_io_queue_reset(c);
844 }
845 }
846
conn_cleanup(conn * c)847 static void conn_cleanup(conn *c) {
848 assert(c != NULL);
849
850 conn_release_items(c);
851 #ifdef PROXY
852 if (c->proxy_rctx) {
853 proxy_cleanup_conn(c);
854 }
855 #endif
856 if (c->sasl_conn) {
857 assert(settings.sasl);
858 sasl_dispose(&c->sasl_conn);
859 c->sasl_conn = NULL;
860 }
861
862 if (IS_UDP(c->transport)) {
863 conn_set_state(c, conn_read);
864 }
865 }
866
867 /*
868 * Frees a connection.
869 */
conn_free(conn * c)870 void conn_free(conn *c) {
871 if (c) {
872 assert(c != NULL);
873 assert(c->sfd >= 0 && c->sfd < max_fds);
874
875 MEMCACHED_CONN_DESTROY(c);
876 conns[c->sfd] = NULL;
877 if (c->rbuf)
878 free(c->rbuf);
879 #ifdef TLS
880 if (c->ssl_wbuf)
881 c->ssl_wbuf = NULL;
882 #endif
883
884 free(c);
885 }
886 }
887
conn_close(conn * c)888 static void conn_close(conn *c) {
889 assert(c != NULL);
890
891 if (c->thread) {
892 LOGGER_LOG(c->thread->l, LOG_CONNEVENTS, LOGGER_CONNECTION_CLOSE, NULL,
893 &c->request_addr, c->request_addr_size, c->transport,
894 c->close_reason, c->sfd);
895 }
896
897 /* delete the event, the socket and the conn */
898 event_del(&c->event);
899
900 if (settings.verbose > 1)
901 fprintf(stderr, "<%d connection closed.\n", c->sfd);
902
903 conn_cleanup(c);
904
905 // force release of read buffer.
906 if (c->thread) {
907 c->rbytes = 0;
908 rbuf_release(c);
909 }
910
911 MEMCACHED_CONN_RELEASE(c->sfd);
912 conn_set_state(c, conn_closed);
913 if (c->ssl_enabled) {
914 ssl_conn_close(c->ssl);
915 }
916 close(c->sfd);
917 c->close_reason = 0;
918 pthread_mutex_lock(&conn_lock);
919 allow_new_conns = true;
920 pthread_mutex_unlock(&conn_lock);
921
922 STATS_LOCK();
923 stats_state.curr_conns--;
924 STATS_UNLOCK();
925
926 return;
927 }
928
929 // Since some connections might be off on side threads and some are managed as
930 // listeners we need to walk through them all from a central point.
931 // Must be called with all worker threads hung or in the process of closing.
conn_close_all(void)932 void conn_close_all(void) {
933 int i;
934 for (i = 0; i < max_fds; i++) {
935 if (conns[i] && conns[i]->state != conn_closed) {
936 conn_close(conns[i]);
937 }
938 }
939 }
940
941 /**
942 * Convert a state name to a human readable form.
943 */
state_text(enum conn_states state)944 static const char *state_text(enum conn_states state) {
945 const char* const statenames[] = { "conn_listening",
946 "conn_new_cmd",
947 "conn_waiting",
948 "conn_read",
949 "conn_parse_cmd",
950 "conn_write",
951 "conn_nread",
952 "conn_swallow",
953 "conn_closing",
954 "conn_mwrite",
955 "conn_closed",
956 "conn_watch",
957 "conn_io_queue",
958 "conn_io_resume",
959 "conn_io_pending" };
960 return statenames[state];
961 }
962
963 /*
964 * Sets a connection's current state in the state machine. Any special
965 * processing that needs to happen on certain state transitions can
966 * happen here.
967 */
conn_set_state(conn * c,enum conn_states state)968 void conn_set_state(conn *c, enum conn_states state) {
969 assert(c != NULL);
970 assert(state >= conn_listening && state < conn_max_state);
971
972 if (state != c->state) {
973 if (settings.verbose > 2) {
974 fprintf(stderr, "%d: going from %s to %s\n",
975 c->sfd, state_text(c->state),
976 state_text(state));
977 }
978
979 if (state == conn_write || state == conn_mwrite) {
980 MEMCACHED_PROCESS_COMMAND_END(c->sfd, c->resp->wbuf, c->resp->wbytes);
981 }
982 c->state = state;
983 }
984 }
985
986 /*
987 * response object helper functions
988 */
resp_reset(mc_resp * resp)989 void resp_reset(mc_resp *resp) {
990 if (resp->item) {
991 item_remove(resp->item);
992 resp->item = NULL;
993 }
994 if (resp->write_and_free) {
995 #ifdef PROXY
996 if (resp->proxy_res) {
997 LIBEVENT_THREAD *t = resp->bundle->thread;
998 pthread_mutex_lock(&t->proxy_limit_lock);
999 t->proxy_buffer_memory_used -= resp->wbytes;
1000 pthread_mutex_unlock(&t->proxy_limit_lock);
1001 }
1002 #endif
1003 free(resp->write_and_free);
1004 resp->write_and_free = NULL;
1005 }
1006 resp->wbytes = 0;
1007 resp->tosend = 0;
1008 resp->iovcnt = 0;
1009 resp->chunked_data_iov = 0;
1010 resp->chunked_total = 0;
1011 resp->skip = false;
1012 }
1013
resp_add_iov(mc_resp * resp,const void * buf,int len)1014 void resp_add_iov(mc_resp *resp, const void *buf, int len) {
1015 assert(resp->iovcnt < MC_RESP_IOVCOUNT);
1016 int x = resp->iovcnt;
1017 resp->iov[x].iov_base = (void *)buf;
1018 resp->iov[x].iov_len = len;
1019 resp->iovcnt++;
1020 resp->tosend += len;
1021 }
1022
1023 // Notes that an IOV should be handled as a chunked item header.
1024 // TODO: I'm hoping this isn't a permanent abstraction while I learn what the
1025 // API should be.
resp_add_chunked_iov(mc_resp * resp,const void * buf,int len)1026 void resp_add_chunked_iov(mc_resp *resp, const void *buf, int len) {
1027 resp->chunked_data_iov = resp->iovcnt;
1028 resp->chunked_total = len;
1029 resp_add_iov(resp, buf, len);
1030 }
1031
1032 // resp_allocate and resp_free are a wrapper around read buffers which makes
1033 // read buffers the only network memory to track.
1034 // Normally this would be too excessive. In this case it allows end users to
1035 // track a single memory limit for ephemeral connection buffers.
1036 // Fancy bit twiddling tricks are avoided to help keep this straightforward.
resp_allocate(conn * c)1037 static mc_resp* resp_allocate(conn *c) {
1038 LIBEVENT_THREAD *th = c->thread;
1039 mc_resp *resp = NULL;
1040 mc_resp_bundle *b = th->open_bundle;
1041
1042 if (b != NULL) {
1043 for (int i = 0; i < MAX_RESP_PER_BUNDLE; i++) {
1044 // loop around starting from the most likely to be free
1045 int x = (i + b->next_check) % MAX_RESP_PER_BUNDLE;
1046 if (b->r[x].free) {
1047 resp = &b->r[x];
1048 b->next_check = x+1;
1049 break;
1050 }
1051 }
1052
1053 if (resp != NULL) {
1054 b->refcount++;
1055 memset(resp, 0, sizeof(*resp));
1056 resp->free = false; // redundant, for clarity.
1057 resp->bundle = b;
1058 if (b->refcount == MAX_RESP_PER_BUNDLE) {
1059 assert(b->prev == NULL);
1060 // We only allocate off the head. Assign new head.
1061 th->open_bundle = b->next;
1062 // Remove ourselves from the list.
1063 if (b->next) {
1064 b->next->prev = 0;
1065 b->next = 0;
1066 }
1067 }
1068 }
1069 }
1070
1071 if (resp == NULL) {
1072 assert(th->open_bundle == NULL);
1073 b = do_cache_alloc(th->rbuf_cache);
1074 if (b) {
1075 THR_STATS_LOCK(th);
1076 th->stats.response_obj_bytes += READ_BUFFER_SIZE;
1077 THR_STATS_UNLOCK(th);
1078 b->next_check = 1;
1079 b->refcount = 1;
1080 for (int i = 0; i < MAX_RESP_PER_BUNDLE; i++) {
1081 b->r[i].free = true;
1082 }
1083 b->next = 0;
1084 b->prev = 0;
1085 b->thread = th;
1086 th->open_bundle = b;
1087 resp = &b->r[0];
1088 memset(resp, 0, sizeof(*resp));
1089 resp->free = false; // redundant. for clarity.
1090 resp->bundle = b;
1091 } else {
1092 return NULL;
1093 }
1094 }
1095
1096 return resp;
1097 }
1098
resp_free(LIBEVENT_THREAD * th,mc_resp * resp)1099 void resp_free(LIBEVENT_THREAD *th, mc_resp *resp) {
1100 mc_resp_bundle *b = resp->bundle;
1101
1102 resp->free = true;
1103 b->refcount--;
1104 if (b->refcount == 0) {
1105 if (b == th->open_bundle && b->next == 0) {
1106 // This is the final bundle. Just hold and reuse to skip init loop
1107 assert(b->prev == 0);
1108 b->next_check = 0;
1109 } else {
1110 // Assert that we're either in the list or at the head.
1111 assert((b->next || b->prev) || b == th->open_bundle);
1112
1113 // unlink from list.
1114 mc_resp_bundle **head = &th->open_bundle;
1115 if (*head == b) *head = b->next;
1116 // Not tracking the tail.
1117 assert(b->next != b && b->prev != b);
1118
1119 if (b->next) b->next->prev = b->prev;
1120 if (b->prev) b->prev->next = b->next;
1121
1122 // Now completely done with this buffer.
1123 do_cache_free(th->rbuf_cache, b);
1124 THR_STATS_LOCK(th);
1125 th->stats.response_obj_bytes -= READ_BUFFER_SIZE;
1126 THR_STATS_UNLOCK(th);
1127 }
1128 } else {
1129 mc_resp_bundle **head = &th->open_bundle;
1130 // NOTE: since we're not tracking tail, latest free ends up in head.
1131 if (b == th->open_bundle || (b->prev || b->next)) {
1132 // If we're already linked, leave it in place to save CPU.
1133 } else {
1134 // Non-zero refcount, need to link into the freelist.
1135 b->prev = 0;
1136 b->next = *head;
1137 if (b->next) b->next->prev = b;
1138 *head = b;
1139 }
1140
1141 }
1142 THR_STATS_LOCK(th);
1143 th->stats.response_obj_count--;
1144 THR_STATS_UNLOCK(th);
1145 }
1146
resp_start(conn * c)1147 bool resp_start(conn *c) {
1148 mc_resp *resp = resp_allocate(c);
1149 if (!resp) {
1150 THR_STATS_LOCK(c->thread);
1151 c->thread->stats.response_obj_oom++;
1152 THR_STATS_UNLOCK(c->thread);
1153 return false;
1154 }
1155
1156 // handling the stats counters here to simplify testing
1157 THR_STATS_LOCK(c->thread);
1158 c->thread->stats.response_obj_count++;
1159 THR_STATS_UNLOCK(c->thread);
1160
1161 if (!c->resp_head) {
1162 c->resp_head = resp;
1163 }
1164 if (!c->resp) {
1165 c->resp = resp;
1166 } else {
1167 c->resp->next = resp;
1168 c->resp = resp;
1169 }
1170 if (IS_UDP(c->transport)) {
1171 // need to hold on to some data for async responses.
1172 c->resp->request_id = c->request_id;
1173 c->resp->request_addr = c->request_addr;
1174 c->resp->request_addr_size = c->request_addr_size;
1175 }
1176 return true;
1177 }
1178
resp_start_unlinked(conn * c)1179 mc_resp *resp_start_unlinked(conn *c) {
1180 mc_resp *resp = resp_allocate(c);
1181 if (!resp) {
1182 THR_STATS_LOCK(c->thread);
1183 c->thread->stats.response_obj_oom++;
1184 THR_STATS_UNLOCK(c->thread);
1185 return false;
1186 }
1187
1188 // handling the stats counters here to simplify testing
1189 THR_STATS_LOCK(c->thread);
1190 c->thread->stats.response_obj_count++;
1191 THR_STATS_UNLOCK(c->thread);
1192
1193 if (IS_UDP(c->transport)) {
1194 // need to hold on to some data for async responses.
1195 c->resp->request_id = c->request_id;
1196 c->resp->request_addr = c->request_addr;
1197 c->resp->request_addr_size = c->request_addr_size;
1198 }
1199
1200 return resp;
1201 }
1202
1203 // returns next response in chain.
resp_finish(conn * c,mc_resp * resp)1204 mc_resp* resp_finish(conn *c, mc_resp *resp) {
1205 mc_resp *next = resp->next;
1206 if (resp->item) {
1207 // TODO: cache hash value in resp obj?
1208 item_remove(resp->item);
1209 resp->item = NULL;
1210 }
1211 if (resp->write_and_free) {
1212 #ifdef PROXY
1213 if (resp->proxy_res) {
1214 LIBEVENT_THREAD *t = resp->bundle->thread;
1215 pthread_mutex_lock(&t->proxy_limit_lock);
1216 t->proxy_buffer_memory_used -= resp->wbytes;
1217 pthread_mutex_unlock(&t->proxy_limit_lock);
1218 }
1219 #endif
1220 free(resp->write_and_free);
1221 }
1222 if (resp->io_pending) {
1223 io_pending_t *io = resp->io_pending;
1224 // If we had a pending IO, tell it to internally clean up then return
1225 // the main object back to our thread cache.
1226 io->finalize_cb(io);
1227 do_cache_free(c->thread->io_cache, io);
1228 resp->io_pending = NULL;
1229 }
1230 if (c->resp_head == resp) {
1231 c->resp_head = next;
1232 }
1233 if (c->resp == resp) {
1234 c->resp = NULL;
1235 }
1236 resp_free(c->thread, resp);
1237 return next;
1238 }
1239
1240 // tells if connection has a depth of response objects to process.
resp_has_stack(conn * c)1241 bool resp_has_stack(conn *c) {
1242 return c->resp_head->next != NULL ? true : false;
1243 }
1244
out_string(conn * c,const char * str)1245 void out_string(conn *c, const char *str) {
1246 size_t len;
1247 assert(c != NULL);
1248 mc_resp *resp = c->resp;
1249
1250 // if response was original filled with something, but we're now writing
1251 // out an error or similar, have to reset the object first.
1252 // TODO: since this is often redundant with allocation, how many callers
1253 // are actually requiring it be reset? Can we fast test by just looking at
1254 // tosend and reset if nonzero?
1255 resp_reset(resp);
1256
1257 if (c->noreply) {
1258 // TODO: just invalidate the response since nothing's been attempted
1259 // to send yet?
1260 resp->skip = true;
1261 if (settings.verbose > 1)
1262 fprintf(stderr, ">%d NOREPLY %s\n", c->sfd, str);
1263 conn_set_state(c, conn_new_cmd);
1264 return;
1265 }
1266
1267 if (settings.verbose > 1)
1268 fprintf(stderr, ">%d %s\n", c->sfd, str);
1269
1270 // Fill response object with static string.
1271
1272 len = strlen(str);
1273 if ((len + 2) > WRITE_BUFFER_SIZE) {
1274 /* ought to be always enough. just fail for simplicity */
1275 str = "SERVER_ERROR output line too long";
1276 len = strlen(str);
1277 }
1278
1279 memcpy(resp->wbuf, str, len);
1280 memcpy(resp->wbuf + len, "\r\n", 2);
1281 resp_add_iov(resp, resp->wbuf, len + 2);
1282
1283 conn_set_state(c, conn_new_cmd);
1284 return;
1285 }
1286
1287 // For metaget-style ASCII commands. Ignores noreply, ensuring clients see
1288 // protocol level errors.
out_errstring(conn * c,const char * str)1289 void out_errstring(conn *c, const char *str) {
1290 c->noreply = false;
1291 out_string(c, str);
1292 }
1293
1294 /*
1295 * Outputs a protocol-specific "out of memory" error. For ASCII clients,
1296 * this is equivalent to out_string().
1297 */
out_of_memory(conn * c,char * ascii_error)1298 void out_of_memory(conn *c, char *ascii_error) {
1299 const static char error_prefix[] = "SERVER_ERROR ";
1300 const static int error_prefix_len = sizeof(error_prefix) - 1;
1301
1302 if (c->protocol == binary_prot) {
1303 /* Strip off the generic error prefix; it's irrelevant in binary */
1304 if (!strncmp(ascii_error, error_prefix, error_prefix_len)) {
1305 ascii_error += error_prefix_len;
1306 }
1307 write_bin_error(c, PROTOCOL_BINARY_RESPONSE_ENOMEM, ascii_error, 0);
1308 } else {
1309 out_string(c, ascii_error);
1310 }
1311 }
1312
append_bin_stats(const char * key,const uint16_t klen,const char * val,const uint32_t vlen,conn * c)1313 static void append_bin_stats(const char *key, const uint16_t klen,
1314 const char *val, const uint32_t vlen,
1315 conn *c) {
1316 char *buf = c->stats.buffer + c->stats.offset;
1317 uint32_t bodylen = klen + vlen;
1318 protocol_binary_response_header header = {
1319 .response.magic = (uint8_t)PROTOCOL_BINARY_RES,
1320 .response.opcode = PROTOCOL_BINARY_CMD_STAT,
1321 .response.keylen = (uint16_t)htons(klen),
1322 .response.datatype = (uint8_t)PROTOCOL_BINARY_RAW_BYTES,
1323 .response.bodylen = htonl(bodylen),
1324 .response.opaque = c->opaque
1325 };
1326
1327 memcpy(buf, header.bytes, sizeof(header.response));
1328 buf += sizeof(header.response);
1329
1330 if (klen > 0) {
1331 memcpy(buf, key, klen);
1332 buf += klen;
1333
1334 if (vlen > 0) {
1335 memcpy(buf, val, vlen);
1336 }
1337 }
1338
1339 c->stats.offset += sizeof(header.response) + bodylen;
1340 }
1341
append_ascii_stats(const char * key,const uint16_t klen,const char * val,const uint32_t vlen,conn * c)1342 static void append_ascii_stats(const char *key, const uint16_t klen,
1343 const char *val, const uint32_t vlen,
1344 conn *c) {
1345 char *pos = c->stats.buffer + c->stats.offset;
1346 uint32_t nbytes = 0;
1347 int remaining = c->stats.size - c->stats.offset;
1348 int room = remaining - 1;
1349
1350 if (klen == 0 && vlen == 0) {
1351 nbytes = snprintf(pos, room, "END\r\n");
1352 } else if (vlen == 0) {
1353 nbytes = snprintf(pos, room, "STAT %s\r\n", key);
1354 } else {
1355 nbytes = snprintf(pos, room, "STAT %s %s\r\n", key, val);
1356 }
1357
1358 c->stats.offset += nbytes;
1359 }
1360
grow_stats_buf(conn * c,size_t needed)1361 static bool grow_stats_buf(conn *c, size_t needed) {
1362 size_t nsize = c->stats.size;
1363 size_t available = nsize - c->stats.offset;
1364 bool rv = true;
1365
1366 /* Special case: No buffer -- need to allocate fresh */
1367 if (c->stats.buffer == NULL) {
1368 nsize = 1024;
1369 available = c->stats.size = c->stats.offset = 0;
1370 }
1371
1372 while (needed > available) {
1373 assert(nsize > 0);
1374 nsize = nsize << 1;
1375 available = nsize - c->stats.offset;
1376 }
1377
1378 if (nsize != c->stats.size) {
1379 char *ptr = realloc(c->stats.buffer, nsize);
1380 if (ptr) {
1381 c->stats.buffer = ptr;
1382 c->stats.size = nsize;
1383 } else {
1384 STATS_LOCK();
1385 stats.malloc_fails++;
1386 STATS_UNLOCK();
1387 rv = false;
1388 }
1389 }
1390
1391 return rv;
1392 }
1393
append_stats(const char * key,const uint16_t klen,const char * val,const uint32_t vlen,const void * cookie)1394 void append_stats(const char *key, const uint16_t klen,
1395 const char *val, const uint32_t vlen,
1396 const void *cookie)
1397 {
1398 /* value without a key is invalid */
1399 if (klen == 0 && vlen > 0) {
1400 return;
1401 }
1402
1403 conn *c = (conn*)cookie;
1404
1405 if (c->protocol == binary_prot) {
1406 size_t needed = vlen + klen + sizeof(protocol_binary_response_header);
1407 if (!grow_stats_buf(c, needed)) {
1408 return;
1409 }
1410 append_bin_stats(key, klen, val, vlen, c);
1411 } else {
1412 size_t needed = vlen + klen + 10; // 10 == "STAT = \r\n"
1413 if (!grow_stats_buf(c, needed)) {
1414 return;
1415 }
1416 append_ascii_stats(key, klen, val, vlen, c);
1417 }
1418
1419 assert(c->stats.offset <= c->stats.size);
1420 }
1421
reset_cmd_handler(conn * c)1422 static void reset_cmd_handler(conn *c) {
1423 c->cmd = -1;
1424 c->substate = bin_no_state;
1425 if (c->item != NULL) {
1426 // TODO: Any other way to get here?
1427 // SASL auth was mistakenly using it. Nothing else should?
1428 if (c->item_malloced) {
1429 free(c->item);
1430 c->item_malloced = false;
1431 } else {
1432 item_remove(c->item);
1433 }
1434 c->item = NULL;
1435 }
1436 if (c->rbytes > 0) {
1437 conn_set_state(c, conn_parse_cmd);
1438 } else if (c->resp_head) {
1439 conn_set_state(c, conn_mwrite);
1440 } else {
1441 conn_set_state(c, conn_waiting);
1442 }
1443 }
1444
complete_nread(conn * c)1445 static void complete_nread(conn *c) {
1446 assert(c != NULL);
1447 #ifdef PROXY
1448 assert(c->protocol == ascii_prot
1449 || c->protocol == binary_prot
1450 || c->protocol == proxy_prot);
1451 #else
1452 assert(c->protocol == ascii_prot
1453 || c->protocol == binary_prot);
1454 #endif
1455 if (c->protocol == ascii_prot) {
1456 complete_nread_ascii(c);
1457 } else if (c->protocol == binary_prot) {
1458 complete_nread_binary(c);
1459 #ifdef PROXY
1460 } else if (c->protocol == proxy_prot) {
1461 complete_nread_proxy(c);
1462 #endif
1463 }
1464 }
1465
1466 /* Destination must always be chunked */
1467 /* This should be part of item.c */
_store_item_copy_chunks(item * d_it,item * s_it,const int len)1468 static int _store_item_copy_chunks(item *d_it, item *s_it, const int len) {
1469 item_chunk *dch = (item_chunk *) ITEM_schunk(d_it);
1470 /* Advance dch until we find free space */
1471 while (dch->size == dch->used) {
1472 if (dch->next) {
1473 dch = dch->next;
1474 } else {
1475 break;
1476 }
1477 }
1478
1479 if (s_it->it_flags & ITEM_CHUNKED) {
1480 int remain = len;
1481 item_chunk *sch = (item_chunk *) ITEM_schunk(s_it);
1482 int copied = 0;
1483 /* Fills dch's to capacity, not straight copy sch in case data is
1484 * being added or removed (ie append/prepend)
1485 */
1486 while (sch && dch && remain) {
1487 assert(dch->used <= dch->size);
1488 int todo = (dch->size - dch->used < sch->used - copied)
1489 ? dch->size - dch->used : sch->used - copied;
1490 if (remain < todo)
1491 todo = remain;
1492 memcpy(dch->data + dch->used, sch->data + copied, todo);
1493 dch->used += todo;
1494 copied += todo;
1495 remain -= todo;
1496 assert(dch->used <= dch->size);
1497 if (dch->size == dch->used) {
1498 item_chunk *tch = do_item_alloc_chunk(dch, remain);
1499 if (tch) {
1500 dch = tch;
1501 } else {
1502 return -1;
1503 }
1504 }
1505 assert(copied <= sch->used);
1506 if (copied == sch->used) {
1507 copied = 0;
1508 sch = sch->next;
1509 }
1510 }
1511 /* assert that the destination had enough space for the source */
1512 assert(remain == 0);
1513 } else {
1514 int done = 0;
1515 /* Fill dch's via a non-chunked item. */
1516 while (len > done && dch) {
1517 int todo = (dch->size - dch->used < len - done)
1518 ? dch->size - dch->used : len - done;
1519 //assert(dch->size - dch->used != 0);
1520 memcpy(dch->data + dch->used, ITEM_data(s_it) + done, todo);
1521 done += todo;
1522 dch->used += todo;
1523 assert(dch->used <= dch->size);
1524 if (dch->size == dch->used) {
1525 item_chunk *tch = do_item_alloc_chunk(dch, len - done);
1526 if (tch) {
1527 dch = tch;
1528 } else {
1529 return -1;
1530 }
1531 }
1532 }
1533 assert(len == done);
1534 }
1535 return 0;
1536 }
1537
_store_item_copy_data(int comm,item * old_it,item * new_it,item * add_it)1538 static int _store_item_copy_data(int comm, item *old_it, item *new_it, item *add_it) {
1539 if (comm == NREAD_APPEND || comm == NREAD_APPENDVIV) {
1540 if (new_it->it_flags & ITEM_CHUNKED) {
1541 if (_store_item_copy_chunks(new_it, old_it, old_it->nbytes - 2) == -1 ||
1542 _store_item_copy_chunks(new_it, add_it, add_it->nbytes) == -1) {
1543 return -1;
1544 }
1545 } else {
1546 memcpy(ITEM_data(new_it), ITEM_data(old_it), old_it->nbytes);
1547 memcpy(ITEM_data(new_it) + old_it->nbytes - 2 /* CRLF */, ITEM_data(add_it), add_it->nbytes);
1548 }
1549 } else {
1550 /* NREAD_PREPEND */
1551 if (new_it->it_flags & ITEM_CHUNKED) {
1552 if (_store_item_copy_chunks(new_it, add_it, add_it->nbytes - 2) == -1 ||
1553 _store_item_copy_chunks(new_it, old_it, old_it->nbytes) == -1) {
1554 return -1;
1555 }
1556 } else {
1557 memcpy(ITEM_data(new_it), ITEM_data(add_it), add_it->nbytes);
1558 memcpy(ITEM_data(new_it) + add_it->nbytes - 2 /* CRLF */, ITEM_data(old_it), old_it->nbytes);
1559 }
1560 }
1561 return 0;
1562 }
1563
1564 /*
1565 * Stores an item in the cache according to the semantics of one of the set
1566 * commands. Protected by the item lock.
1567 *
1568 * Returns the state of storage.
1569 */
do_store_item(item * it,int comm,LIBEVENT_THREAD * t,const uint32_t hv,int * nbytes,uint64_t * cas,uint64_t cas_in,bool cas_stale)1570 enum store_item_type do_store_item(item *it, int comm, LIBEVENT_THREAD *t, const uint32_t hv, int *nbytes, uint64_t *cas, uint64_t cas_in, bool cas_stale) {
1571 char *key = ITEM_key(it);
1572 item *old_it = do_item_get(key, it->nkey, hv, t, DONT_UPDATE);
1573 enum store_item_type stored = NOT_STORED;
1574
1575 enum cas_result { CAS_NONE, CAS_MATCH, CAS_BADVAL, CAS_STALE, CAS_MISS };
1576
1577 item *new_it = NULL;
1578 client_flags_t flags;
1579
1580 /* Do the CAS test up front so we can apply to all store modes */
1581 enum cas_result cas_res = CAS_NONE;
1582
1583 bool do_store = false;
1584 if (old_it != NULL) {
1585 // Most of the CAS work requires something to compare to.
1586 uint64_t it_cas = ITEM_get_cas(it);
1587 uint64_t old_cas = ITEM_get_cas(old_it);
1588 if (it_cas == 0) {
1589 cas_res = CAS_NONE;
1590 } else if (it_cas == old_cas) {
1591 cas_res = CAS_MATCH;
1592 } else if (cas_stale && it_cas < old_cas) {
1593 cas_res = CAS_STALE;
1594 } else {
1595 cas_res = CAS_BADVAL;
1596 }
1597
1598 switch (comm) {
1599 case NREAD_ADD:
1600 /* add only adds a nonexistent item, but promote to head of LRU */
1601 do_item_update(old_it);
1602 break;
1603 case NREAD_CAS:
1604 if (cas_res == CAS_MATCH) {
1605 // cas validates
1606 // it and old_it may belong to different classes.
1607 // I'm updating the stats for the one that's getting pushed out
1608 pthread_mutex_lock(&t->stats.mutex);
1609 t->stats.slab_stats[ITEM_clsid(old_it)].cas_hits++;
1610 pthread_mutex_unlock(&t->stats.mutex);
1611 do_store = true;
1612 } else if (cas_res == CAS_STALE) {
1613 // if we're allowed to set a stale value, CAS must be lower than
1614 // the current item's CAS.
1615 // This replaces the value, but should preserve TTL, and stale
1616 // item marker bit + token sent if exists.
1617 it->exptime = old_it->exptime;
1618 it->it_flags |= ITEM_STALE;
1619 if (old_it->it_flags & ITEM_TOKEN_SENT) {
1620 it->it_flags |= ITEM_TOKEN_SENT;
1621 }
1622
1623 pthread_mutex_lock(&t->stats.mutex);
1624 t->stats.slab_stats[ITEM_clsid(old_it)].cas_hits++;
1625 pthread_mutex_unlock(&t->stats.mutex);
1626 do_store = true;
1627 } else {
1628 // NONE or BADVAL are the same for CAS cmd
1629 pthread_mutex_lock(&t->stats.mutex);
1630 t->stats.slab_stats[ITEM_clsid(old_it)].cas_badval++;
1631 pthread_mutex_unlock(&t->stats.mutex);
1632
1633 if (settings.verbose > 1) {
1634 fprintf(stderr, "CAS: failure: expected %llu, got %llu\n",
1635 (unsigned long long)ITEM_get_cas(old_it),
1636 (unsigned long long)ITEM_get_cas(it));
1637 }
1638 stored = EXISTS;
1639 }
1640 break;
1641 case NREAD_APPEND:
1642 case NREAD_PREPEND:
1643 case NREAD_APPENDVIV:
1644 case NREAD_PREPENDVIV:
1645 if (cas_res != CAS_NONE && cas_res != CAS_MATCH) {
1646 stored = EXISTS;
1647 break;
1648 }
1649 #ifdef EXTSTORE
1650 if ((old_it->it_flags & ITEM_HDR) != 0) {
1651 /* block append/prepend from working with extstore-d items.
1652 * leave response code to NOT_STORED default */
1653 break;
1654 }
1655 #endif
1656 /* we have it and old_it here - alloc memory to hold both */
1657 FLAGS_CONV(old_it, flags);
1658 new_it = do_item_alloc(key, it->nkey, flags, old_it->exptime, it->nbytes + old_it->nbytes - 2 /* CRLF */);
1659
1660 // OOM trying to copy.
1661 if (new_it == NULL)
1662 break;
1663 /* copy data from it and old_it to new_it */
1664 if (_store_item_copy_data(comm, old_it, new_it, it) == -1) {
1665 // failed data copy
1666 break;
1667 } else {
1668 // refcount of new_it is 1 here. will end up 2 after link.
1669 // it's original ref is managed outside of this function
1670 it = new_it;
1671 do_store = true;
1672 // Upstream final object size for meta
1673 if (nbytes != NULL) {
1674 *nbytes = it->nbytes;
1675 }
1676 }
1677 break;
1678 case NREAD_REPLACE:
1679 case NREAD_SET:
1680 do_store = true;
1681 break;
1682 }
1683
1684 if (do_store) {
1685 STORAGE_delete(t->storage, old_it);
1686 item_replace(old_it, it, hv, cas_in);
1687 stored = STORED;
1688 }
1689
1690 do_item_remove(old_it); /* release our reference */
1691 if (new_it != NULL) {
1692 // append/prepend end up with an extra reference for new_it.
1693 do_item_remove(new_it);
1694 }
1695 } else {
1696 /* No pre-existing item to replace or compare to. */
1697 if (ITEM_get_cas(it) != 0) {
1698 /* Asked for a CAS match but nothing to compare it to. */
1699 cas_res = CAS_MISS;
1700 }
1701
1702 switch (comm) {
1703 case NREAD_ADD:
1704 case NREAD_SET:
1705 case NREAD_APPENDVIV:
1706 case NREAD_PREPENDVIV:
1707 do_store = true;
1708 break;
1709 case NREAD_CAS:
1710 // LRU expired
1711 stored = NOT_FOUND;
1712 pthread_mutex_lock(&t->stats.mutex);
1713 t->stats.cas_misses++;
1714 pthread_mutex_unlock(&t->stats.mutex);
1715 break;
1716 case NREAD_REPLACE:
1717 case NREAD_APPEND:
1718 case NREAD_PREPEND:
1719 /* Requires an existing item. */
1720 break;
1721 }
1722
1723 if (do_store) {
1724 do_item_link(it, hv, cas_in);
1725 stored = STORED;
1726 }
1727 }
1728
1729 if (stored == STORED && cas != NULL) {
1730 *cas = ITEM_get_cas(it);
1731 }
1732 LOGGER_LOG(t->l, LOG_MUTATIONS, LOGGER_ITEM_STORE, NULL,
1733 stored, comm, ITEM_key(it), it->nkey, it->nbytes, it->exptime,
1734 ITEM_clsid(it), t->cur_sfd);
1735
1736 return stored;
1737 }
1738
1739 /* set up a connection to write a buffer then free it, used for stats */
write_and_free(conn * c,char * buf,int bytes)1740 void write_and_free(conn *c, char *buf, int bytes) {
1741 if (buf) {
1742 mc_resp *resp = c->resp;
1743 resp->write_and_free = buf;
1744 resp_add_iov(resp, buf, bytes);
1745 conn_set_state(c, conn_new_cmd);
1746 } else {
1747 out_of_memory(c, "SERVER_ERROR out of memory writing stats");
1748 }
1749 }
1750
append_stat(const char * name,ADD_STAT add_stats,conn * c,const char * fmt,...)1751 void append_stat(const char *name, ADD_STAT add_stats, conn *c,
1752 const char *fmt, ...) {
1753 char val_str[STAT_VAL_LEN];
1754 int vlen;
1755 va_list ap;
1756
1757 assert(name);
1758 assert(add_stats);
1759 assert(c);
1760 assert(fmt);
1761
1762 va_start(ap, fmt);
1763 vlen = vsnprintf(val_str, sizeof(val_str) - 1, fmt, ap);
1764 va_end(ap);
1765
1766 add_stats(name, strlen(name), val_str, vlen, c);
1767 }
1768
1769 /* return server specific stats only */
server_stats(ADD_STAT add_stats,void * c)1770 void server_stats(ADD_STAT add_stats, void *c) {
1771 pid_t pid = getpid();
1772 rel_time_t now = current_time;
1773
1774 struct thread_stats thread_stats;
1775 threadlocal_stats_aggregate(&thread_stats);
1776 struct slab_stats slab_stats;
1777 slab_stats_aggregate(&thread_stats, &slab_stats);
1778 #ifndef WIN32
1779 struct rusage usage;
1780 getrusage(RUSAGE_SELF, &usage);
1781 #endif /* !WIN32 */
1782
1783 STATS_LOCK();
1784
1785 APPEND_STAT("pid", "%lu", (long)pid);
1786 APPEND_STAT("uptime", "%u", now - ITEM_UPDATE_INTERVAL);
1787 APPEND_STAT("time", "%ld", now + (long)process_started);
1788 APPEND_STAT("version", "%s", VERSION);
1789 APPEND_STAT("libevent", "%s", event_get_version());
1790 APPEND_STAT("pointer_size", "%d", (int)(8 * sizeof(void *)));
1791
1792 #ifndef WIN32
1793 append_stat("rusage_user", add_stats, c, "%ld.%06ld",
1794 (long)usage.ru_utime.tv_sec,
1795 (long)usage.ru_utime.tv_usec);
1796 append_stat("rusage_system", add_stats, c, "%ld.%06ld",
1797 (long)usage.ru_stime.tv_sec,
1798 (long)usage.ru_stime.tv_usec);
1799 #endif /* !WIN32 */
1800
1801 APPEND_STAT("max_connections", "%d", settings.maxconns);
1802 APPEND_STAT("curr_connections", "%llu", (unsigned long long)stats_state.curr_conns - 1);
1803 APPEND_STAT("total_connections", "%llu", (unsigned long long)stats.total_conns);
1804 if (settings.maxconns_fast) {
1805 APPEND_STAT("rejected_connections", "%llu", (unsigned long long)stats.rejected_conns);
1806 }
1807 APPEND_STAT("connection_structures", "%u", stats_state.conn_structs);
1808 APPEND_STAT("response_obj_oom", "%llu", (unsigned long long)thread_stats.response_obj_oom);
1809 APPEND_STAT("response_obj_count", "%llu", (unsigned long long)thread_stats.response_obj_count);
1810 APPEND_STAT("response_obj_bytes", "%llu", (unsigned long long)thread_stats.response_obj_bytes);
1811 APPEND_STAT("read_buf_count", "%llu", (unsigned long long)thread_stats.read_buf_count);
1812 APPEND_STAT("read_buf_bytes", "%llu", (unsigned long long)thread_stats.read_buf_bytes);
1813 APPEND_STAT("read_buf_bytes_free", "%llu", (unsigned long long)thread_stats.read_buf_bytes_free);
1814 APPEND_STAT("read_buf_oom", "%llu", (unsigned long long)thread_stats.read_buf_oom);
1815 APPEND_STAT("reserved_fds", "%u", stats_state.reserved_fds);
1816 #ifdef PROXY
1817 if (settings.proxy_enabled) {
1818 APPEND_STAT("proxy_conn_requests", "%llu", (unsigned long long)thread_stats.proxy_conn_requests);
1819 APPEND_STAT("proxy_conn_errors", "%llu", (unsigned long long)thread_stats.proxy_conn_errors);
1820 APPEND_STAT("proxy_conn_oom", "%llu", (unsigned long long)thread_stats.proxy_conn_oom);
1821 APPEND_STAT("proxy_req_active", "%llu", (unsigned long long)thread_stats.proxy_req_active);
1822 }
1823 #endif
1824 APPEND_STAT("cmd_get", "%llu", (unsigned long long)thread_stats.get_cmds);
1825 APPEND_STAT("cmd_set", "%llu", (unsigned long long)slab_stats.set_cmds);
1826 APPEND_STAT("cmd_flush", "%llu", (unsigned long long)thread_stats.flush_cmds);
1827 APPEND_STAT("cmd_touch", "%llu", (unsigned long long)thread_stats.touch_cmds);
1828 APPEND_STAT("cmd_meta", "%llu", (unsigned long long)thread_stats.meta_cmds);
1829 APPEND_STAT("get_hits", "%llu", (unsigned long long)slab_stats.get_hits);
1830 APPEND_STAT("get_misses", "%llu", (unsigned long long)thread_stats.get_misses);
1831 APPEND_STAT("get_expired", "%llu", (unsigned long long)thread_stats.get_expired);
1832 APPEND_STAT("get_flushed", "%llu", (unsigned long long)thread_stats.get_flushed);
1833 #ifdef EXTSTORE
1834 if (ext_storage) {
1835 APPEND_STAT("get_extstore", "%llu", (unsigned long long)thread_stats.get_extstore);
1836 APPEND_STAT("get_aborted_extstore", "%llu", (unsigned long long)thread_stats.get_aborted_extstore);
1837 APPEND_STAT("get_oom_extstore", "%llu", (unsigned long long)thread_stats.get_oom_extstore);
1838 APPEND_STAT("recache_from_extstore", "%llu", (unsigned long long)thread_stats.recache_from_extstore);
1839 APPEND_STAT("miss_from_extstore", "%llu", (unsigned long long)thread_stats.miss_from_extstore);
1840 APPEND_STAT("badcrc_from_extstore", "%llu", (unsigned long long)thread_stats.badcrc_from_extstore);
1841 }
1842 #endif
1843 APPEND_STAT("delete_misses", "%llu", (unsigned long long)thread_stats.delete_misses);
1844 APPEND_STAT("delete_hits", "%llu", (unsigned long long)slab_stats.delete_hits);
1845 APPEND_STAT("incr_misses", "%llu", (unsigned long long)thread_stats.incr_misses);
1846 APPEND_STAT("incr_hits", "%llu", (unsigned long long)slab_stats.incr_hits);
1847 APPEND_STAT("decr_misses", "%llu", (unsigned long long)thread_stats.decr_misses);
1848 APPEND_STAT("decr_hits", "%llu", (unsigned long long)slab_stats.decr_hits);
1849 APPEND_STAT("cas_misses", "%llu", (unsigned long long)thread_stats.cas_misses);
1850 APPEND_STAT("cas_hits", "%llu", (unsigned long long)slab_stats.cas_hits);
1851 APPEND_STAT("cas_badval", "%llu", (unsigned long long)slab_stats.cas_badval);
1852 APPEND_STAT("touch_hits", "%llu", (unsigned long long)slab_stats.touch_hits);
1853 APPEND_STAT("touch_misses", "%llu", (unsigned long long)thread_stats.touch_misses);
1854 APPEND_STAT("store_too_large", "%llu", (unsigned long long)thread_stats.store_too_large);
1855 APPEND_STAT("store_no_memory", "%llu", (unsigned long long)thread_stats.store_no_memory);
1856 APPEND_STAT("auth_cmds", "%llu", (unsigned long long)thread_stats.auth_cmds);
1857 APPEND_STAT("auth_errors", "%llu", (unsigned long long)thread_stats.auth_errors);
1858 if (settings.idle_timeout) {
1859 APPEND_STAT("idle_kicks", "%llu", (unsigned long long)thread_stats.idle_kicks);
1860 }
1861 APPEND_STAT("bytes_read", "%llu", (unsigned long long)thread_stats.bytes_read);
1862 APPEND_STAT("bytes_written", "%llu", (unsigned long long)thread_stats.bytes_written);
1863 APPEND_STAT("limit_maxbytes", "%llu", (unsigned long long)settings.maxbytes);
1864 APPEND_STAT("accepting_conns", "%u", stats_state.accepting_conns);
1865 APPEND_STAT("listen_disabled_num", "%llu", (unsigned long long)stats.listen_disabled_num);
1866 APPEND_STAT("time_in_listen_disabled_us", "%llu", stats.time_in_listen_disabled_us);
1867 APPEND_STAT("threads", "%d", settings.num_threads);
1868 APPEND_STAT("conn_yields", "%llu", (unsigned long long)thread_stats.conn_yields);
1869 APPEND_STAT("hash_power_level", "%u", stats_state.hash_power_level);
1870 APPEND_STAT("hash_bytes", "%llu", (unsigned long long)stats_state.hash_bytes);
1871 APPEND_STAT("hash_is_expanding", "%u", stats_state.hash_is_expanding);
1872 if (settings.slab_reassign) {
1873 APPEND_STAT("slab_reassign_rescues", "%llu", stats.slab_reassign_rescues);
1874 APPEND_STAT("slab_reassign_chunk_rescues", "%llu", stats.slab_reassign_chunk_rescues);
1875 APPEND_STAT("slab_reassign_evictions_nomem", "%llu", stats.slab_reassign_evictions_nomem);
1876 APPEND_STAT("slab_reassign_inline_reclaim", "%llu", stats.slab_reassign_inline_reclaim);
1877 APPEND_STAT("slab_reassign_busy_items", "%llu", stats.slab_reassign_busy_items);
1878 APPEND_STAT("slab_reassign_busy_deletes", "%llu", stats.slab_reassign_busy_deletes);
1879 APPEND_STAT("slab_reassign_running", "%u", stats_state.slab_reassign_running);
1880 APPEND_STAT("slabs_moved", "%llu", stats.slabs_moved);
1881 }
1882 if (settings.lru_crawler) {
1883 APPEND_STAT("lru_crawler_running", "%u", stats_state.lru_crawler_running);
1884 APPEND_STAT("lru_crawler_starts", "%u", stats.lru_crawler_starts);
1885 }
1886 if (settings.lru_maintainer_thread) {
1887 APPEND_STAT("lru_maintainer_juggles", "%llu", (unsigned long long)stats.lru_maintainer_juggles);
1888 }
1889 APPEND_STAT("malloc_fails", "%llu",
1890 (unsigned long long)stats.malloc_fails);
1891 APPEND_STAT("log_worker_dropped", "%llu", (unsigned long long)stats.log_worker_dropped);
1892 APPEND_STAT("log_worker_written", "%llu", (unsigned long long)stats.log_worker_written);
1893 APPEND_STAT("log_watcher_skipped", "%llu", (unsigned long long)stats.log_watcher_skipped);
1894 APPEND_STAT("log_watcher_sent", "%llu", (unsigned long long)stats.log_watcher_sent);
1895 APPEND_STAT("log_watchers", "%llu", (unsigned long long)stats_state.log_watchers);
1896 STATS_UNLOCK();
1897 #ifdef EXTSTORE
1898 storage_stats(add_stats, c);
1899 #endif
1900 #ifdef PROXY
1901 proxy_stats(settings.proxy_ctx, add_stats, c);
1902 #endif
1903 #ifdef TLS
1904 if (settings.ssl_enabled) {
1905 if (settings.ssl_session_cache) {
1906 APPEND_STAT("ssl_new_sessions", "%llu", (unsigned long long)stats.ssl_new_sessions);
1907 }
1908 APPEND_STAT("ssl_handshake_errors", "%llu", (unsigned long long)stats.ssl_handshake_errors);
1909 APPEND_STAT("ssl_proto_errors", "%llu", (unsigned long long)stats.ssl_proto_errors);
1910 APPEND_STAT("time_since_server_cert_refresh", "%u", now - settings.ssl_last_cert_refresh_time);
1911 }
1912 #endif
1913 APPEND_STAT("unexpected_napi_ids", "%llu", (unsigned long long)stats.unexpected_napi_ids);
1914 APPEND_STAT("round_robin_fallback", "%llu", (unsigned long long)stats.round_robin_fallback);
1915 }
1916
process_stat_settings(ADD_STAT add_stats,void * c)1917 void process_stat_settings(ADD_STAT add_stats, void *c) {
1918 assert(add_stats);
1919 APPEND_STAT("maxbytes", "%llu", (unsigned long long)settings.maxbytes);
1920 APPEND_STAT("maxconns", "%d", settings.maxconns);
1921 APPEND_STAT("tcpport", "%d", settings.port);
1922 APPEND_STAT("udpport", "%d", settings.udpport);
1923 APPEND_STAT("inter", "%s", settings.inter ? settings.inter : "NULL");
1924 APPEND_STAT("verbosity", "%d", settings.verbose);
1925 APPEND_STAT("oldest", "%lu", (unsigned long)settings.oldest_live);
1926 APPEND_STAT("evictions", "%s", settings.evict_to_free ? "on" : "off");
1927 APPEND_STAT("domain_socket", "%s",
1928 settings.socketpath ? settings.socketpath : "NULL");
1929 APPEND_STAT("umask", "%o", settings.access);
1930 APPEND_STAT("shutdown_command", "%s",
1931 settings.shutdown_command ? "yes" : "no");
1932 APPEND_STAT("growth_factor", "%.2f", settings.factor);
1933 APPEND_STAT("chunk_size", "%d", settings.chunk_size);
1934 APPEND_STAT("num_threads", "%d", settings.num_threads);
1935 APPEND_STAT("num_threads_per_udp", "%d", settings.num_threads_per_udp);
1936 APPEND_STAT("stat_key_prefix", "%c", settings.prefix_delimiter);
1937 APPEND_STAT("detail_enabled", "%s",
1938 settings.detail_enabled ? "yes" : "no");
1939 APPEND_STAT("reqs_per_event", "%d", settings.reqs_per_event);
1940 APPEND_STAT("cas_enabled", "%s", settings.use_cas ? "yes" : "no");
1941 APPEND_STAT("tcp_backlog", "%d", settings.backlog);
1942 APPEND_STAT("binding_protocol", "%s",
1943 prot_text(settings.binding_protocol));
1944 APPEND_STAT("auth_enabled_sasl", "%s", settings.sasl ? "yes" : "no");
1945 APPEND_STAT("auth_enabled_ascii", "%s", settings.auth_file ? settings.auth_file : "no");
1946 APPEND_STAT("item_size_max", "%d", settings.item_size_max);
1947 APPEND_STAT("maxconns_fast", "%s", settings.maxconns_fast ? "yes" : "no");
1948 APPEND_STAT("hashpower_init", "%d", settings.hashpower_init);
1949 APPEND_STAT("slab_reassign", "%s", settings.slab_reassign ? "yes" : "no");
1950 APPEND_STAT("slab_automove", "%d", settings.slab_automove);
1951 APPEND_STAT("slab_automove_ratio", "%.2f", settings.slab_automove_ratio);
1952 APPEND_STAT("slab_automove_window", "%u", settings.slab_automove_window);
1953 APPEND_STAT("slab_chunk_max", "%d", settings.slab_chunk_size_max);
1954 APPEND_STAT("lru_crawler", "%s", settings.lru_crawler ? "yes" : "no");
1955 APPEND_STAT("lru_crawler_sleep", "%d", settings.lru_crawler_sleep);
1956 APPEND_STAT("lru_crawler_tocrawl", "%lu", (unsigned long)settings.lru_crawler_tocrawl);
1957 APPEND_STAT("tail_repair_time", "%d", settings.tail_repair_time);
1958 APPEND_STAT("flush_enabled", "%s", settings.flush_enabled ? "yes" : "no");
1959 APPEND_STAT("dump_enabled", "%s", settings.dump_enabled ? "yes" : "no");
1960 APPEND_STAT("hash_algorithm", "%s", settings.hash_algorithm);
1961 APPEND_STAT("lru_maintainer_thread", "%s", settings.lru_maintainer_thread ? "yes" : "no");
1962 APPEND_STAT("lru_segmented", "%s", settings.lru_segmented ? "yes" : "no");
1963 APPEND_STAT("hot_lru_pct", "%d", settings.hot_lru_pct);
1964 APPEND_STAT("warm_lru_pct", "%d", settings.warm_lru_pct);
1965 APPEND_STAT("hot_max_factor", "%.2f", settings.hot_max_factor);
1966 APPEND_STAT("warm_max_factor", "%.2f", settings.warm_max_factor);
1967 APPEND_STAT("temp_lru", "%s", settings.temp_lru ? "yes" : "no");
1968 APPEND_STAT("temporary_ttl", "%u", settings.temporary_ttl);
1969 APPEND_STAT("idle_timeout", "%d", settings.idle_timeout);
1970 APPEND_STAT("watcher_logbuf_size", "%u", settings.logger_watcher_buf_size);
1971 APPEND_STAT("worker_logbuf_size", "%u", settings.logger_buf_size);
1972 APPEND_STAT("read_buf_mem_limit", "%u", settings.read_buf_mem_limit);
1973 APPEND_STAT("track_sizes", "%s", item_stats_sizes_status() ? "yes" : "no");
1974 APPEND_STAT("inline_ascii_response", "%s", "no"); // setting is dead, cannot be yes.
1975 #ifdef HAVE_DROP_PRIVILEGES
1976 APPEND_STAT("drop_privileges", "%s", settings.drop_privileges ? "yes" : "no");
1977 #endif
1978 #ifdef EXTSTORE
1979 APPEND_STAT("ext_item_size", "%u", settings.ext_item_size);
1980 APPEND_STAT("ext_item_age", "%u", settings.ext_item_age);
1981 APPEND_STAT("ext_low_ttl", "%u", settings.ext_low_ttl);
1982 APPEND_STAT("ext_recache_rate", "%u", settings.ext_recache_rate);
1983 APPEND_STAT("ext_wbuf_size", "%u", settings.ext_wbuf_size);
1984 APPEND_STAT("ext_compact_under", "%u", settings.ext_compact_under);
1985 APPEND_STAT("ext_drop_under", "%u", settings.ext_drop_under);
1986 APPEND_STAT("ext_max_sleep", "%u", settings.ext_max_sleep);
1987 APPEND_STAT("ext_max_frag", "%.2f", settings.ext_max_frag);
1988 APPEND_STAT("slab_automove_freeratio", "%.3f", settings.slab_automove_freeratio);
1989 APPEND_STAT("ext_drop_unread", "%s", settings.ext_drop_unread ? "yes" : "no");
1990 #endif
1991 #ifdef TLS
1992 APPEND_STAT("ssl_enabled", "%s", settings.ssl_enabled ? "yes" : "no");
1993 APPEND_STAT("ssl_chain_cert", "%s", settings.ssl_chain_cert);
1994 APPEND_STAT("ssl_key", "%s", settings.ssl_key);
1995 APPEND_STAT("ssl_verify_mode", "%d", settings.ssl_verify_mode);
1996 APPEND_STAT("ssl_keyformat", "%d", settings.ssl_keyformat);
1997 APPEND_STAT("ssl_ciphers", "%s", settings.ssl_ciphers ? settings.ssl_ciphers : "NULL");
1998 APPEND_STAT("ssl_ca_cert", "%s", settings.ssl_ca_cert ? settings.ssl_ca_cert : "NULL");
1999 APPEND_STAT("ssl_wbuf_size", "%u", settings.ssl_wbuf_size);
2000 APPEND_STAT("ssl_session_cache", "%s", settings.ssl_session_cache ? "yes" : "no");
2001 APPEND_STAT("ssl_kernel_tls", "%s", settings.ssl_kernel_tls ? "yes" : "no");
2002 APPEND_STAT("ssl_min_version", "%s", ssl_proto_text(settings.ssl_min_version));
2003 #endif
2004 #ifdef PROXY
2005 APPEND_STAT("proxy_enabled", "%s", settings.proxy_enabled ? "yes" : "no");
2006 APPEND_STAT("proxy_uring_enabled", "%s", settings.proxy_uring ? "yes" : "no");
2007 #endif
2008 APPEND_STAT("num_napi_ids", "%s", settings.num_napi_ids);
2009 APPEND_STAT("memory_file", "%s", settings.memory_file);
2010 APPEND_STAT("client_flags_size", "%d", sizeof(client_flags_t));
2011 }
2012
nz_strcmp(int nzlength,const char * nz,const char * z)2013 static int nz_strcmp(int nzlength, const char *nz, const char *z) {
2014 int zlength=strlen(z);
2015 return (zlength == nzlength) && (strncmp(nz, z, zlength) == 0) ? 0 : -1;
2016 }
2017
get_stats(const char * stat_type,int nkey,ADD_STAT add_stats,void * c)2018 bool get_stats(const char *stat_type, int nkey, ADD_STAT add_stats, void *c) {
2019 bool ret = true;
2020
2021 if (add_stats != NULL) {
2022 if (!stat_type) {
2023 /* prepare general statistics for the engine */
2024 STATS_LOCK();
2025 APPEND_STAT("bytes", "%llu", (unsigned long long)stats_state.curr_bytes);
2026 APPEND_STAT("curr_items", "%llu", (unsigned long long)stats_state.curr_items);
2027 APPEND_STAT("total_items", "%llu", (unsigned long long)stats.total_items);
2028 STATS_UNLOCK();
2029 APPEND_STAT("slab_global_page_pool", "%u", global_page_pool_size(NULL));
2030 item_stats_totals(add_stats, c);
2031 } else if (nz_strcmp(nkey, stat_type, "items") == 0) {
2032 item_stats(add_stats, c);
2033 } else if (nz_strcmp(nkey, stat_type, "slabs") == 0) {
2034 slabs_stats(add_stats, c);
2035 } else if (nz_strcmp(nkey, stat_type, "sizes") == 0) {
2036 item_stats_sizes(add_stats, c);
2037 } else {
2038 ret = false;
2039 }
2040 } else {
2041 ret = false;
2042 }
2043
2044 return ret;
2045 }
2046
get_conn_text(const conn * c,const int af,char * addr,struct sockaddr * sock_addr)2047 static inline void get_conn_text(const conn *c, const int af,
2048 char* addr, struct sockaddr *sock_addr) {
2049 char addr_text[MAXPATHLEN];
2050 addr_text[0] = '\0';
2051 const char *protoname = "?";
2052 unsigned short port = 0;
2053
2054 switch (af) {
2055 case AF_INET:
2056 (void) inet_ntop(af,
2057 &((struct sockaddr_in *)sock_addr)->sin_addr,
2058 addr_text,
2059 sizeof(addr_text) - 1);
2060 port = ntohs(((struct sockaddr_in *)sock_addr)->sin_port);
2061 protoname = IS_UDP(c->transport) ? "udp" : "tcp";
2062 break;
2063
2064 case AF_INET6:
2065 addr_text[0] = '[';
2066 addr_text[1] = '\0';
2067 if (inet_ntop(af,
2068 &((struct sockaddr_in6 *)sock_addr)->sin6_addr,
2069 addr_text + 1,
2070 sizeof(addr_text) - 2)) {
2071 strncat(addr_text, "]", 2);
2072 }
2073 port = ntohs(((struct sockaddr_in6 *)sock_addr)->sin6_port);
2074 protoname = IS_UDP(c->transport) ? "udp6" : "tcp6";
2075 break;
2076
2077 #ifndef DISABLE_UNIX_SOCKET
2078 case AF_UNIX:
2079 {
2080 size_t pathlen = 0;
2081 // this strncpy call originally could piss off an address
2082 // sanitizer; we supplied the size of the dest buf as a limiter,
2083 // but optimized versions of strncpy could read past the end of
2084 // *src while looking for a null terminator. Since buf and
2085 // sun_path here are both on the stack they could even overlap,
2086 // which is "undefined". In all OSS versions of strncpy I could
2087 // find this has no effect; it'll still only copy until the first null
2088 // terminator is found. Thus it's possible to get the OS to
2089 // examine past the end of sun_path but it's unclear to me if this
2090 // can cause any actual problem.
2091 //
2092 // We need a safe_strncpy util function but I'll punt on figuring
2093 // that out for now.
2094 pathlen = sizeof(((struct sockaddr_un *)sock_addr)->sun_path);
2095 if (MAXPATHLEN <= pathlen) {
2096 pathlen = MAXPATHLEN - 1;
2097 }
2098 strncpy(addr_text,
2099 ((struct sockaddr_un *)sock_addr)->sun_path,
2100 pathlen);
2101 addr_text[pathlen] = '\0';
2102 protoname = "unix";
2103 }
2104 break;
2105 #endif /* #ifndef DISABLE_UNIX_SOCKET */
2106 }
2107
2108 if (strlen(addr_text) < 2) {
2109 /* Most likely this is a connected UNIX-domain client which
2110 * has no peer socket address, but there's no portable way
2111 * to tell for sure.
2112 */
2113 snprintf(addr_text, MAXPATHLEN, "<AF %d>", af);
2114 }
2115
2116 if (port) {
2117 snprintf(addr, MAXPATHLEN + 11, "%s:%s:%u", protoname, addr_text, port);
2118 } else {
2119 snprintf(addr, MAXPATHLEN + 11, "%s:%s", protoname, addr_text);
2120 }
2121 }
2122
conn_to_str(const conn * c,char * addr,char * svr_addr)2123 static void conn_to_str(const conn *c, char *addr, char *svr_addr) {
2124 if (!c) {
2125 memcpy(addr, "<null>", 6);
2126 } else if (c->state == conn_closed) {
2127 memcpy(addr, "<closed>", 8);
2128 } else {
2129 struct sockaddr_in6 local_addr;
2130 struct sockaddr *sock_addr = (void *)&c->request_addr;
2131
2132 /* For listen ports and idle UDP ports, show listen address */
2133 if (c->state == conn_listening ||
2134 (IS_UDP(c->transport) &&
2135 c->state == conn_read)) {
2136 memset(&local_addr, 0, sizeof(local_addr));
2137 socklen_t local_addr_len = sizeof(local_addr);
2138
2139 if (getsockname(c->sfd,
2140 (struct sockaddr *)&local_addr,
2141 &local_addr_len) == 0) {
2142 sock_addr = (struct sockaddr *)&local_addr;
2143 }
2144 }
2145 get_conn_text(c, sock_addr->sa_family, addr, sock_addr);
2146
2147 if (c->state != conn_listening && !(IS_UDP(c->transport) &&
2148 c->state == conn_read)) {
2149 struct sockaddr_storage svr_sock_addr;
2150 memset(&svr_sock_addr, 0, sizeof(svr_sock_addr));
2151 socklen_t svr_addr_len = sizeof(svr_sock_addr);
2152 getsockname(c->sfd, (struct sockaddr *)&svr_sock_addr, &svr_addr_len);
2153 get_conn_text(c, svr_sock_addr.ss_family, svr_addr, (struct sockaddr *)&svr_sock_addr);
2154 }
2155 }
2156 }
2157
conn_queue_to_str(const conn * c,io_queue_t * q)2158 static char *conn_queue_to_str(const conn *c, io_queue_t *q) {
2159 if (q->type == IO_QUEUE_EXTSTORE) {
2160 return "queue_extstore";
2161 } else if (q->type == IO_QUEUE_PROXY) {
2162 return "queue_proxy";
2163 } else {
2164 return "queue_unknown";
2165 }
2166 }
2167
process_stats_conns(ADD_STAT add_stats,void * c)2168 void process_stats_conns(ADD_STAT add_stats, void *c) {
2169 int i;
2170 char key_str[STAT_KEY_LEN];
2171 char val_str[STAT_VAL_LEN];
2172 size_t extras_len = sizeof(":unix:") + sizeof("65535");
2173 char addr[MAXPATHLEN + extras_len];
2174 char svr_addr[MAXPATHLEN + extras_len];
2175 memset(addr, 0, sizeof(addr));
2176 memset(svr_addr, 0, sizeof(svr_addr));
2177 int klen = 0, vlen = 0;
2178
2179 assert(add_stats);
2180
2181 for (i = 0; i < max_fds; i++) {
2182 if (conns[i]) {
2183 /* This is safe to do unlocked because conns are never freed; the
2184 * worst that'll happen will be a minor inconsistency in the
2185 * output -- not worth the complexity of the locking that'd be
2186 * required to prevent it.
2187 */
2188 if (IS_UDP(conns[i]->transport)) {
2189 APPEND_NUM_STAT(i, "UDP", "%s", "UDP");
2190 }
2191 if (conns[i]->state != conn_closed) {
2192 conn *sc = conns[i];
2193 conn_to_str(sc, addr, svr_addr);
2194
2195 APPEND_NUM_STAT(i, "addr", "%s", addr);
2196 if (sc->state != conn_listening &&
2197 !(IS_UDP(sc->transport) && sc->state == conn_read)) {
2198 APPEND_NUM_STAT(i, "listen_addr", "%s", svr_addr);
2199 }
2200 APPEND_NUM_STAT(i, "state", "%s",
2201 state_text(sc->state));
2202 if (sc->io_queues_submitted) {
2203 APPEND_NUM_STAT(i, "queues_waiting", "%d", sc->io_queues_submitted);
2204 for (io_queue_t *q = sc->io_queues; q->type != IO_QUEUE_NONE; q++) {
2205 if (q->count) {
2206 const char *qname = conn_queue_to_str(sc, q);
2207 APPEND_NUM_STAT(i, qname, "%d", q->count);
2208 }
2209 }
2210 }
2211 APPEND_NUM_STAT(i, "secs_since_last_cmd", "%d",
2212 current_time - sc->last_cmd_time);
2213 }
2214 }
2215 }
2216 }
2217
2218 #define IT_REFCOUNT_LIMIT 60000
limited_get(const char * key,size_t nkey,LIBEVENT_THREAD * t,uint32_t exptime,bool should_touch,bool do_update,bool * overflow)2219 item* limited_get(const char *key, size_t nkey, LIBEVENT_THREAD *t, uint32_t exptime, bool should_touch, bool do_update, bool *overflow) {
2220 item *it;
2221 if (should_touch) {
2222 it = item_touch(key, nkey, exptime, t);
2223 } else {
2224 it = item_get(key, nkey, t, do_update);
2225 }
2226 if (it && it->refcount > IT_REFCOUNT_LIMIT) {
2227 item_remove(it);
2228 it = NULL;
2229 *overflow = true;
2230 } else {
2231 *overflow = false;
2232 }
2233 return it;
2234 }
2235
2236 // Semantics are different than limited_get; since the item is returned
2237 // locked, caller can directly change what it needs.
2238 // though it might eventually be a better interface to sink it all into
2239 // items.c.
limited_get_locked(const char * key,size_t nkey,LIBEVENT_THREAD * t,bool do_update,uint32_t * hv,bool * overflow)2240 item* limited_get_locked(const char *key, size_t nkey, LIBEVENT_THREAD *t, bool do_update, uint32_t *hv, bool *overflow) {
2241 item *it;
2242 it = item_get_locked(key, nkey, t, do_update, hv);
2243 if (it && it->refcount > IT_REFCOUNT_LIMIT) {
2244 do_item_remove(it);
2245 it = NULL;
2246 item_unlock(*hv);
2247 *overflow = true;
2248 } else {
2249 *overflow = false;
2250 }
2251 return it;
2252 }
2253
2254 /*
2255 * adds a delta value to a numeric item.
2256 *
2257 * c connection requesting the operation
2258 * it item to adjust
2259 * incr true to increment value, false to decrement
2260 * delta amount to adjust value by
2261 * buf buffer for response string
2262 *
2263 * returns a response string to send back to the client.
2264 */
do_add_delta(LIBEVENT_THREAD * t,const char * key,const size_t nkey,const bool incr,const int64_t delta,char * buf,uint64_t * cas,const uint32_t hv,item ** it_ret)2265 enum delta_result_type do_add_delta(LIBEVENT_THREAD *t, const char *key, const size_t nkey,
2266 const bool incr, const int64_t delta,
2267 char *buf, uint64_t *cas,
2268 const uint32_t hv,
2269 item **it_ret) {
2270 char *ptr;
2271 uint64_t value;
2272 int res;
2273 item *it;
2274
2275 it = do_item_get(key, nkey, hv, t, DONT_UPDATE);
2276 if (!it) {
2277 return DELTA_ITEM_NOT_FOUND;
2278 }
2279
2280 /* Can't delta zero byte values. 2-byte are the "\r\n" */
2281 /* Also can't delta for chunked items. Too large to be a number */
2282 #ifdef EXTSTORE
2283 if (it->nbytes <= 2 || (it->it_flags & (ITEM_CHUNKED|ITEM_HDR)) != 0) {
2284 #else
2285 if (it->nbytes <= 2 || (it->it_flags & (ITEM_CHUNKED)) != 0) {
2286 #endif
2287 do_item_remove(it);
2288 return NON_NUMERIC;
2289 }
2290
2291 if (cas != NULL && *cas != 0 && ITEM_get_cas(it) != *cas) {
2292 do_item_remove(it);
2293 return DELTA_ITEM_CAS_MISMATCH;
2294 }
2295
2296 ptr = ITEM_data(it);
2297
2298 if (!safe_strtoull(ptr, &value)) {
2299 do_item_remove(it);
2300 return NON_NUMERIC;
2301 }
2302
2303 if (incr) {
2304 value += delta;
2305 //MEMCACHED_COMMAND_INCR(c->sfd, ITEM_key(it), it->nkey, value);
2306 } else {
2307 if(delta > value) {
2308 value = 0;
2309 } else {
2310 value -= delta;
2311 }
2312 //MEMCACHED_COMMAND_DECR(c->sfd, ITEM_key(it), it->nkey, value);
2313 }
2314
2315 pthread_mutex_lock(&t->stats.mutex);
2316 if (incr) {
2317 t->stats.slab_stats[ITEM_clsid(it)].incr_hits++;
2318 } else {
2319 t->stats.slab_stats[ITEM_clsid(it)].decr_hits++;
2320 }
2321 pthread_mutex_unlock(&t->stats.mutex);
2322
2323 itoa_u64(value, buf);
2324 res = strlen(buf);
2325 /* refcount == 2 means we are the only ones holding the item, and it is
2326 * linked. We hold the item's lock in this function, so refcount cannot
2327 * increase. */
2328 if (res + 2 <= it->nbytes && it->refcount == 2) { /* replace in-place */
2329 /* When changing the value without replacing the item, we
2330 need to update the CAS on the existing item. */
2331 /* We also need to fiddle it in the sizes tracker in case the tracking
2332 * was enabled at runtime, since it relies on the CAS value to know
2333 * whether to remove an item or not. */
2334 item_stats_sizes_remove(it);
2335 ITEM_set_cas(it, (settings.use_cas) ? get_cas_id() : 0);
2336 item_stats_sizes_add(it);
2337 memcpy(ITEM_data(it), buf, res);
2338 memset(ITEM_data(it) + res, ' ', it->nbytes - res - 2);
2339 do_item_update(it);
2340 } else if (it->refcount > 1) {
2341 item *new_it;
2342 client_flags_t flags;
2343 FLAGS_CONV(it, flags);
2344 new_it = do_item_alloc(ITEM_key(it), it->nkey, flags, it->exptime, res + 2);
2345 if (new_it == 0) {
2346 do_item_remove(it);
2347 return EOM;
2348 }
2349 memcpy(ITEM_data(new_it), buf, res);
2350 memcpy(ITEM_data(new_it) + res, "\r\n", 2);
2351 item_replace(it, new_it, hv, (settings.use_cas) ? get_cas_id() : 0);
2352 // Overwrite the older item's CAS with our new CAS since we're
2353 // returning the CAS of the old item below.
2354 ITEM_set_cas(it, (settings.use_cas) ? ITEM_get_cas(new_it) : 0);
2355 do_item_remove(new_it); /* release our reference */
2356 } else {
2357 /* Should never get here. This means we somehow fetched an unlinked
2358 * item. TODO: Add a counter? */
2359 if (settings.verbose) {
2360 fprintf(stderr, "Tried to do incr/decr on invalid item\n");
2361 }
2362 if (it->refcount == 1)
2363 do_item_remove(it);
2364 return DELTA_ITEM_NOT_FOUND;
2365 }
2366
2367 if (cas) {
2368 *cas = ITEM_get_cas(it); /* swap the incoming CAS value */
2369 }
2370 if (it_ret != NULL) {
2371 *it_ret = it;
2372 } else {
2373 do_item_remove(it); /* release our reference */
2374 }
2375 return OK;
2376 }
2377
2378 static int try_read_command_negotiate(conn *c) {
2379 assert(c != NULL);
2380 assert(c->protocol == negotiating_prot);
2381 assert(c->rcurr <= (c->rbuf + c->rsize));
2382 assert(c->rbytes > 0);
2383
2384 if ((unsigned char)c->rbuf[0] == (unsigned char)PROTOCOL_BINARY_REQ) {
2385 c->protocol = binary_prot;
2386 c->try_read_command = try_read_command_binary;
2387 } else {
2388 // authentication doesn't work with negotiated protocol.
2389 c->protocol = ascii_prot;
2390 c->try_read_command = try_read_command_ascii;
2391 }
2392
2393 if (settings.verbose > 1) {
2394 fprintf(stderr, "%d: Client using the %s protocol\n", c->sfd,
2395 prot_text(c->protocol));
2396 }
2397
2398 return c->try_read_command(c);
2399 }
2400
2401 static int try_read_command_udp(conn *c) {
2402 assert(c != NULL);
2403 assert(c->rcurr <= (c->rbuf + c->rsize));
2404 assert(c->rbytes > 0);
2405
2406 if ((unsigned char)c->rbuf[0] == (unsigned char)PROTOCOL_BINARY_REQ) {
2407 c->protocol = binary_prot;
2408 return try_read_command_binary(c);
2409 } else {
2410 c->protocol = ascii_prot;
2411 return try_read_command_ascii(c);
2412 }
2413 }
2414
2415 /*
2416 * read a UDP request.
2417 */
2418 static enum try_read_result try_read_udp(conn *c) {
2419 int res;
2420
2421 assert(c != NULL);
2422
2423 c->request_addr_size = sizeof(c->request_addr);
2424 res = recvfrom(c->sfd, c->rbuf, c->rsize,
2425 0, (struct sockaddr *)&c->request_addr,
2426 &c->request_addr_size);
2427 if (res > 8) {
2428 unsigned char *buf = (unsigned char *)c->rbuf;
2429 pthread_mutex_lock(&c->thread->stats.mutex);
2430 c->thread->stats.bytes_read += res;
2431 pthread_mutex_unlock(&c->thread->stats.mutex);
2432
2433 /* Beginning of UDP packet is the request ID; save it. */
2434 c->request_id = buf[0] * 256 + buf[1];
2435
2436 /* If this is a multi-packet request, drop it. */
2437 if (buf[4] != 0 || buf[5] != 1) {
2438 return READ_NO_DATA_RECEIVED;
2439 }
2440
2441 /* Don't care about any of the rest of the header. */
2442 res -= 8;
2443 memmove(c->rbuf, c->rbuf + 8, res);
2444
2445 c->rbytes = res;
2446 c->rcurr = c->rbuf;
2447 return READ_DATA_RECEIVED;
2448 }
2449 return READ_NO_DATA_RECEIVED;
2450 }
2451
2452 /*
2453 * read from network as much as we can, handle buffer overflow and connection
2454 * close.
2455 * before reading, move the remaining incomplete fragment of a command
2456 * (if any) to the beginning of the buffer.
2457 *
2458 * To protect us from someone flooding a connection with bogus data causing
2459 * the connection to eat up all available memory, break out and start looking
2460 * at the data I've got after a number of reallocs...
2461 *
2462 * @return enum try_read_result
2463 */
2464 static enum try_read_result try_read_network(conn *c) {
2465 enum try_read_result gotdata = READ_NO_DATA_RECEIVED;
2466 int res;
2467 int num_allocs = 0;
2468 assert(c != NULL);
2469
2470 if (c->rcurr != c->rbuf) {
2471 if (c->rbytes > 0) /* otherwise there's nothing to copy */
2472 memmove(c->rbuf, c->rcurr, c->rbytes);
2473 c->rcurr = c->rbuf;
2474 }
2475
2476 while (1) {
2477 // TODO: move to rbuf_* func?
2478 if (c->rbytes >= c->rsize && c->rbuf_malloced) {
2479 if (num_allocs == 4) {
2480 return gotdata;
2481 }
2482 ++num_allocs;
2483 char *new_rbuf = realloc(c->rbuf, c->rsize * 2);
2484 if (!new_rbuf) {
2485 STATS_LOCK();
2486 stats.malloc_fails++;
2487 STATS_UNLOCK();
2488 if (settings.verbose > 0) {
2489 fprintf(stderr, "Couldn't realloc input buffer\n");
2490 }
2491 c->rbytes = 0; /* ignore what we read */
2492 out_of_memory(c, "SERVER_ERROR out of memory reading request");
2493 c->close_after_write = true;
2494 return READ_MEMORY_ERROR;
2495 }
2496 c->rcurr = c->rbuf = new_rbuf;
2497 c->rsize *= 2;
2498 }
2499
2500 int avail = c->rsize - c->rbytes;
2501 res = c->read(c, c->rbuf + c->rbytes, avail);
2502 if (res > 0) {
2503 pthread_mutex_lock(&c->thread->stats.mutex);
2504 c->thread->stats.bytes_read += res;
2505 pthread_mutex_unlock(&c->thread->stats.mutex);
2506 gotdata = READ_DATA_RECEIVED;
2507 c->rbytes += res;
2508 if (res == avail && c->rbuf_malloced) {
2509 // Resize rbuf and try a few times if huge ascii multiget.
2510 continue;
2511 } else {
2512 break;
2513 }
2514 }
2515 if (res == 0) {
2516 c->close_reason = NORMAL_CLOSE;
2517 return READ_ERROR;
2518 }
2519 if (res == -1) {
2520 if (errno == EAGAIN || errno == EWOULDBLOCK) {
2521 break;
2522 }
2523 return READ_ERROR;
2524 }
2525 }
2526 return gotdata;
2527 }
2528
2529 static bool update_event(conn *c, const int new_flags) {
2530 assert(c != NULL);
2531
2532 struct event_base *base = c->event.ev_base;
2533 if (c->ev_flags == new_flags)
2534 return true;
2535 if (event_del(&c->event) == -1) return false;
2536 event_set(&c->event, c->sfd, new_flags, event_handler, (void *)c);
2537 event_base_set(base, &c->event);
2538 c->ev_flags = new_flags;
2539 if (event_add(&c->event, 0) == -1) return false;
2540 return true;
2541 }
2542
2543 /*
2544 * Sets whether we are listening for new connections or not.
2545 */
2546 void do_accept_new_conns(const bool do_accept) {
2547 conn *next;
2548
2549 for (next = listen_conn; next; next = next->next) {
2550 if (do_accept) {
2551 update_event(next, EV_READ | EV_PERSIST);
2552 if (listen(next->sfd, settings.backlog) != 0) {
2553 perror("listen");
2554 }
2555 }
2556 else {
2557 update_event(next, 0);
2558 if (listen(next->sfd, 0) != 0) {
2559 perror("listen");
2560 }
2561 }
2562 }
2563
2564 if (do_accept) {
2565 struct timeval maxconns_exited;
2566 uint64_t elapsed_us;
2567 gettimeofday(&maxconns_exited,NULL);
2568 STATS_LOCK();
2569 elapsed_us =
2570 (maxconns_exited.tv_sec - stats.maxconns_entered.tv_sec) * 1000000
2571 + (maxconns_exited.tv_usec - stats.maxconns_entered.tv_usec);
2572 stats.time_in_listen_disabled_us += elapsed_us;
2573 stats_state.accepting_conns = true;
2574 STATS_UNLOCK();
2575 } else {
2576 STATS_LOCK();
2577 stats_state.accepting_conns = false;
2578 gettimeofday(&stats.maxconns_entered,NULL);
2579 stats.listen_disabled_num++;
2580 STATS_UNLOCK();
2581 allow_new_conns = false;
2582 maxconns_handler(-42, 0, 0);
2583 }
2584 }
2585
2586 #define TRANSMIT_ONE_RESP true
2587 #define TRANSMIT_ALL_RESP false
2588 static int _transmit_pre(conn *c, struct iovec *iovs, int iovused, bool one_resp) {
2589 mc_resp *resp = c->resp_head;
2590 while (resp && iovused + resp->iovcnt < IOV_MAX-1) {
2591 if (resp->skip) {
2592 // Don't actually unchain the resp obj here since it's singly-linked.
2593 // Just let the post function handle it linearly.
2594 resp = resp->next;
2595 continue;
2596 }
2597 if (resp->chunked_data_iov) {
2598 // Handle chunked items specially.
2599 // They spend much more time in send so we can be a bit wasteful
2600 // in rebuilding iovecs for them.
2601 item_chunk *ch = (item_chunk *)ITEM_schunk((item *)resp->iov[resp->chunked_data_iov].iov_base);
2602 int x;
2603 for (x = 0; x < resp->iovcnt; x++) {
2604 // This iov is tracking how far we've copied so far.
2605 if (x == resp->chunked_data_iov) {
2606 int done = resp->chunked_total - resp->iov[x].iov_len;
2607 // Start from the len to allow binprot to cut the \r\n
2608 int todo = resp->iov[x].iov_len;
2609 while (ch && todo > 0 && iovused < IOV_MAX-1) {
2610 int skip = 0;
2611 if (!ch->used) {
2612 ch = ch->next;
2613 continue;
2614 }
2615 // Skip parts we've already sent.
2616 if (done >= ch->used) {
2617 done -= ch->used;
2618 ch = ch->next;
2619 continue;
2620 } else if (done) {
2621 skip = done;
2622 done = 0;
2623 }
2624 iovs[iovused].iov_base = ch->data + skip;
2625 // Stupid binary protocol makes this go negative.
2626 iovs[iovused].iov_len = ch->used - skip > todo ? todo : ch->used - skip;
2627 iovused++;
2628 todo -= ch->used - skip;
2629 ch = ch->next;
2630 }
2631 } else {
2632 iovs[iovused].iov_base = resp->iov[x].iov_base;
2633 iovs[iovused].iov_len = resp->iov[x].iov_len;
2634 iovused++;
2635 }
2636 if (iovused >= IOV_MAX-1)
2637 break;
2638 }
2639 } else {
2640 memcpy(&iovs[iovused], resp->iov, sizeof(struct iovec)*resp->iovcnt);
2641 iovused += resp->iovcnt;
2642 }
2643
2644 // done looking at first response, walk down the chain.
2645 resp = resp->next;
2646 // used for UDP mode: UDP cannot send multiple responses per packet.
2647 if (one_resp)
2648 break;
2649 }
2650 return iovused;
2651 }
2652
2653 /*
2654 * Decrements and completes responses based on how much data was transmitted.
2655 * Takes the connection and current result bytes.
2656 */
2657 static void _transmit_post(conn *c, ssize_t res) {
2658 // We've written some of the data. Remove the completed
2659 // responses from the list of pending writes.
2660 mc_resp *resp = c->resp_head;
2661 while (resp) {
2662 int x;
2663 if (resp->skip) {
2664 resp = resp_finish(c, resp);
2665 continue;
2666 }
2667
2668 // fastpath check. all small responses should cut here.
2669 if (res >= resp->tosend) {
2670 res -= resp->tosend;
2671 resp = resp_finish(c, resp);
2672 continue;
2673 }
2674
2675 // it's fine to re-check iov's that were zeroed out before.
2676 for (x = 0; x < resp->iovcnt; x++) {
2677 struct iovec *iov = &resp->iov[x];
2678 if (res >= iov->iov_len) {
2679 resp->tosend -= iov->iov_len;
2680 res -= iov->iov_len;
2681 iov->iov_len = 0;
2682 } else {
2683 // Dumb special case for chunked items. Currently tracking
2684 // where to inject the chunked item via iov_base.
2685 // Extra not-great since chunked items can't be the first
2686 // index, so we have to check for non-zero c_d_iov first.
2687 if (!resp->chunked_data_iov || x != resp->chunked_data_iov) {
2688 iov->iov_base = (char *)iov->iov_base + res;
2689 }
2690 iov->iov_len -= res;
2691 resp->tosend -= res;
2692 res = 0;
2693 break;
2694 }
2695 }
2696
2697 // are we done with this response object?
2698 if (resp->tosend == 0) {
2699 resp = resp_finish(c, resp);
2700 } else {
2701 // Jammed up here. This is the new head.
2702 break;
2703 }
2704 }
2705 }
2706
2707 /*
2708 * Transmit the next chunk of data from our list of msgbuf structures.
2709 *
2710 * Returns:
2711 * TRANSMIT_COMPLETE All done writing.
2712 * TRANSMIT_INCOMPLETE More data remaining to write.
2713 * TRANSMIT_SOFT_ERROR Can't write any more right now.
2714 * TRANSMIT_HARD_ERROR Can't write (c->state is set to conn_closing)
2715 */
2716 static enum transmit_result transmit(conn *c) {
2717 assert(c != NULL);
2718 struct iovec iovs[IOV_MAX];
2719 struct msghdr msg;
2720 int iovused = 0;
2721
2722 // init the msg.
2723 memset(&msg, 0, sizeof(struct msghdr));
2724 msg.msg_iov = iovs;
2725
2726 iovused = _transmit_pre(c, iovs, iovused, TRANSMIT_ALL_RESP);
2727 if (iovused == 0) {
2728 // Avoid the syscall if we're only handling a noreply.
2729 // Return the response object.
2730 _transmit_post(c, 0);
2731 return TRANSMIT_COMPLETE;
2732 }
2733
2734 // Alright, send.
2735 ssize_t res;
2736 msg.msg_iovlen = iovused;
2737 res = c->sendmsg(c, &msg, 0);
2738 if (res >= 0) {
2739 pthread_mutex_lock(&c->thread->stats.mutex);
2740 c->thread->stats.bytes_written += res;
2741 pthread_mutex_unlock(&c->thread->stats.mutex);
2742
2743 // Decrement any partial IOV's and complete any finished resp's.
2744 _transmit_post(c, res);
2745
2746 if (c->resp_head) {
2747 return TRANSMIT_INCOMPLETE;
2748 } else {
2749 return TRANSMIT_COMPLETE;
2750 }
2751 }
2752
2753 if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
2754 if (!update_event(c, EV_WRITE | EV_PERSIST)) {
2755 if (settings.verbose > 0)
2756 fprintf(stderr, "Couldn't update event\n");
2757 conn_set_state(c, conn_closing);
2758 return TRANSMIT_HARD_ERROR;
2759 }
2760 return TRANSMIT_SOFT_ERROR;
2761 }
2762 /* if res == -1 and error is not EAGAIN or EWOULDBLOCK,
2763 we have a real error, on which we close the connection */
2764 if (settings.verbose > 0)
2765 perror("Failed to write, and not due to blocking");
2766
2767 conn_set_state(c, conn_closing);
2768 return TRANSMIT_HARD_ERROR;
2769 }
2770
2771 static void build_udp_header(unsigned char *hdr, mc_resp *resp) {
2772 // We need to communicate the total number of packets
2773 // If this isn't set, it's the first time this response is building a udp
2774 // header, so "tosend" must be static.
2775 if (!resp->udp_total) {
2776 uint32_t total;
2777 total = resp->tosend / UDP_DATA_SIZE;
2778 if (resp->tosend % UDP_DATA_SIZE)
2779 total++;
2780 // The spec doesn't really say what we should do here. It's _probably_
2781 // better to bail out?
2782 if (total > USHRT_MAX) {
2783 total = USHRT_MAX;
2784 }
2785 resp->udp_total = total;
2786 }
2787
2788 // TODO: why wasn't this hto*'s and casts?
2789 // this ends up sending UDP hdr data specifically in host byte order.
2790 *hdr++ = resp->request_id / 256;
2791 *hdr++ = resp->request_id % 256;
2792 *hdr++ = resp->udp_sequence / 256;
2793 *hdr++ = resp->udp_sequence % 256;
2794 *hdr++ = resp->udp_total / 256;
2795 *hdr++ = resp->udp_total % 256;
2796 *hdr++ = 0;
2797 *hdr++ = 0;
2798 resp->udp_sequence++;
2799 }
2800
2801 /*
2802 * UDP specific transmit function. Uses its own function rather than check
2803 * IS_UDP() five times. If we ever implement sendmmsg or similar support they
2804 * will diverge even more.
2805 * Does not use TLS.
2806 *
2807 * Returns:
2808 * TRANSMIT_COMPLETE All done writing.
2809 * TRANSMIT_INCOMPLETE More data remaining to write.
2810 * TRANSMIT_SOFT_ERROR Can't write any more right now.
2811 * TRANSMIT_HARD_ERROR Can't write (c->state is set to conn_closing)
2812 */
2813 static enum transmit_result transmit_udp(conn *c) {
2814 assert(c != NULL);
2815 struct iovec iovs[IOV_MAX];
2816 struct msghdr msg;
2817 mc_resp *resp;
2818 int iovused = 0;
2819 unsigned char udp_hdr[UDP_HEADER_SIZE];
2820
2821 // We only send one UDP packet per call (ugh), so we can only operate on a
2822 // single response at a time.
2823 resp = c->resp_head;
2824
2825 if (!resp) {
2826 return TRANSMIT_COMPLETE;
2827 }
2828
2829 if (resp->skip) {
2830 resp = resp_finish(c, resp);
2831 return TRANSMIT_INCOMPLETE;
2832 }
2833
2834 // clear the message and initialize it.
2835 memset(&msg, 0, sizeof(struct msghdr));
2836 msg.msg_iov = iovs;
2837
2838 // the UDP source to return to.
2839 msg.msg_name = &resp->request_addr;
2840 msg.msg_namelen = resp->request_addr_size;
2841
2842 // First IOV is the custom UDP header.
2843 iovs[0].iov_base = (void *)udp_hdr;
2844 iovs[0].iov_len = UDP_HEADER_SIZE;
2845 build_udp_header(udp_hdr, resp);
2846 iovused++;
2847
2848 // Fill the IOV's the standard way.
2849 // TODO: might get a small speedup if we let it break early with a length
2850 // limit.
2851 iovused = _transmit_pre(c, iovs, iovused, TRANSMIT_ONE_RESP);
2852
2853 // Clip the IOV's to the max UDP packet size.
2854 // If we add support for send_mmsg, this can be where we split msg's.
2855 {
2856 int x = 0;
2857 int len = 0;
2858 for (x = 0; x < iovused; x++) {
2859 if (len + iovs[x].iov_len >= UDP_MAX_PAYLOAD_SIZE) {
2860 iovs[x].iov_len = UDP_MAX_PAYLOAD_SIZE - len;
2861 x++;
2862 break;
2863 } else {
2864 len += iovs[x].iov_len;
2865 }
2866 }
2867 iovused = x;
2868 }
2869
2870 ssize_t res;
2871 msg.msg_iovlen = iovused;
2872 // NOTE: uses system sendmsg since we have no support for indirect UDP.
2873 res = sendmsg(c->sfd, &msg, 0);
2874 if (res >= 0) {
2875 pthread_mutex_lock(&c->thread->stats.mutex);
2876 c->thread->stats.bytes_written += res;
2877 pthread_mutex_unlock(&c->thread->stats.mutex);
2878
2879 // Ignore the header size from forwarding the IOV's
2880 res -= UDP_HEADER_SIZE;
2881
2882 // Decrement any partial IOV's and complete any finished resp's.
2883 _transmit_post(c, res);
2884
2885 if (c->resp_head) {
2886 return TRANSMIT_INCOMPLETE;
2887 } else {
2888 return TRANSMIT_COMPLETE;
2889 }
2890 }
2891
2892 if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
2893 if (!update_event(c, EV_WRITE | EV_PERSIST)) {
2894 if (settings.verbose > 0)
2895 fprintf(stderr, "Couldn't update event\n");
2896 conn_set_state(c, conn_closing);
2897 return TRANSMIT_HARD_ERROR;
2898 }
2899 return TRANSMIT_SOFT_ERROR;
2900 }
2901 /* if res == -1 and error is not EAGAIN or EWOULDBLOCK,
2902 we have a real error, on which we close the connection */
2903 if (settings.verbose > 0)
2904 perror("Failed to write, and not due to blocking");
2905
2906 conn_set_state(c, conn_read);
2907 return TRANSMIT_HARD_ERROR;
2908 }
2909
2910
2911 /* Does a looped read to fill data chunks */
2912 /* TODO: restrict number of times this can loop.
2913 * Also, benchmark using readv's.
2914 */
2915 static int read_into_chunked_item(conn *c) {
2916 int total = 0;
2917 int res;
2918 assert(c->rcurr != c->ritem);
2919
2920 while (c->rlbytes > 0) {
2921 item_chunk *ch = (item_chunk *)c->ritem;
2922 if (ch->size == ch->used) {
2923 // FIXME: ch->next is currently always 0. remove this?
2924 if (ch->next) {
2925 c->ritem = (char *) ch->next;
2926 } else {
2927 /* Allocate next chunk. Binary protocol needs 2b for \r\n */
2928 c->ritem = (char *) do_item_alloc_chunk(ch, c->rlbytes +
2929 ((c->protocol == binary_prot) ? 2 : 0));
2930 if (!c->ritem) {
2931 // We failed an allocation. Let caller handle cleanup.
2932 total = -2;
2933 break;
2934 }
2935 // ritem has new chunk, restart the loop.
2936 continue;
2937 //assert(c->rlbytes == 0);
2938 }
2939 }
2940
2941 int unused = ch->size - ch->used;
2942 /* first check if we have leftovers in the conn_read buffer */
2943 if (c->rbytes > 0) {
2944 total = 0;
2945 int tocopy = c->rbytes > c->rlbytes ? c->rlbytes : c->rbytes;
2946 tocopy = tocopy > unused ? unused : tocopy;
2947 if (c->ritem != c->rcurr) {
2948 memmove(ch->data + ch->used, c->rcurr, tocopy);
2949 }
2950 total += tocopy;
2951 c->rlbytes -= tocopy;
2952 c->rcurr += tocopy;
2953 c->rbytes -= tocopy;
2954 ch->used += tocopy;
2955 if (c->rlbytes == 0) {
2956 break;
2957 }
2958 } else {
2959 /* now try reading from the socket */
2960 res = c->read(c, ch->data + ch->used,
2961 (unused > c->rlbytes ? c->rlbytes : unused));
2962 if (res > 0) {
2963 pthread_mutex_lock(&c->thread->stats.mutex);
2964 c->thread->stats.bytes_read += res;
2965 pthread_mutex_unlock(&c->thread->stats.mutex);
2966 ch->used += res;
2967 total += res;
2968 c->rlbytes -= res;
2969 } else {
2970 /* Reset total to the latest result so caller can handle it */
2971 total = res;
2972 break;
2973 }
2974 }
2975 }
2976
2977 /* At some point I will be able to ditch the \r\n from item storage and
2978 remove all of these kludges.
2979 The above binprot check ensures inline space for \r\n, but if we do
2980 exactly enough allocs there will be no additional chunk for \r\n.
2981 */
2982 if (c->rlbytes == 0 && c->protocol == binary_prot && total >= 0) {
2983 item_chunk *ch = (item_chunk *)c->ritem;
2984 if (ch->size - ch->used < 2) {
2985 c->ritem = (char *) do_item_alloc_chunk(ch, 2);
2986 if (!c->ritem) {
2987 total = -2;
2988 }
2989 }
2990 }
2991 return total;
2992 }
2993
2994 static void drive_machine(conn *c) {
2995 bool stop = false;
2996 int sfd;
2997 socklen_t addrlen;
2998 struct sockaddr_storage addr;
2999 int nreqs = settings.reqs_per_event;
3000 int res;
3001 const char *str;
3002 #ifdef HAVE_ACCEPT4
3003 static int use_accept4 = 1;
3004 #else
3005 static int use_accept4 = 0;
3006 #endif
3007
3008 assert(c != NULL);
3009
3010 while (!stop) {
3011
3012 switch(c->state) {
3013 case conn_listening:
3014 addrlen = sizeof(addr);
3015 #ifdef HAVE_ACCEPT4
3016 if (use_accept4) {
3017 sfd = accept4(c->sfd, (struct sockaddr *)&addr, &addrlen, SOCK_NONBLOCK);
3018 } else {
3019 sfd = accept(c->sfd, (struct sockaddr *)&addr, &addrlen);
3020 }
3021 #else
3022 sfd = accept(c->sfd, (struct sockaddr *)&addr, &addrlen);
3023 #endif
3024 if (sfd == -1) {
3025 if (use_accept4 && errno == ENOSYS) {
3026 use_accept4 = 0;
3027 continue;
3028 }
3029 perror(use_accept4 ? "accept4()" : "accept()");
3030 if (errno == EAGAIN || errno == EWOULDBLOCK) {
3031 /* these are transient, so don't log anything */
3032 stop = true;
3033 } else if (errno == EMFILE) {
3034 if (settings.verbose > 0)
3035 fprintf(stderr, "Too many open connections\n");
3036 accept_new_conns(false);
3037 stop = true;
3038 } else {
3039 perror("accept()");
3040 stop = true;
3041 }
3042 break;
3043 }
3044 if (!use_accept4) {
3045 if (fcntl(sfd, F_SETFL, fcntl(sfd, F_GETFL) | O_NONBLOCK) < 0) {
3046 perror("setting O_NONBLOCK");
3047 close(sfd);
3048 break;
3049 }
3050 }
3051
3052 bool reject;
3053 if (settings.maxconns_fast) {
3054 reject = sfd >= settings.maxconns - 1;
3055 if (reject) {
3056 STATS_LOCK();
3057 stats.rejected_conns++;
3058 STATS_UNLOCK();
3059 }
3060 } else {
3061 reject = false;
3062 }
3063
3064 if (reject) {
3065 str = "ERROR Too many open connections\r\n";
3066 res = write(sfd, str, strlen(str));
3067 close(sfd);
3068 } else {
3069 // run accept routine if ssl is compiled + enabled
3070 bool fail = false;
3071 void *ssl_v = ssl_accept(c, sfd, &fail);
3072 if (fail) {
3073 close(sfd);
3074 break;
3075 }
3076
3077 dispatch_conn_new(sfd, conn_new_cmd, EV_READ | EV_PERSIST,
3078 READ_BUFFER_CACHED, c->transport, ssl_v, c->tag, c->protocol);
3079 }
3080
3081 stop = true;
3082 break;
3083
3084 case conn_waiting:
3085 rbuf_release(c);
3086 if (!update_event(c, EV_READ | EV_PERSIST)) {
3087 if (settings.verbose > 0)
3088 fprintf(stderr, "Couldn't update event\n");
3089 conn_set_state(c, conn_closing);
3090 break;
3091 }
3092
3093 conn_set_state(c, conn_read);
3094 stop = true;
3095 break;
3096
3097 case conn_read:
3098 if (!IS_UDP(c->transport)) {
3099 // Assign a read buffer if necessary.
3100 if (!rbuf_alloc(c)) {
3101 // TODO: Some way to allow for temporary failures.
3102 conn_set_state(c, conn_closing);
3103 break;
3104 }
3105 res = try_read_network(c);
3106 } else {
3107 // UDP connections always have a static buffer.
3108 res = try_read_udp(c);
3109 }
3110
3111 switch (res) {
3112 case READ_NO_DATA_RECEIVED:
3113 conn_set_state(c, conn_waiting);
3114 break;
3115 case READ_DATA_RECEIVED:
3116 conn_set_state(c, conn_parse_cmd);
3117 break;
3118 case READ_ERROR:
3119 conn_set_state(c, conn_closing);
3120 break;
3121 case READ_MEMORY_ERROR: /* Failed to allocate more memory */
3122 /* State already set by try_read_network */
3123 break;
3124 }
3125 break;
3126
3127 case conn_parse_cmd:
3128 c->noreply = false;
3129 if (c->try_read_command(c) == 0) {
3130 /* we need more data! */
3131 if (c->resp_head) {
3132 // Buffered responses waiting, flush in the meantime.
3133 conn_set_state(c, conn_mwrite);
3134 } else {
3135 conn_set_state(c, conn_waiting);
3136 }
3137 }
3138
3139 break;
3140
3141 case conn_new_cmd:
3142 /* Only process nreqs at a time to avoid starving other
3143 connections */
3144
3145 --nreqs;
3146 if (nreqs >= 0) {
3147 reset_cmd_handler(c);
3148 } else if (c->resp_head) {
3149 // flush response pipe on yield.
3150 conn_set_state(c, conn_mwrite);
3151 } else {
3152 pthread_mutex_lock(&c->thread->stats.mutex);
3153 c->thread->stats.conn_yields++;
3154 pthread_mutex_unlock(&c->thread->stats.mutex);
3155 if (c->rbytes > 0) {
3156 /* We have already read in data into the input buffer,
3157 so libevent will most likely not signal read events
3158 on the socket (unless more data is available. As a
3159 hack we should just put in a request to write data,
3160 because that should be possible ;-)
3161 */
3162 if (!update_event(c, EV_WRITE | EV_PERSIST)) {
3163 if (settings.verbose > 0)
3164 fprintf(stderr, "Couldn't update event\n");
3165 conn_set_state(c, conn_closing);
3166 break;
3167 }
3168 }
3169 stop = true;
3170 }
3171 break;
3172
3173 case conn_nread:
3174 if (c->rlbytes == 0) {
3175 complete_nread(c);
3176 break;
3177 }
3178
3179 /* Check if rbytes < 0, to prevent crash */
3180 if (c->rlbytes < 0) {
3181 if (settings.verbose) {
3182 fprintf(stderr, "Invalid rlbytes to read: len %d\n", c->rlbytes);
3183 }
3184 conn_set_state(c, conn_closing);
3185 break;
3186 }
3187
3188 if (c->item_malloced || ((((item *)c->item)->it_flags & ITEM_CHUNKED) == 0) ) {
3189 /* first check if we have leftovers in the conn_read buffer */
3190 if (c->rbytes > 0) {
3191 int tocopy = c->rbytes > c->rlbytes ? c->rlbytes : c->rbytes;
3192 memmove(c->ritem, c->rcurr, tocopy);
3193 c->ritem += tocopy;
3194 c->rlbytes -= tocopy;
3195 c->rcurr += tocopy;
3196 c->rbytes -= tocopy;
3197 if (c->rlbytes == 0) {
3198 break;
3199 }
3200 }
3201
3202 /* now try reading from the socket */
3203 res = c->read(c, c->ritem, c->rlbytes);
3204 if (res > 0) {
3205 pthread_mutex_lock(&c->thread->stats.mutex);
3206 c->thread->stats.bytes_read += res;
3207 pthread_mutex_unlock(&c->thread->stats.mutex);
3208 if (c->rcurr == c->ritem) {
3209 c->rcurr += res;
3210 }
3211 c->ritem += res;
3212 c->rlbytes -= res;
3213 break;
3214 }
3215 } else {
3216 res = read_into_chunked_item(c);
3217 if (res > 0)
3218 break;
3219 }
3220
3221 if (res == 0) { /* end of stream */
3222 c->close_reason = NORMAL_CLOSE;
3223 conn_set_state(c, conn_closing);
3224 break;
3225 }
3226
3227 if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
3228 if (!update_event(c, EV_READ | EV_PERSIST)) {
3229 if (settings.verbose > 0)
3230 fprintf(stderr, "Couldn't update event\n");
3231 conn_set_state(c, conn_closing);
3232 break;
3233 }
3234 stop = true;
3235 break;
3236 }
3237
3238 /* Memory allocation failure */
3239 if (res == -2) {
3240 out_of_memory(c, "SERVER_ERROR Out of memory during read");
3241 c->sbytes = c->rlbytes;
3242 conn_set_state(c, conn_swallow);
3243 // Ensure this flag gets cleared. It gets killed on conn_new()
3244 // so any conn_closing is fine, calling complete_nread is
3245 // fine. This swallow semms to be the only other case.
3246 c->set_stale = false;
3247 c->mset_res = false;
3248 break;
3249 }
3250 /* otherwise we have a real error, on which we close the connection */
3251 if (settings.verbose > 0) {
3252 fprintf(stderr, "Failed to read, and not due to blocking:\n"
3253 "errno: %d %s \n"
3254 "rcurr=%p ritem=%p rbuf=%p rlbytes=%d rsize=%d\n",
3255 errno, strerror(errno),
3256 (void *)c->rcurr, (void *)c->ritem, (void *)c->rbuf,
3257 (int)c->rlbytes, (int)c->rsize);
3258 }
3259 conn_set_state(c, conn_closing);
3260 break;
3261
3262 case conn_swallow:
3263 /* we are reading sbytes and throwing them away */
3264 if (c->sbytes <= 0) {
3265 conn_set_state(c, conn_new_cmd);
3266 break;
3267 }
3268
3269 /* first check if we have leftovers in the conn_read buffer */
3270 if (c->rbytes > 0) {
3271 int tocopy = c->rbytes > c->sbytes ? c->sbytes : c->rbytes;
3272 c->sbytes -= tocopy;
3273 c->rcurr += tocopy;
3274 c->rbytes -= tocopy;
3275 break;
3276 }
3277
3278 /* now try reading from the socket */
3279 res = c->read(c, c->rbuf, c->rsize > c->sbytes ? c->sbytes : c->rsize);
3280 if (res > 0) {
3281 pthread_mutex_lock(&c->thread->stats.mutex);
3282 c->thread->stats.bytes_read += res;
3283 pthread_mutex_unlock(&c->thread->stats.mutex);
3284 c->sbytes -= res;
3285 break;
3286 }
3287 if (res == 0) { /* end of stream */
3288 c->close_reason = NORMAL_CLOSE;
3289 conn_set_state(c, conn_closing);
3290 break;
3291 }
3292 if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
3293 if (!update_event(c, EV_READ | EV_PERSIST)) {
3294 if (settings.verbose > 0)
3295 fprintf(stderr, "Couldn't update event\n");
3296 conn_set_state(c, conn_closing);
3297 break;
3298 }
3299 stop = true;
3300 break;
3301 }
3302 /* otherwise we have a real error, on which we close the connection */
3303 if (settings.verbose > 0)
3304 fprintf(stderr, "Failed to read, and not due to blocking\n");
3305 conn_set_state(c, conn_closing);
3306 break;
3307
3308 case conn_write:
3309 case conn_mwrite:
3310 /* have side IO's that must process before transmit() can run.
3311 * remove the connection from the worker thread and dispatch the
3312 * IO queue
3313 */
3314 assert(c->io_queues_submitted == 0);
3315
3316 for (io_queue_t *q = c->io_queues; q->type != IO_QUEUE_NONE; q++) {
3317 if (q->stack_ctx != NULL) {
3318 io_queue_cb_t *qcb = thread_io_queue_get(c->thread, q->type);
3319 qcb->submit_cb(q);
3320 c->io_queues_submitted++;
3321 }
3322 }
3323 if (c->io_queues_submitted != 0) {
3324 conn_set_state(c, conn_io_queue);
3325
3326 stop = true;
3327 break;
3328 }
3329
3330 switch (!IS_UDP(c->transport) ? transmit(c) : transmit_udp(c)) {
3331 case TRANSMIT_COMPLETE:
3332 if (c->state == conn_mwrite) {
3333 // Free up IO wraps and any half-uploaded items.
3334 conn_release_items(c);
3335 conn_set_state(c, conn_new_cmd);
3336 if (c->close_after_write) {
3337 conn_set_state(c, conn_closing);
3338 }
3339 } else {
3340 if (settings.verbose > 0)
3341 fprintf(stderr, "Unexpected state %d\n", c->state);
3342 conn_set_state(c, conn_closing);
3343 }
3344 break;
3345
3346 case TRANSMIT_INCOMPLETE:
3347 case TRANSMIT_HARD_ERROR:
3348 break; /* Continue in state machine. */
3349
3350 case TRANSMIT_SOFT_ERROR:
3351 stop = true;
3352 break;
3353 }
3354 break;
3355
3356 case conn_closing:
3357 if (IS_UDP(c->transport))
3358 conn_cleanup(c);
3359 else
3360 conn_close(c);
3361 stop = true;
3362 break;
3363
3364 case conn_closed:
3365 /* This only happens if dormando is an idiot. */
3366 abort();
3367 break;
3368
3369 case conn_watch:
3370 /* We handed off our connection to the logger thread. */
3371 stop = true;
3372 break;
3373 case conn_io_queue:
3374 /* Woke up while waiting for an async return, but not ready. */
3375 event_del(&c->event);
3376 conn_set_state(c, conn_io_pending);
3377 stop = true;
3378 break;
3379 case conn_io_pending:
3380 /* Should not be reachable */
3381 assert(false);
3382 break;
3383 case conn_io_resume:
3384 /* Complete our queued IO's from within the worker thread. */
3385 conn_set_state(c, conn_mwrite);
3386 break;
3387 case conn_max_state:
3388 assert(false);
3389 break;
3390 }
3391 }
3392
3393 return;
3394 }
3395
3396 void event_handler(const evutil_socket_t fd, const short which, void *arg) {
3397 conn *c;
3398
3399 c = (conn *)arg;
3400 assert(c != NULL);
3401
3402 c->which = which;
3403
3404 /* sanity */
3405 if (fd != c->sfd) {
3406 if (settings.verbose > 0)
3407 fprintf(stderr, "Catastrophic: event fd doesn't match conn fd!\n");
3408 conn_close(c);
3409 return;
3410 }
3411
3412 drive_machine(c);
3413
3414 /* wait for next event */
3415 return;
3416 }
3417
3418 static int new_socket(struct addrinfo *ai) {
3419 int sfd;
3420 int flags;
3421
3422 if ((sfd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol)) == -1) {
3423 return -1;
3424 }
3425
3426 if ((flags = fcntl(sfd, F_GETFL, 0)) < 0 ||
3427 fcntl(sfd, F_SETFL, flags | O_NONBLOCK) < 0) {
3428 perror("setting O_NONBLOCK");
3429 close(sfd);
3430 return -1;
3431 }
3432 return sfd;
3433 }
3434
3435
3436 /*
3437 * Sets a socket's send buffer size to the maximum allowed by the system.
3438 */
3439 static void maximize_sndbuf(const int sfd) {
3440 socklen_t intsize = sizeof(int);
3441 int last_good = 0;
3442 int min, max, avg;
3443 int old_size;
3444
3445 /* Start with the default size. */
3446 #ifdef _WIN32
3447 if (getsockopt((SOCKET)sfd, SOL_SOCKET, SO_SNDBUF, (char *)&old_size, &intsize) != 0) {
3448 #else
3449 if (getsockopt(sfd, SOL_SOCKET, SO_SNDBUF, &old_size, &intsize) != 0) {
3450 #endif /* #ifdef _WIN32 */
3451 if (settings.verbose > 0)
3452 perror("getsockopt(SO_SNDBUF)");
3453 return;
3454 }
3455
3456 /* Binary-search for the real maximum. */
3457 min = old_size;
3458 max = MAX_SENDBUF_SIZE;
3459
3460 while (min <= max) {
3461 avg = ((unsigned int)(min + max)) / 2;
3462 if (setsockopt(sfd, SOL_SOCKET, SO_SNDBUF, (void *)&avg, intsize) == 0) {
3463 last_good = avg;
3464 min = avg + 1;
3465 } else {
3466 max = avg - 1;
3467 }
3468 }
3469
3470 if (settings.verbose > 1)
3471 fprintf(stderr, "<%d send buffer was %d, now %d\n", sfd, old_size, last_good);
3472 }
3473
3474 /**
3475 * Create a socket and bind it to a specific port number
3476 * @param interface the interface to bind to
3477 * @param port the port number to bind to
3478 * @param transport the transport protocol (TCP / UDP)
3479 * @param portnumber_file A filepointer to write the port numbers to
3480 * when they are successfully added to the list of ports we
3481 * listen on.
3482 */
3483 static int server_socket(const char *interface,
3484 int port,
3485 enum network_transport transport,
3486 FILE *portnumber_file, uint8_t ssl_enabled,
3487 uint64_t conntag,
3488 enum protocol bproto) {
3489 int sfd;
3490 struct linger ling = {0, 0};
3491 struct addrinfo *ai;
3492 struct addrinfo *next;
3493 struct addrinfo hints = { .ai_flags = AI_PASSIVE,
3494 .ai_family = AF_UNSPEC };
3495 char port_buf[NI_MAXSERV];
3496 int error;
3497 int success = 0;
3498 int flags =1;
3499
3500 hints.ai_socktype = IS_UDP(transport) ? SOCK_DGRAM : SOCK_STREAM;
3501
3502 if (port == -1) {
3503 port = 0;
3504 }
3505 snprintf(port_buf, sizeof(port_buf), "%d", port);
3506 error= getaddrinfo(interface, port_buf, &hints, &ai);
3507 if (error != 0) {
3508 if (error != EAI_SYSTEM)
3509 fprintf(stderr, "getaddrinfo(): %s\n", gai_strerror(error));
3510 else
3511 perror("getaddrinfo()");
3512 return 1;
3513 }
3514
3515 for (next= ai; next; next= next->ai_next) {
3516 conn *listen_conn_add;
3517 if ((sfd = new_socket(next)) == -1) {
3518 /* getaddrinfo can return "junk" addresses,
3519 * we make sure at least one works before erroring.
3520 */
3521 if (errno == EMFILE) {
3522 /* ...unless we're out of fds */
3523 perror("server_socket");
3524 exit(EX_OSERR);
3525 }
3526 continue;
3527 }
3528
3529 if (settings.num_napi_ids) {
3530 socklen_t len = sizeof(socklen_t);
3531 int napi_id;
3532 error = getsockopt(sfd, SOL_SOCKET, SO_INCOMING_NAPI_ID, &napi_id, &len);
3533 if (error != 0) {
3534 fprintf(stderr, "-N <num_napi_ids> option not supported\n");
3535 exit(EXIT_FAILURE);
3536 }
3537 }
3538
3539 #ifdef IPV6_V6ONLY
3540 if (next->ai_family == AF_INET6) {
3541 error = setsockopt(sfd, IPPROTO_IPV6, IPV6_V6ONLY, (char *) &flags, sizeof(flags));
3542 if (error != 0) {
3543 perror("setsockopt");
3544 close(sfd);
3545 continue;
3546 }
3547 }
3548 #endif
3549 #ifdef SOCK_COOKIE_ID
3550 if (settings.sock_cookie_id != 0) {
3551 error = setsockopt(sfd, SOL_SOCKET, SOCK_COOKIE_ID, (void *)&settings.sock_cookie_id, sizeof(uint32_t));
3552 if (error != 0)
3553 perror("setsockopt");
3554 }
3555 #endif
3556
3557 setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (void *)&flags, sizeof(flags));
3558 if (IS_UDP(transport)) {
3559 maximize_sndbuf(sfd);
3560 } else {
3561 error = setsockopt(sfd, SOL_SOCKET, SO_KEEPALIVE, (void *)&flags, sizeof(flags));
3562 if (error != 0)
3563 perror("setsockopt");
3564
3565 error = setsockopt(sfd, SOL_SOCKET, SO_LINGER, (void *)&ling, sizeof(ling));
3566 if (error != 0)
3567 perror("setsockopt");
3568
3569 error = setsockopt(sfd, IPPROTO_TCP, TCP_NODELAY, (void *)&flags, sizeof(flags));
3570 if (error != 0)
3571 perror("setsockopt");
3572 }
3573
3574 if (bind(sfd, next->ai_addr, next->ai_addrlen) == -1) {
3575 if (errno != EADDRINUSE) {
3576 perror("bind()");
3577 close(sfd);
3578 freeaddrinfo(ai);
3579 return 1;
3580 }
3581 close(sfd);
3582 continue;
3583 } else {
3584 success++;
3585 if (!IS_UDP(transport) && listen(sfd, settings.backlog) == -1) {
3586 perror("listen()");
3587 close(sfd);
3588 freeaddrinfo(ai);
3589 return 1;
3590 }
3591 if (portnumber_file != NULL &&
3592 (next->ai_addr->sa_family == AF_INET ||
3593 next->ai_addr->sa_family == AF_INET6)) {
3594 union {
3595 struct sockaddr_in in;
3596 struct sockaddr_in6 in6;
3597 } my_sockaddr;
3598 socklen_t len = sizeof(my_sockaddr);
3599 if (getsockname(sfd, (struct sockaddr*)&my_sockaddr, &len)==0) {
3600 if (next->ai_addr->sa_family == AF_INET) {
3601 fprintf(portnumber_file, "%s INET: %u\n",
3602 IS_UDP(transport) ? "UDP" : "TCP",
3603 ntohs(my_sockaddr.in.sin_port));
3604 } else {
3605 fprintf(portnumber_file, "%s INET6: %u\n",
3606 IS_UDP(transport) ? "UDP" : "TCP",
3607 ntohs(my_sockaddr.in6.sin6_port));
3608 }
3609 }
3610 }
3611 }
3612
3613 if (IS_UDP(transport)) {
3614 int c;
3615
3616 for (c = 0; c < settings.num_threads_per_udp; c++) {
3617 /* Allocate one UDP file descriptor per worker thread;
3618 * this allows "stats conns" to separately list multiple
3619 * parallel UDP requests in progress.
3620 *
3621 * The dispatch code round-robins new connection requests
3622 * among threads, so this is guaranteed to assign one
3623 * FD to each thread.
3624 */
3625 int per_thread_fd;
3626 if (c == 0) {
3627 per_thread_fd = sfd;
3628 } else {
3629 per_thread_fd = dup(sfd);
3630 if (per_thread_fd < 0) {
3631 perror("Failed to duplicate file descriptor");
3632 exit(EXIT_FAILURE);
3633 }
3634 }
3635 dispatch_conn_new(per_thread_fd, conn_read,
3636 EV_READ | EV_PERSIST,
3637 UDP_READ_BUFFER_SIZE, transport, NULL, conntag, bproto);
3638 }
3639 } else {
3640 if (!(listen_conn_add = conn_new(sfd, conn_listening,
3641 EV_READ | EV_PERSIST, 1,
3642 transport, main_base, NULL, conntag, bproto))) {
3643 fprintf(stderr, "failed to create listening connection\n");
3644 exit(EXIT_FAILURE);
3645 }
3646 #ifdef TLS
3647 listen_conn_add->ssl_enabled = ssl_enabled;
3648 #else
3649 assert(ssl_enabled == false);
3650 #endif
3651 listen_conn_add->next = listen_conn;
3652 listen_conn = listen_conn_add;
3653 }
3654 }
3655
3656 freeaddrinfo(ai);
3657
3658 /* Return zero iff we detected no errors in starting up connections */
3659 return success == 0;
3660 }
3661
3662 static int server_sockets(int port, enum network_transport transport,
3663 FILE *portnumber_file) {
3664 uint8_t ssl_enabled = MC_SSL_DISABLED;
3665
3666 const char *notls = "notls";
3667 const char *btls = "btls";
3668 const char *mtls = "mtls";
3669 if (settings.ssl_enabled) {
3670 ssl_enabled = MC_SSL_ENABLED_DEFAULT;
3671 }
3672
3673 if (settings.inter == NULL) {
3674 return server_socket(settings.inter, port, transport, portnumber_file, ssl_enabled, 0, settings.binding_protocol);
3675 } else {
3676 // tokenize them and bind to each one of them..
3677 char *b;
3678 int ret = 0;
3679 char *list = strdup(settings.inter);
3680
3681 if (list == NULL) {
3682 fprintf(stderr, "Failed to allocate memory for parsing server interface string\n");
3683 return 1;
3684 }
3685 // If we encounter any failure, preserve the first errno for the caller.
3686 int errno_save = 0;
3687 for (char *p = strtok_r(list, ";,", &b);
3688 p != NULL;
3689 p = strtok_r(NULL, ";,", &b)) {
3690 uint64_t conntag = 0;
3691 int the_port = port;
3692 if (settings.ssl_enabled) {
3693 ssl_enabled = MC_SSL_ENABLED_DEFAULT;
3694 }
3695 // "notls" option is valid only when memcached is run with SSL enabled.
3696 if (strncmp(p, notls, strlen(notls)) == 0) {
3697 if (!settings.ssl_enabled) {
3698 fprintf(stderr, "'notls' option is valid only when SSL is enabled\n");
3699 free(list);
3700 return 1;
3701 }
3702 ssl_enabled = MC_SSL_DISABLED;
3703 p += strlen(notls) + 1;
3704 } else if (strncmp(p, btls, strlen(btls)) == 0) {
3705 if (!settings.ssl_enabled) {
3706 fprintf(stderr, "'btls' option is valid only when SSL is enabled\n");
3707 free(list);
3708 return 1;
3709 }
3710 ssl_enabled = MC_SSL_ENABLED_NOPEER;
3711 p += strlen(btls) + 1;
3712 } else if (strncmp(p, mtls, strlen(mtls)) == 0) {
3713 if (!settings.ssl_enabled) {
3714 fprintf(stderr, "'otls' option is valid only when SSL is enabled\n");
3715 free(list);
3716 return 1;
3717 }
3718 ssl_enabled = MC_SSL_ENABLED_PEER;
3719 p += strlen(mtls) + 1;
3720 }
3721
3722 // Allow forcing the protocol of this listener.
3723 const char *protostr = "proto";
3724 enum protocol bproto = settings.binding_protocol;
3725 if (strncmp(p, protostr, strlen(protostr)) == 0) {
3726 p += strlen(protostr);
3727 if (*p == '[') {
3728 char *e = strchr(p, ']');
3729 if (e == NULL) {
3730 fprintf(stderr, "Invalid protocol spec: \"%s\"\n", p);
3731 free(list);
3732 return 1;
3733 }
3734 char *st = ++p; // skip '[';
3735 *e = '\0';
3736 size_t len = e - st;
3737 p = ++e; // skip ']'
3738 p++; // skip an assumed ':'
3739
3740 if (strncmp(st, "ascii", len) == 0) {
3741 bproto = ascii_prot;
3742 } else if (strncmp(st, "binary", len) == 0) {
3743 bproto = binary_prot;
3744 } else if (strncmp(st, "negotiating", len) == 0) {
3745 bproto = negotiating_prot;
3746 } else if (strncmp(st, "proxy", len) == 0) {
3747 #ifdef PROXY
3748 if (settings.proxy_enabled) {
3749 bproto = proxy_prot;
3750 } else {
3751 fprintf(stderr, "Proxy must be enabled to use: \"%s\"\n", list);
3752 free(list);
3753 return 1;
3754 }
3755 #else
3756 fprintf(stderr, "Server not built with proxy: \"%s\"\n", list);
3757 free(list);
3758 return 1;
3759 #endif
3760 }
3761 }
3762 }
3763
3764 const char *tagstr = "tag";
3765 if (strncmp(p, tagstr, strlen(tagstr)) == 0) {
3766 p += strlen(tagstr);
3767 // NOTE: should probably retire the [] dumbassery. those're
3768 // shell characters.
3769 if (*p == '[' || *p == '_') {
3770 char *e = strchr(p, ']');
3771 if (e == NULL) {
3772 e = strchr(p+1, '_');
3773 }
3774 if (e == NULL) {
3775 fprintf(stderr, "Invalid tag in socket config: \"%s\"\n", p);
3776 free(list);
3777 return 1;
3778 }
3779 char *st = ++p; // skip '['
3780 *e = '\0';
3781 size_t len = e - st;
3782 p = ++e; // skip ']'
3783 p++; // skip an assumed ':'
3784
3785 // validate the tag and copy it in.
3786 if (len > 8 || len < 1) {
3787 fprintf(stderr, "Listener tags must be between 1 and 8 characters: \"%s\"\n", st);
3788 free(list);
3789 return 1;
3790 }
3791
3792 // C programmers love turning string comparisons into
3793 // integer comparisons.
3794 memcpy(&conntag, st, len);
3795 }
3796 }
3797
3798 char *h = NULL;
3799 if (*p == '[') {
3800 // expecting it to be an IPv6 address enclosed in []
3801 // i.e. RFC3986 style recommended by RFC5952
3802 char *e = strchr(p, ']');
3803 if (e == NULL) {
3804 fprintf(stderr, "Invalid IPV6 address: \"%s\"", p);
3805 free(list);
3806 return 1;
3807 }
3808 h = ++p; // skip the opening '['
3809 *e = '\0';
3810 p = ++e; // skip the closing ']'
3811 }
3812
3813 char *s = strchr(p, ':');
3814 if (s != NULL) {
3815 // If no more semicolons - attempt to treat as port number.
3816 // Otherwise the only valid option is an unenclosed IPv6 without port, until
3817 // of course there was an RFC3986 IPv6 address previously specified -
3818 // in such a case there is no good option, will just send it to fail as port number.
3819 if (strchr(s + 1, ':') == NULL || h != NULL) {
3820 *s = '\0';
3821 ++s;
3822 if (!safe_strtol(s, &the_port)) {
3823 fprintf(stderr, "Invalid port number: \"%s\"\n", s);
3824 free(list);
3825 return 1;
3826 }
3827 }
3828 }
3829
3830 if (h != NULL)
3831 p = h;
3832
3833 if (strcmp(p, "*") == 0) {
3834 p = NULL;
3835 }
3836 ret |= server_socket(p, the_port, transport, portnumber_file, ssl_enabled, conntag, bproto);
3837 if (ret != 0 && errno_save == 0) errno_save = errno;
3838 }
3839 free(list);
3840 errno = errno_save;
3841 return ret;
3842 }
3843 }
3844
3845 #ifndef DISABLE_UNIX_SOCKET
3846 static int new_socket_unix(void) {
3847 int sfd;
3848 int flags;
3849
3850 if ((sfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
3851 perror("socket()");
3852 return -1;
3853 }
3854
3855 if ((flags = fcntl(sfd, F_GETFL, 0)) < 0 ||
3856 fcntl(sfd, F_SETFL, flags | O_NONBLOCK) < 0) {
3857 perror("setting O_NONBLOCK");
3858 close(sfd);
3859 return -1;
3860 }
3861 return sfd;
3862 }
3863
3864 static int server_socket_unix(const char *path, int access_mask) {
3865 int sfd;
3866 struct linger ling = {0, 0};
3867 struct sockaddr_un addr;
3868 struct stat tstat;
3869 int flags =1;
3870 int old_umask;
3871
3872 if (!path) {
3873 return 1;
3874 }
3875
3876 if ((sfd = new_socket_unix()) == -1) {
3877 return 1;
3878 }
3879
3880 /*
3881 * Clean up a previous socket file if we left it around
3882 */
3883 if (lstat(path, &tstat) == 0) {
3884 if (S_ISSOCK(tstat.st_mode))
3885 unlink(path);
3886 }
3887
3888 setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (void *)&flags, sizeof(flags));
3889 setsockopt(sfd, SOL_SOCKET, SO_KEEPALIVE, (void *)&flags, sizeof(flags));
3890 setsockopt(sfd, SOL_SOCKET, SO_LINGER, (void *)&ling, sizeof(ling));
3891
3892 /*
3893 * the memset call clears nonstandard fields in some implementations
3894 * that otherwise mess things up.
3895 */
3896 memset(&addr, 0, sizeof(addr));
3897
3898 addr.sun_family = AF_UNIX;
3899 strncpy(addr.sun_path, path, sizeof(addr.sun_path) - 1);
3900 assert(strcmp(addr.sun_path, path) == 0);
3901 old_umask = umask( ~(access_mask&0777));
3902 if (bind(sfd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
3903 perror("bind()");
3904 close(sfd);
3905 umask(old_umask);
3906 return 1;
3907 }
3908 umask(old_umask);
3909 if (listen(sfd, settings.backlog) == -1) {
3910 perror("listen()");
3911 close(sfd);
3912 return 1;
3913 }
3914 if (!(listen_conn = conn_new(sfd, conn_listening,
3915 EV_READ | EV_PERSIST, 1,
3916 local_transport, main_base, NULL, 0, settings.binding_protocol))) {
3917 fprintf(stderr, "failed to create listening connection\n");
3918 exit(EXIT_FAILURE);
3919 }
3920
3921 return 0;
3922 }
3923 #else
3924 #define server_socket_unix(path, access_mask) -1
3925 #endif /* #ifndef DISABLE_UNIX_SOCKET */
3926
3927 /*
3928 * We keep the current time of day in a global variable that's updated by a
3929 * timer event. This saves us a bunch of time() system calls (we really only
3930 * need to get the time once a second, whereas there can be tens of thousands
3931 * of requests a second) and allows us to use server-start-relative timestamps
3932 * rather than absolute UNIX timestamps, a space savings on systems where
3933 * sizeof(time_t) > sizeof(unsigned int).
3934 */
3935 volatile rel_time_t current_time;
3936 static struct event clockevent;
3937 #ifdef MEMCACHED_DEBUG
3938 volatile bool is_paused;
3939 volatile int64_t delta;
3940 #endif
3941 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
3942 static bool monotonic = false;
3943 static int64_t monotonic_start;
3944 #endif
3945
3946 /* libevent uses a monotonic clock when available for event scheduling. Aside
3947 * from jitter, simply ticking our internal timer here is accurate enough.
3948 * Note that users who are setting explicit dates for expiration times *must*
3949 * ensure their clocks are correct before starting memcached. */
3950 static void clock_handler(const evutil_socket_t fd, const short which, void *arg) {
3951 struct timeval t = {.tv_sec = 1, .tv_usec = 0};
3952 static bool initialized = false;
3953
3954 if (initialized) {
3955 /* only delete the event if it's actually there. */
3956 evtimer_del(&clockevent);
3957 } else {
3958 initialized = true;
3959 }
3960
3961 // While we're here, check for hash table expansion.
3962 // This function should be quick to avoid delaying the timer.
3963 assoc_start_expand(stats_state.curr_items);
3964 // also, if HUP'ed we need to do some maintenance.
3965 // for now that's just the authfile reload.
3966 if (settings.sig_hup) {
3967 settings.sig_hup = false;
3968
3969 authfile_load(settings.auth_file);
3970 #ifdef PROXY
3971 if (settings.proxy_ctx) {
3972 proxy_start_reload(settings.proxy_ctx);
3973 }
3974 #endif
3975 }
3976
3977 evtimer_set(&clockevent, clock_handler, 0);
3978 event_base_set(main_base, &clockevent);
3979 evtimer_add(&clockevent, &t);
3980
3981 #ifdef MEMCACHED_DEBUG
3982 if (is_paused) return;
3983 #endif
3984
3985 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
3986 if (monotonic) {
3987 struct timespec ts;
3988 if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1)
3989 return;
3990 #ifdef MEMCACHED_DEBUG
3991 current_time = (rel_time_t) (ts.tv_sec - monotonic_start + delta);
3992 #else
3993 current_time = (rel_time_t) (ts.tv_sec - monotonic_start);
3994 #endif
3995 return;
3996 }
3997 #endif
3998 {
3999 struct timeval tv;
4000 gettimeofday(&tv, NULL);
4001 #ifdef MEMCACHED_DEBUG
4002 current_time = (rel_time_t) (tv.tv_sec - process_started + delta);
4003 #else
4004 current_time = (rel_time_t) (tv.tv_sec - process_started);
4005 #endif
4006 }
4007 }
4008
4009 static const char* flag_enabled_disabled(bool flag) {
4010 return (flag ? "enabled" : "disabled");
4011 }
4012
4013 void verify_default(const char* param, bool condition) {
4014 if (!condition) {
4015 printf("Default value of [%s] has changed."
4016 " Modify the help text and default value check.\n", param);
4017 exit(EXIT_FAILURE);
4018 }
4019 }
4020
4021 static void usage(void) {
4022 printf(PACKAGE " " VERSION "\n");
4023 printf("-p, --port=<num> TCP port to listen on (default: %d)\n"
4024 "-U, --udp-port=<num> UDP port to listen on (default: %d, off)\n",
4025 settings.port, settings.udpport);
4026 #ifndef DISABLE_UNIX_SOCKET
4027 printf("-s, --unix-socket=<file> UNIX socket to listen on (disables network support)\n");
4028 printf("-a, --unix-mask=<mask> access mask for UNIX socket, in octal (default: %o)\n",
4029 settings.access);
4030 #endif /* #ifndef DISABLE_UNIX_SOCKET */
4031 printf("-A, --enable-shutdown enable ascii \"shutdown\" command\n");
4032 printf("-l, --listen=<addr> interface to listen on (default: INADDR_ANY)\n");
4033 #ifdef TLS
4034 printf(" if TLS/SSL is enabled, 'notls' prefix can be used to\n"
4035 " disable for specific listeners (-l notls:<ip>:<port>) \n");
4036 #endif
4037 printf("-d, --daemon run as a daemon\n"
4038 "-r, --enable-coredumps maximize core file limit\n"
4039 "-u, --user=<user> assume identity of <username> (only when run as root)\n"
4040 "-m, --memory-limit=<num> item memory in megabytes (default: %lu)\n"
4041 "-M, --disable-evictions return error on memory exhausted instead of evicting\n"
4042 "-c, --conn-limit=<num> max simultaneous connections (default: %d)\n"
4043 "-k, --lock-memory lock down all paged memory\n"
4044 "-v, --verbose verbose (print errors/warnings while in event loop)\n"
4045 "-vv very verbose (also print client commands/responses)\n"
4046 "-vvv extremely verbose (internal state transitions)\n"
4047 "-h, --help print this help and exit\n"
4048 "-i, --license print memcached and libevent license\n"
4049 "-V, --version print version and exit\n"
4050 "-P, --pidfile=<file> save PID in <file>, only used with -d option\n"
4051 "-f, --slab-growth-factor=<num> chunk size growth factor (default: %2.2f)\n"
4052 "-n, --slab-min-size=<bytes> min space used for key+value+flags (default: %d)\n",
4053 (unsigned long) settings.maxbytes / (1 << 20),
4054 settings.maxconns, settings.factor, settings.chunk_size);
4055 verify_default("udp-port",settings.udpport == 0);
4056 printf("-L, --enable-largepages try to use large memory pages (if available)\n");
4057 printf("-D <char> Use <char> as the delimiter between key prefixes and IDs.\n"
4058 " This is used for per-prefix stats reporting. The default is\n"
4059 " \"%c\" (colon). If this option is specified, stats collection\n"
4060 " is turned on automatically; if not, then it may be turned on\n"
4061 " by sending the \"stats detail on\" command to the server.\n",
4062 settings.prefix_delimiter);
4063 printf("-t, --threads=<num> number of threads to use (default: %d)\n", settings.num_threads);
4064 printf("-R, --max-reqs-per-event maximum number of requests per event, limits the\n"
4065 " requests processed per connection to prevent \n"
4066 " starvation (default: %d)\n", settings.reqs_per_event);
4067 printf("-C, --disable-cas disable use of CAS\n");
4068 printf("-b, --listen-backlog=<num> set the backlog queue limit (default: %d)\n", settings.backlog);
4069 printf("-B, --protocol=<name> protocol - one of ascii, binary, or auto (default: %s)\n",
4070 prot_text(settings.binding_protocol));
4071 printf("-I, --max-item-size=<num> adjusts max item size\n"
4072 " (default: %dm, min: %dk, max: %dm)\n",
4073 settings.item_size_max/ (1 << 20), ITEM_SIZE_MAX_LOWER_LIMIT / (1 << 10), ITEM_SIZE_MAX_UPPER_LIMIT / (1 << 20));
4074 #ifdef ENABLE_SASL
4075 printf("-S, --enable-sasl turn on Sasl authentication\n");
4076 #endif
4077 printf("-F, --disable-flush-all disable flush_all command\n");
4078 printf("-X, --disable-dumping disable stats cachedump and lru_crawler metadump\n");
4079 printf("-W --disable-watch disable watch commands (live logging)\n");
4080 printf("-Y, --auth-file=<file> (EXPERIMENTAL) enable ASCII protocol authentication. format:\n"
4081 " user:pass\\nuser2:pass2\\n\n");
4082 printf("-e, --memory-file=<file> (EXPERIMENTAL) mmap a file for item memory.\n"
4083 " use only in ram disks or persistent memory mounts!\n"
4084 " enables restartable cache (stop with SIGUSR1)\n");
4085 #ifdef TLS
4086 printf("-Z, --enable-ssl enable TLS/SSL\n");
4087 #endif
4088 printf("-o, --extended comma separated list of extended options\n"
4089 " most options have a 'no_' prefix to disable\n"
4090 " - maxconns_fast: immediately close new connections after limit (default: %s)\n"
4091 " - hashpower: an integer multiplier for how large the hash\n"
4092 " table should be. normally grows at runtime. (default starts at: %d)\n"
4093 " set based on \"STAT hash_power_level\"\n"
4094 " - tail_repair_time: time in seconds for how long to wait before\n"
4095 " forcefully killing LRU tail item.\n"
4096 " disabled by default; very dangerous option.\n"
4097 " - hash_algorithm: the hash table algorithm\n"
4098 " default is murmur3 hash. options: jenkins, murmur3, xxh3\n"
4099 " - no_lru_crawler: disable LRU Crawler background thread.\n"
4100 " - lru_crawler_sleep: microseconds to sleep between items\n"
4101 " default is %d.\n"
4102 " - lru_crawler_tocrawl: max items to crawl per slab per run\n"
4103 " default is %u (unlimited)\n",
4104 flag_enabled_disabled(settings.maxconns_fast), settings.hashpower_init,
4105 settings.lru_crawler_sleep, settings.lru_crawler_tocrawl);
4106 printf(" - read_buf_mem_limit: limit in megabytes for connection read/response buffers.\n"
4107 " do not adjust unless you have high (20k+) conn. limits.\n"
4108 " 0 means unlimited (default: %u)\n",
4109 settings.read_buf_mem_limit);
4110 verify_default("read_buf_mem_limit", settings.read_buf_mem_limit == 0);
4111 printf(" - no_lru_maintainer: disable new LRU system + background thread.\n"
4112 " - hot_lru_pct: pct of slab memory to reserve for hot lru.\n"
4113 " (requires lru_maintainer, default pct: %d)\n"
4114 " - warm_lru_pct: pct of slab memory to reserve for warm lru.\n"
4115 " (requires lru_maintainer, default pct: %d)\n"
4116 " - hot_max_factor: items idle > cold lru age * drop from hot lru. (default: %.2f)\n"
4117 " - warm_max_factor: items idle > cold lru age * this drop from warm. (default: %.2f)\n"
4118 " - temporary_ttl: TTL's below get separate LRU, can't be evicted.\n"
4119 " (requires lru_maintainer, default: %d)\n"
4120 " - idle_timeout: timeout for idle connections. (default: %d, no timeout)\n",
4121 settings.hot_lru_pct, settings.warm_lru_pct, settings.hot_max_factor, settings.warm_max_factor,
4122 settings.temporary_ttl, settings.idle_timeout);
4123 printf(" - slab_chunk_max: (EXPERIMENTAL) maximum slab size in kilobytes. use extreme care. (default: %d)\n"
4124 " - watcher_logbuf_size: size in kilobytes of per-watcher write buffer. (default: %u)\n"
4125 " - worker_logbuf_size: size in kilobytes of per-worker-thread buffer\n"
4126 " read by background thread, then written to watchers. (default: %u)\n"
4127 " - track_sizes: enable dynamic reports for 'stats sizes' command.\n"
4128 " note that counts for each size are approximate.\n"
4129 " - no_hashexpand: disables hash table expansion (dangerous)\n"
4130 " - modern: enables options which will be default in future.\n"
4131 " currently: nothing\n"
4132 " - no_modern: uses defaults of previous major version (1.4.x)\n",
4133 settings.slab_chunk_size_max / (1 << 10), settings.logger_watcher_buf_size / (1 << 10),
4134 settings.logger_buf_size / (1 << 10));
4135 verify_default("tail_repair_time", settings.tail_repair_time == TAIL_REPAIR_TIME_DEFAULT);
4136 verify_default("lru_crawler_tocrawl", settings.lru_crawler_tocrawl == 0);
4137 verify_default("idle_timeout", settings.idle_timeout == 0);
4138 #ifdef HAVE_DROP_PRIVILEGES
4139 printf(" - drop_privileges: enable dropping extra syscall privileges\n"
4140 " - no_drop_privileges: disable drop_privileges in case it causes issues with\n"
4141 " some customisation.\n"
4142 " (default is no_drop_privileges)\n");
4143 verify_default("drop_privileges", !settings.drop_privileges);
4144 #ifdef MEMCACHED_DEBUG
4145 printf(" - relaxed_privileges: running tests requires extra privileges. (default: %s)\n",
4146 flag_enabled_disabled(settings.relaxed_privileges));
4147 #endif
4148 #endif
4149 #ifdef SOCK_COOKIE_ID
4150 printf(" - sock_cookie_id: attributes an ID to a socket for ip filtering/firewalls \n");
4151 #endif
4152 #ifdef EXTSTORE
4153 printf("\n - External storage (ext_*) related options (see: https://memcached.org/extstore)\n");
4154 printf(" - ext_path: file to write to for external storage.\n"
4155 " ie: ext_path=/mnt/d1/extstore:1G\n"
4156 " - ext_page_size: size in megabytes of storage pages. (default: %u)\n"
4157 " - ext_wbuf_size: size in megabytes of page write buffers. (default: %u)\n"
4158 " - ext_threads: number of IO threads to run. (default: %u)\n"
4159 " - ext_item_size: store items larger than this (bytes, default %u)\n"
4160 " - ext_item_age: store items idle at least this long (seconds, default: no age limit)\n"
4161 " - ext_low_ttl: consider TTLs lower than this specially (default: %u)\n"
4162 " - ext_drop_unread: don't re-write unread values during compaction (default: %s)\n"
4163 " - ext_recache_rate: recache an item every N accesses (default: %u)\n"
4164 " - ext_compact_under: compact when fewer than this many free pages\n"
4165 " (default: 1 percent of the assigned storage)\n"
4166 " - ext_drop_under: drop COLD items when fewer than this many free pages\n"
4167 " (default: 1/4th of the assigned storage)\n"
4168 " - ext_max_frag: only defrag pages if they are less full than this pct-wise (default: %.2f)\n"
4169 " - ext_max_sleep: max sleep time of background threads in us (default: %u)\n"
4170 " - slab_automove_freeratio: ratio of memory to hold free as buffer.\n"
4171 " (see doc/storage.txt for more info, default: %.3f)\n",
4172 settings.ext_page_size / (1 << 20), settings.ext_wbuf_size / (1 << 20), settings.ext_io_threadcount,
4173 settings.ext_item_size, settings.ext_low_ttl,
4174 flag_enabled_disabled(settings.ext_drop_unread), settings.ext_recache_rate,
4175 settings.ext_max_frag, settings.ext_max_sleep, settings.slab_automove_freeratio);
4176 verify_default("ext_item_age", settings.ext_item_age == UINT_MAX);
4177 #endif
4178 #ifdef PROXY
4179 printf(" - proxy_config: path to lua library file. separate with ':' for multiple files\n");
4180 printf(" - proxy_arg: string to pass to lua library\n");
4181 #endif
4182 ssl_help();
4183 printf("-N, --napi_ids number of napi ids. see doc/napi_ids.txt for more details\n");
4184 return;
4185 }
4186
4187 static void usage_license(void) {
4188 printf(PACKAGE " " VERSION "\n\n");
4189 printf(
4190 "Copyright (c) 2003, Danga Interactive, Inc. <http://www.danga.com/>\n"
4191 "All rights reserved.\n"
4192 "\n"
4193 "Redistribution and use in source and binary forms, with or without\n"
4194 "modification, are permitted provided that the following conditions are\n"
4195 "met:\n"
4196 "\n"
4197 " * Redistributions of source code must retain the above copyright\n"
4198 "notice, this list of conditions and the following disclaimer.\n"
4199 "\n"
4200 " * Redistributions in binary form must reproduce the above\n"
4201 "copyright notice, this list of conditions and the following disclaimer\n"
4202 "in the documentation and/or other materials provided with the\n"
4203 "distribution.\n"
4204 "\n"
4205 " * Neither the name of the Danga Interactive nor the names of its\n"
4206 "contributors may be used to endorse or promote products derived from\n"
4207 "this software without specific prior written permission.\n"
4208 "\n"
4209 "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n"
4210 "\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n"
4211 "LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n"
4212 "A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n"
4213 "OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n"
4214 "SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n"
4215 "LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n"
4216 "DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n"
4217 "THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n"
4218 "(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n"
4219 "OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n"
4220 );
4221
4222 return;
4223 }
4224
4225 static void save_pid(const char *pid_file) {
4226 FILE *fp;
4227 if (access(pid_file, F_OK) == 0) {
4228 if ((fp = fopen(pid_file, "r")) != NULL) {
4229 char buffer[1024];
4230 if (fgets(buffer, sizeof(buffer), fp) != NULL) {
4231 unsigned int pid;
4232 if (safe_strtoul(buffer, &pid) && kill((pid_t)pid, 0) == 0) {
4233 fprintf(stderr, "WARNING: The pid file contained the following (running) pid: %u\n", pid);
4234 }
4235 }
4236 fclose(fp);
4237 }
4238 }
4239
4240 /* Create the pid file first with a temporary name, then
4241 * atomically move the file to the real name to avoid a race with
4242 * another process opening the file to read the pid, but finding
4243 * it empty.
4244 */
4245 char tmp_pid_file[1024];
4246 snprintf(tmp_pid_file, sizeof(tmp_pid_file), "%s.tmp", pid_file);
4247
4248 if ((fp = fopen(tmp_pid_file, "w")) == NULL) {
4249 vperror("Could not open the pid file %s for writing", tmp_pid_file);
4250 return;
4251 }
4252
4253 fprintf(fp,"%ld\n", (long)getpid());
4254 if (fclose(fp) == -1) {
4255 vperror("Could not close the pid file %s", tmp_pid_file);
4256 }
4257
4258 if (rename(tmp_pid_file, pid_file) != 0) {
4259 vperror("Could not rename the pid file from %s to %s",
4260 tmp_pid_file, pid_file);
4261 }
4262 }
4263
4264 static void remove_pidfile(const char *pid_file) {
4265 if (pid_file == NULL)
4266 return;
4267
4268 if (unlink(pid_file) != 0) {
4269 vperror("Could not remove the pid file %s", pid_file);
4270 }
4271
4272 }
4273
4274 static void sig_handler(const int sig) {
4275 stop_main_loop = EXIT_NORMALLY;
4276 }
4277
4278 static void sighup_handler(const int sig) {
4279 settings.sig_hup = true;
4280 }
4281
4282 static void sig_usrhandler(const int sig) {
4283 stop_main_loop = GRACE_STOP;
4284 }
4285
4286 /*
4287 * On systems that supports multiple page sizes we may reduce the
4288 * number of TLB-misses by using the biggest available page size
4289 */
4290 static int enable_large_pages(void) {
4291 #if defined(HAVE_GETPAGESIZES) && defined(HAVE_MEMCNTL)
4292 int ret = -1;
4293 size_t sizes[32];
4294 int avail = getpagesizes(sizes, 32);
4295 if (avail != -1) {
4296 size_t max = sizes[0];
4297 struct memcntl_mha arg = {0};
4298 int ii;
4299
4300 for (ii = 1; ii < avail; ++ii) {
4301 if (max < sizes[ii]) {
4302 max = sizes[ii];
4303 }
4304 }
4305
4306 arg.mha_flags = 0;
4307 arg.mha_pagesize = max;
4308 arg.mha_cmd = MHA_MAPSIZE_BSSBRK;
4309
4310 if (memcntl(0, 0, MC_HAT_ADVISE, (caddr_t)&arg, 0, 0) == -1) {
4311 fprintf(stderr, "Failed to set large pages: %s\n",
4312 strerror(errno));
4313 fprintf(stderr, "Will use default page size\n");
4314 } else {
4315 ret = 0;
4316 }
4317 } else {
4318 fprintf(stderr, "Failed to get supported pagesizes: %s\n",
4319 strerror(errno));
4320 fprintf(stderr, "Will use default page size\n");
4321 }
4322
4323 return ret;
4324 #elif defined(__linux__) && defined(MADV_HUGEPAGE)
4325 /* check if transparent hugepages is compiled into the kernel */
4326 /* RH based systems possibly uses a different path */
4327 static const char *mm_thp_paths[] = {
4328 "/sys/kernel/mm/transparent_hugepage/enabled",
4329 "/sys/kernel/mm/redhat_transparent_hugepage/enabled",
4330 NULL
4331 };
4332
4333 char thpb[128] = {0};
4334 int pfd = -1;
4335 for (const char **p = mm_thp_paths; *p; p++) {
4336 if ((pfd = open(*p, O_RDONLY)) != -1)
4337 break;
4338 }
4339
4340 if (pfd == -1) {
4341 fprintf(stderr, "Transparent huge pages support not detected.\n");
4342 fprintf(stderr, "Will use default page size.\n");
4343 return -1;
4344 }
4345 ssize_t rd = read(pfd, thpb, sizeof(thpb));
4346 close(pfd);
4347 if (rd <= 0) {
4348 fprintf(stderr, "Transparent huge pages could not read the configuration.\n");
4349 fprintf(stderr, "Will use default page size.\n");
4350 return -1;
4351 }
4352 thpb[rd] = 0;
4353 if (strstr(thpb, "[never]")) {
4354 fprintf(stderr, "Transparent huge pages support disabled.\n");
4355 fprintf(stderr, "Will use default page size.\n");
4356 return -1;
4357 }
4358 return 0;
4359 #elif defined(__FreeBSD__)
4360 int spages;
4361 size_t spagesl = sizeof(spages);
4362
4363 if (sysctlbyname("vm.pmap.pg_ps_enabled", &spages,
4364 &spagesl, NULL, 0) != 0) {
4365 fprintf(stderr, "Could not evaluate the presence of superpages features.");
4366 return -1;
4367 }
4368 if (spages != 1) {
4369 fprintf(stderr, "Superpages support not detected.\n");
4370 fprintf(stderr, "Will use default page size.\n");
4371 return -1;
4372 }
4373 return 0;
4374 #else
4375 return -1;
4376 #endif
4377 }
4378
4379 /**
4380 * Do basic sanity check of the runtime environment
4381 * @return true if no errors found, false if we can't use this env
4382 */
4383 static bool sanitycheck(void) {
4384 /* One of our biggest problems is old and bogus libevents */
4385 const char *ever = event_get_version();
4386 if (ever != NULL) {
4387 if (strncmp(ever, "1.", 2) == 0) {
4388 fprintf(stderr, "You are using libevent %s.\nPlease upgrade to 2.x"
4389 " or newer\n", event_get_version());
4390 return false;
4391 }
4392 }
4393
4394 return true;
4395 }
4396
4397 static bool _parse_slab_sizes(char *s, uint32_t *slab_sizes) {
4398 char *b = NULL;
4399 uint32_t size = 0;
4400 int i = 0;
4401 uint32_t last_size = 0;
4402
4403 if (strlen(s) < 1)
4404 return false;
4405
4406 for (char *p = strtok_r(s, "-", &b);
4407 p != NULL;
4408 p = strtok_r(NULL, "-", &b)) {
4409 if (!safe_strtoul(p, &size) || size < settings.chunk_size
4410 || size > settings.slab_chunk_size_max) {
4411 fprintf(stderr, "slab size %u is out of valid range\n", size);
4412 return false;
4413 }
4414 if (last_size >= size) {
4415 fprintf(stderr, "slab size %u cannot be lower than or equal to a previous class size\n", size);
4416 return false;
4417 }
4418 if (size <= last_size + CHUNK_ALIGN_BYTES) {
4419 fprintf(stderr, "slab size %u must be at least %d bytes larger than previous class\n",
4420 size, CHUNK_ALIGN_BYTES);
4421 return false;
4422 }
4423 slab_sizes[i++] = size;
4424 last_size = size;
4425 if (i >= MAX_NUMBER_OF_SLAB_CLASSES-1) {
4426 fprintf(stderr, "too many slab classes specified\n");
4427 return false;
4428 }
4429 }
4430
4431 slab_sizes[i] = 0;
4432 return true;
4433 }
4434
4435 struct _mc_meta_data {
4436 void *mmap_base;
4437 uint64_t old_base;
4438 char *slab_config; // string containing either factor or custom slab list.
4439 int64_t time_delta;
4440 uint64_t process_started;
4441 uint32_t current_time;
4442 };
4443
4444 // We need to remember a combination of configuration settings and global
4445 // state for restart viability and resumption of internal services.
4446 // Compared to the number of tunables and state values, relatively little
4447 // does need to be remembered.
4448 // Time is the hardest; we have to assume the sys clock is correct and re-sync for
4449 // the lost time after restart.
4450 static int _mc_meta_save_cb(const char *tag, void *ctx, void *data) {
4451 struct _mc_meta_data *meta = (struct _mc_meta_data *)data;
4452
4453 // Settings to remember.
4454 // TODO: should get a version of version which is numeric, else
4455 // comparisons for compat reasons are difficult.
4456 // it may be possible to punt on this for now; since we can test for the
4457 // absence of another key... such as the new numeric version.
4458 //restart_set_kv(ctx, "version", "%s", VERSION);
4459 // We hold the original factor or subopts _string_
4460 // it can be directly compared without roundtripping through floats or
4461 // serializing/deserializing the long options list.
4462 restart_set_kv(ctx, "slab_config", "%s", meta->slab_config);
4463 restart_set_kv(ctx, "maxbytes", "%llu", (unsigned long long) settings.maxbytes);
4464 restart_set_kv(ctx, "chunk_size", "%d", settings.chunk_size);
4465 restart_set_kv(ctx, "item_size_max", "%d", settings.item_size_max);
4466 restart_set_kv(ctx, "slab_chunk_size_max", "%d", settings.slab_chunk_size_max);
4467 restart_set_kv(ctx, "slab_page_size", "%d", settings.slab_page_size);
4468 restart_set_kv(ctx, "use_cas", "%s", settings.use_cas ? "true" : "false");
4469 restart_set_kv(ctx, "slab_reassign", "%s", settings.slab_reassign ? "true" : "false");
4470
4471 // Online state to remember.
4472
4473 // current time is tough. we need to rely on the clock being correct to
4474 // pull the delta between stop and start times. we also need to know the
4475 // delta between start time and now to restore monotonic clocks.
4476 // for non-monotonic clocks (some OS?), process_started is the only
4477 // important one.
4478 restart_set_kv(ctx, "current_time", "%u", current_time);
4479 // types are great until... this. some systems time_t could be big, but
4480 // I'm assuming never negative.
4481 restart_set_kv(ctx, "process_started", "%llu", (unsigned long long) process_started);
4482 {
4483 struct timeval tv;
4484 gettimeofday(&tv, NULL);
4485 restart_set_kv(ctx, "stop_time", "%lu", tv.tv_sec);
4486 }
4487
4488 // Might as well just fetch the next CAS value to use than tightly
4489 // coupling the internal variable into the restart system.
4490 restart_set_kv(ctx, "current_cas", "%llu", (unsigned long long) get_cas_id());
4491 restart_set_kv(ctx, "logger_gid", "%llu", logger_get_gid());
4492 restart_set_kv(ctx, "hashpower", "%u", stats_state.hash_power_level);
4493 // NOTE: oldest_live is a rel_time_t, which aliases for unsigned int.
4494 // should future proof this with a 64bit upcast, or fetch value from a
4495 // converter function/macro?
4496 restart_set_kv(ctx, "oldest_live", "%u", settings.oldest_live);
4497 // TODO: use uintptr_t etc? is it portable enough?
4498 restart_set_kv(ctx, "mmap_oldbase", "%p", meta->mmap_base);
4499
4500 return 0;
4501 }
4502
4503 // We must see at least this number of checked lines. Else empty/missing lines
4504 // could cause a false-positive.
4505 // TODO: Once crc32'ing of the metadata file is done this could be ensured better by
4506 // the restart module itself (crc32 + count of lines must match on the
4507 // backend)
4508 #define RESTART_REQUIRED_META 16
4509
4510 // With this callback we make a decision on if the current configuration
4511 // matches up enough to allow reusing the cache.
4512 // We also re-load important runtime information.
4513 static int _mc_meta_load_cb(const char *tag, void *ctx, void *data) {
4514 struct _mc_meta_data *meta = (struct _mc_meta_data *)data;
4515 char *key;
4516 char *val;
4517 int reuse_mmap = 0;
4518 meta->process_started = 0;
4519 meta->time_delta = 0;
4520 meta->current_time = 0;
4521 int lines_seen = 0;
4522
4523 // TODO: not sure this is any better than just doing an if/else tree with
4524 // strcmp's...
4525 enum {
4526 R_MMAP_OLDBASE = 0,
4527 R_MAXBYTES,
4528 R_CHUNK_SIZE,
4529 R_ITEM_SIZE_MAX,
4530 R_SLAB_CHUNK_SIZE_MAX,
4531 R_SLAB_PAGE_SIZE,
4532 R_SLAB_CONFIG,
4533 R_USE_CAS,
4534 R_SLAB_REASSIGN,
4535 R_CURRENT_CAS,
4536 R_OLDEST_LIVE,
4537 R_LOGGER_GID,
4538 R_CURRENT_TIME,
4539 R_STOP_TIME,
4540 R_PROCESS_STARTED,
4541 R_HASHPOWER,
4542 };
4543
4544 const char *opts[] = {
4545 [R_MMAP_OLDBASE] = "mmap_oldbase",
4546 [R_MAXBYTES] = "maxbytes",
4547 [R_CHUNK_SIZE] = "chunk_size",
4548 [R_ITEM_SIZE_MAX] = "item_size_max",
4549 [R_SLAB_CHUNK_SIZE_MAX] = "slab_chunk_size_max",
4550 [R_SLAB_PAGE_SIZE] = "slab_page_size",
4551 [R_SLAB_CONFIG] = "slab_config",
4552 [R_USE_CAS] = "use_cas",
4553 [R_SLAB_REASSIGN] = "slab_reassign",
4554 [R_CURRENT_CAS] = "current_cas",
4555 [R_OLDEST_LIVE] = "oldest_live",
4556 [R_LOGGER_GID] = "logger_gid",
4557 [R_CURRENT_TIME] = "current_time",
4558 [R_STOP_TIME] = "stop_time",
4559 [R_PROCESS_STARTED] = "process_started",
4560 [R_HASHPOWER] = "hashpower",
4561 NULL
4562 };
4563
4564 while (restart_get_kv(ctx, &key, &val) == RESTART_OK) {
4565 int type = 0;
4566 int32_t val_int = 0;
4567 uint32_t val_uint = 0;
4568 int64_t bigval_int = 0;
4569 uint64_t bigval_uint = 0;
4570
4571 while (opts[type] != NULL && strcmp(key, opts[type]) != 0) {
4572 type++;
4573 }
4574 if (opts[type] == NULL) {
4575 fprintf(stderr, "[restart] unknown/unhandled key: %s\n", key);
4576 continue;
4577 }
4578 lines_seen++;
4579
4580 // helper for any boolean checkers.
4581 bool val_bool = false;
4582 bool is_bool = true;
4583 if (strcmp(val, "false") == 0) {
4584 val_bool = false;
4585 } else if (strcmp(val, "true") == 0) {
4586 val_bool = true;
4587 } else {
4588 is_bool = false;
4589 }
4590
4591 switch (type) {
4592 case R_MMAP_OLDBASE:
4593 if (!safe_strtoull_hex(val, &meta->old_base)) {
4594 fprintf(stderr, "[restart] failed to parse %s: %s\n", key, val);
4595 reuse_mmap = -1;
4596 }
4597 break;
4598 case R_MAXBYTES:
4599 if (!safe_strtoll(val, &bigval_int) || settings.maxbytes != bigval_int) {
4600 reuse_mmap = -1;
4601 }
4602 break;
4603 case R_CHUNK_SIZE:
4604 if (!safe_strtol(val, &val_int) || settings.chunk_size != val_int) {
4605 reuse_mmap = -1;
4606 }
4607 break;
4608 case R_ITEM_SIZE_MAX:
4609 if (!safe_strtol(val, &val_int) || settings.item_size_max != val_int) {
4610 reuse_mmap = -1;
4611 }
4612 break;
4613 case R_SLAB_CHUNK_SIZE_MAX:
4614 if (!safe_strtol(val, &val_int) || settings.slab_chunk_size_max != val_int) {
4615 reuse_mmap = -1;
4616 }
4617 break;
4618 case R_SLAB_PAGE_SIZE:
4619 if (!safe_strtol(val, &val_int) || settings.slab_page_size != val_int) {
4620 reuse_mmap = -1;
4621 }
4622 break;
4623 case R_SLAB_CONFIG:
4624 if (strcmp(val, meta->slab_config) != 0) {
4625 reuse_mmap = -1;
4626 }
4627 break;
4628 case R_USE_CAS:
4629 if (!is_bool || settings.use_cas != val_bool) {
4630 reuse_mmap = -1;
4631 }
4632 break;
4633 case R_SLAB_REASSIGN:
4634 if (!is_bool || settings.slab_reassign != val_bool) {
4635 reuse_mmap = -1;
4636 }
4637 break;
4638 case R_CURRENT_CAS:
4639 // FIXME: do we need to fail if these values _aren't_ found?
4640 if (!safe_strtoull(val, &bigval_uint)) {
4641 reuse_mmap = -1;
4642 } else {
4643 set_cas_id(bigval_uint);
4644 }
4645 break;
4646 case R_OLDEST_LIVE:
4647 if (!safe_strtoul(val, &val_uint)) {
4648 reuse_mmap = -1;
4649 } else {
4650 settings.oldest_live = val_uint;
4651 }
4652 break;
4653 case R_LOGGER_GID:
4654 if (!safe_strtoull(val, &bigval_uint)) {
4655 reuse_mmap = -1;
4656 } else {
4657 logger_set_gid(bigval_uint);
4658 }
4659 break;
4660 case R_PROCESS_STARTED:
4661 if (!safe_strtoull(val, &bigval_uint)) {
4662 reuse_mmap = -1;
4663 } else {
4664 meta->process_started = bigval_uint;
4665 }
4666 break;
4667 case R_CURRENT_TIME:
4668 if (!safe_strtoul(val, &val_uint)) {
4669 reuse_mmap = -1;
4670 } else {
4671 meta->current_time = val_uint;
4672 }
4673 break;
4674 case R_STOP_TIME:
4675 if (!safe_strtoll(val, &bigval_int)) {
4676 reuse_mmap = -1;
4677 } else {
4678 struct timeval t;
4679 gettimeofday(&t, NULL);
4680 meta->time_delta = t.tv_sec - bigval_int;
4681 // clock has done something crazy.
4682 // there are _lots_ of ways the clock can go wrong here, but
4683 // this is a safe sanity check since there's nothing else we
4684 // can realistically do.
4685 if (meta->time_delta <= 0) {
4686 reuse_mmap = -1;
4687 }
4688 }
4689 break;
4690 case R_HASHPOWER:
4691 if (!safe_strtoul(val, &val_uint)) {
4692 reuse_mmap = -1;
4693 } else {
4694 settings.hashpower_init = val_uint;
4695 }
4696 break;
4697 default:
4698 fprintf(stderr, "[restart] unhandled key: %s\n", key);
4699 }
4700
4701 if (reuse_mmap != 0) {
4702 fprintf(stderr, "[restart] restart incompatible due to setting for [%s] [old value: %s]\n", key, val);
4703 break;
4704 }
4705 }
4706
4707 if (lines_seen < RESTART_REQUIRED_META) {
4708 fprintf(stderr, "[restart] missing some metadata lines\n");
4709 reuse_mmap = -1;
4710 }
4711
4712 return reuse_mmap;
4713 }
4714
4715 int main (int argc, char **argv) {
4716 int c;
4717 bool lock_memory = false;
4718 bool do_daemonize = false;
4719 bool preallocate = false;
4720 int maxcore = 0;
4721 char *username = NULL;
4722 char *pid_file = NULL;
4723 struct passwd *pw;
4724 struct rlimit rlim;
4725 char *buf;
4726 char unit = '\0';
4727 int size_max = 0;
4728 int retval = EXIT_SUCCESS;
4729 bool protocol_specified = false;
4730 bool tcp_specified = false;
4731 bool udp_specified = false;
4732 bool start_lru_maintainer = true;
4733 bool start_lru_crawler = true;
4734 bool start_assoc_maint = true;
4735 enum hashfunc_type hash_type = MURMUR3_HASH;
4736 uint32_t tocrawl;
4737 uint32_t slab_sizes[MAX_NUMBER_OF_SLAB_CLASSES];
4738 bool use_slab_sizes = false;
4739 char *slab_sizes_unparsed = NULL;
4740 bool slab_chunk_size_changed = false;
4741 // struct for restart code. Initialized up here so we can curry
4742 // important settings to save or validate.
4743 struct _mc_meta_data *meta = malloc(sizeof(struct _mc_meta_data));
4744 meta->slab_config = NULL;
4745 char *subopts, *subopts_orig;
4746 char *subopts_value;
4747 enum {
4748 MAXCONNS_FAST = 0,
4749 HASHPOWER_INIT,
4750 NO_HASHEXPAND,
4751 SLAB_REASSIGN,
4752 SLAB_AUTOMOVE,
4753 SLAB_AUTOMOVE_RATIO,
4754 SLAB_AUTOMOVE_WINDOW,
4755 TAIL_REPAIR_TIME,
4756 HASH_ALGORITHM,
4757 LRU_CRAWLER,
4758 LRU_CRAWLER_SLEEP,
4759 LRU_CRAWLER_TOCRAWL,
4760 LRU_MAINTAINER,
4761 HOT_LRU_PCT,
4762 WARM_LRU_PCT,
4763 HOT_MAX_FACTOR,
4764 WARM_MAX_FACTOR,
4765 TEMPORARY_TTL,
4766 IDLE_TIMEOUT,
4767 WATCHER_LOGBUF_SIZE,
4768 WORKER_LOGBUF_SIZE,
4769 SLAB_SIZES,
4770 SLAB_CHUNK_MAX,
4771 TRACK_SIZES,
4772 NO_INLINE_ASCII_RESP,
4773 MODERN,
4774 NO_MODERN,
4775 NO_CHUNKED_ITEMS,
4776 NO_SLAB_REASSIGN,
4777 NO_SLAB_AUTOMOVE,
4778 NO_MAXCONNS_FAST,
4779 INLINE_ASCII_RESP,
4780 NO_LRU_CRAWLER,
4781 NO_LRU_MAINTAINER,
4782 NO_DROP_PRIVILEGES,
4783 DROP_PRIVILEGES,
4784 RESP_OBJ_MEM_LIMIT,
4785 READ_BUF_MEM_LIMIT,
4786 #ifdef TLS
4787 SSL_CERT,
4788 SSL_KEY,
4789 SSL_VERIFY_MODE,
4790 SSL_KEYFORM,
4791 SSL_CIPHERS,
4792 SSL_CA_CERT,
4793 SSL_WBUF_SIZE,
4794 SSL_SESSION_CACHE,
4795 SSL_KERNEL_TLS,
4796 SSL_MIN_VERSION,
4797 #endif
4798 #ifdef PROXY
4799 PROXY_CONFIG,
4800 PROXY_ARG,
4801 PROXY_URING,
4802 PROXY_MEMPROFILE,
4803 #endif
4804 #ifdef MEMCACHED_DEBUG
4805 RELAXED_PRIVILEGES,
4806 #endif
4807 #ifdef SOCK_COOKIE_ID
4808 COOKIE_ID,
4809 #endif
4810 };
4811 char *const subopts_tokens[] = {
4812 [MAXCONNS_FAST] = "maxconns_fast",
4813 [HASHPOWER_INIT] = "hashpower",
4814 [NO_HASHEXPAND] = "no_hashexpand",
4815 [SLAB_REASSIGN] = "slab_reassign",
4816 [SLAB_AUTOMOVE] = "slab_automove",
4817 [SLAB_AUTOMOVE_RATIO] = "slab_automove_ratio",
4818 [SLAB_AUTOMOVE_WINDOW] = "slab_automove_window",
4819 [TAIL_REPAIR_TIME] = "tail_repair_time",
4820 [HASH_ALGORITHM] = "hash_algorithm",
4821 [LRU_CRAWLER] = "lru_crawler",
4822 [LRU_CRAWLER_SLEEP] = "lru_crawler_sleep",
4823 [LRU_CRAWLER_TOCRAWL] = "lru_crawler_tocrawl",
4824 [LRU_MAINTAINER] = "lru_maintainer",
4825 [HOT_LRU_PCT] = "hot_lru_pct",
4826 [WARM_LRU_PCT] = "warm_lru_pct",
4827 [HOT_MAX_FACTOR] = "hot_max_factor",
4828 [WARM_MAX_FACTOR] = "warm_max_factor",
4829 [TEMPORARY_TTL] = "temporary_ttl",
4830 [IDLE_TIMEOUT] = "idle_timeout",
4831 [WATCHER_LOGBUF_SIZE] = "watcher_logbuf_size",
4832 [WORKER_LOGBUF_SIZE] = "worker_logbuf_size",
4833 [SLAB_SIZES] = "slab_sizes",
4834 [SLAB_CHUNK_MAX] = "slab_chunk_max",
4835 [TRACK_SIZES] = "track_sizes",
4836 [NO_INLINE_ASCII_RESP] = "no_inline_ascii_resp",
4837 [MODERN] = "modern",
4838 [NO_MODERN] = "no_modern",
4839 [NO_CHUNKED_ITEMS] = "no_chunked_items",
4840 [NO_SLAB_REASSIGN] = "no_slab_reassign",
4841 [NO_SLAB_AUTOMOVE] = "no_slab_automove",
4842 [NO_MAXCONNS_FAST] = "no_maxconns_fast",
4843 [INLINE_ASCII_RESP] = "inline_ascii_resp",
4844 [NO_LRU_CRAWLER] = "no_lru_crawler",
4845 [NO_LRU_MAINTAINER] = "no_lru_maintainer",
4846 [NO_DROP_PRIVILEGES] = "no_drop_privileges",
4847 [DROP_PRIVILEGES] = "drop_privileges",
4848 [RESP_OBJ_MEM_LIMIT] = "resp_obj_mem_limit",
4849 [READ_BUF_MEM_LIMIT] = "read_buf_mem_limit",
4850 #ifdef TLS
4851 [SSL_CERT] = "ssl_chain_cert",
4852 [SSL_KEY] = "ssl_key",
4853 [SSL_VERIFY_MODE] = "ssl_verify_mode",
4854 [SSL_KEYFORM] = "ssl_keyformat",
4855 [SSL_CIPHERS] = "ssl_ciphers",
4856 [SSL_CA_CERT] = "ssl_ca_cert",
4857 [SSL_WBUF_SIZE] = "ssl_wbuf_size",
4858 [SSL_SESSION_CACHE] = "ssl_session_cache",
4859 [SSL_KERNEL_TLS] = "ssl_kernel_tls",
4860 [SSL_MIN_VERSION] = "ssl_min_version",
4861 #endif
4862 #ifdef PROXY
4863 [PROXY_CONFIG] = "proxy_config",
4864 [PROXY_ARG] = "proxy_arg",
4865 [PROXY_URING] = "proxy_uring",
4866 [PROXY_MEMPROFILE] = "proxy_memprofile",
4867 #endif
4868 #ifdef MEMCACHED_DEBUG
4869 [RELAXED_PRIVILEGES] = "relaxed_privileges",
4870 #endif
4871 #ifdef SOCK_COOKIE_ID
4872 [COOKIE_ID] = "sock_cookie_id",
4873 #endif
4874 NULL
4875 };
4876
4877 if (!sanitycheck()) {
4878 free(meta);
4879 return EX_OSERR;
4880 }
4881
4882 /* handle SIGINT, SIGTERM */
4883 signal(SIGINT, sig_handler);
4884 signal(SIGTERM, sig_handler);
4885 signal(SIGHUP, sighup_handler);
4886 signal(SIGUSR1, sig_usrhandler);
4887
4888 /* init settings */
4889 settings_init();
4890 verify_default("hash_algorithm", hash_type == MURMUR3_HASH);
4891 #ifdef EXTSTORE
4892 void *storage = NULL;
4893 void *storage_cf = storage_init_config(&settings);
4894 bool storage_enabled = false;
4895 if (storage_cf == NULL) {
4896 fprintf(stderr, "failed to allocate extstore config\n");
4897 return 1;
4898 }
4899 #endif
4900
4901 /* set stderr non-buffering (for running under, say, daemontools) */
4902 setbuf(stderr, NULL);
4903
4904 char *shortopts =
4905 "a:" /* access mask for unix socket */
4906 "A" /* enable admin shutdown command */
4907 "Z" /* enable SSL */
4908 "p:" /* TCP port number to listen on */
4909 "s:" /* unix socket path to listen on */
4910 "U:" /* UDP port number to listen on */
4911 "m:" /* max memory to use for items in megabytes */
4912 "M" /* return error on memory exhausted */
4913 "c:" /* max simultaneous connections */
4914 "k" /* lock down all paged memory */
4915 "hiV" /* help, licence info, version */
4916 "r" /* maximize core file limit */
4917 "v" /* verbose */
4918 "d" /* daemon mode */
4919 "l:" /* interface to listen on */
4920 "u:" /* user identity to run as */
4921 "P:" /* save PID in file */
4922 "f:" /* factor? */
4923 "n:" /* minimum space allocated for key+value+flags */
4924 "t:" /* threads */
4925 "D:" /* prefix delimiter? */
4926 "L" /* Large memory pages */
4927 "R:" /* max requests per event */
4928 "C" /* Disable use of CAS */
4929 "b:" /* backlog queue limit */
4930 "B:" /* Binding protocol */
4931 "I:" /* Max item size */
4932 "S" /* Sasl ON */
4933 "F" /* Disable flush_all */
4934 "X" /* Disable dump commands */
4935 "W" /* Disable watch commands */
4936 "Y:" /* Enable token auth */
4937 "e:" /* mmap path for external item memory */
4938 "o:" /* Extended generic options */
4939 "N:" /* NAPI ID based thread selection */
4940 ;
4941
4942 /* process arguments */
4943 #ifdef HAVE_GETOPT_LONG
4944 const struct option longopts[] = {
4945 {"unix-mask", required_argument, 0, 'a'},
4946 {"enable-shutdown", no_argument, 0, 'A'},
4947 {"enable-ssl", no_argument, 0, 'Z'},
4948 {"port", required_argument, 0, 'p'},
4949 {"unix-socket", required_argument, 0, 's'},
4950 {"udp-port", required_argument, 0, 'U'},
4951 {"memory-limit", required_argument, 0, 'm'},
4952 {"disable-evictions", no_argument, 0, 'M'},
4953 {"conn-limit", required_argument, 0, 'c'},
4954 {"lock-memory", no_argument, 0, 'k'},
4955 {"help", no_argument, 0, 'h'},
4956 {"license", no_argument, 0, 'i'},
4957 {"version", no_argument, 0, 'V'},
4958 {"enable-coredumps", no_argument, 0, 'r'},
4959 {"verbose", optional_argument, 0, 'v'},
4960 {"daemon", no_argument, 0, 'd'},
4961 {"listen", required_argument, 0, 'l'},
4962 {"user", required_argument, 0, 'u'},
4963 {"pidfile", required_argument, 0, 'P'},
4964 {"slab-growth-factor", required_argument, 0, 'f'},
4965 {"slab-min-size", required_argument, 0, 'n'},
4966 {"threads", required_argument, 0, 't'},
4967 {"enable-largepages", no_argument, 0, 'L'},
4968 {"max-reqs-per-event", required_argument, 0, 'R'},
4969 {"disable-cas", no_argument, 0, 'C'},
4970 {"listen-backlog", required_argument, 0, 'b'},
4971 {"protocol", required_argument, 0, 'B'},
4972 {"max-item-size", required_argument, 0, 'I'},
4973 {"enable-sasl", no_argument, 0, 'S'},
4974 {"disable-flush-all", no_argument, 0, 'F'},
4975 {"disable-dumping", no_argument, 0, 'X'},
4976 {"disable-watch", no_argument, 0, 'W'},
4977 {"auth-file", required_argument, 0, 'Y'},
4978 {"memory-file", required_argument, 0, 'e'},
4979 {"extended", required_argument, 0, 'o'},
4980 {"napi-ids", required_argument, 0, 'N'},
4981 {0, 0, 0, 0}
4982 };
4983 int optindex;
4984 while (-1 != (c = getopt_long(argc, argv, shortopts,
4985 longopts, &optindex))) {
4986 #else
4987 while (-1 != (c = getopt(argc, argv, shortopts))) {
4988 #endif
4989 if (optarg) {
4990 while(isspace(optarg[0])) {
4991 optarg++;
4992 }
4993 }
4994 switch (c) {
4995 case 'A':
4996 /* enables "shutdown" command */
4997 settings.shutdown_command = true;
4998 break;
4999 case 'Z':
5000 /* enable secure communication*/
5001 #ifdef TLS
5002 settings.ssl_enabled = true;
5003 #else
5004 fprintf(stderr, "This server is not built with TLS support.\n");
5005 exit(EX_USAGE);
5006 #endif
5007 break;
5008 case 'a':
5009 #ifndef DISABLE_UNIX_SOCKET
5010 /* access for unix domain socket, as octal mask (like chmod)*/
5011 settings.access= strtol(optarg,NULL,8);
5012 #else
5013 fprintf(stderr, "This server is not built with unix socket support.\n");
5014 exit(EX_USAGE);
5015 #endif /* #ifndef DISABLE_UNIX_SOCKET */
5016 break;
5017 case 'U':
5018 settings.udpport = atoi(optarg);
5019 udp_specified = true;
5020 break;
5021 case 'p':
5022 settings.port = atoi(optarg);
5023 tcp_specified = true;
5024 break;
5025 case 's':
5026 #ifndef DISABLE_UNIX_SOCKET
5027 settings.socketpath = optarg;
5028 #else
5029 fprintf(stderr, "This server is not built with unix socket support.\n");
5030 exit(EX_USAGE);
5031 #endif /* #ifndef DISABLE_UNIX_SOCKET */
5032 break;
5033 case 'm':
5034 settings.maxbytes = ((size_t)atoi(optarg)) * 1024 * 1024;
5035 break;
5036 case 'M':
5037 settings.evict_to_free = 0;
5038 break;
5039 case 'c':
5040 settings.maxconns = atoi(optarg);
5041 if (settings.maxconns <= 0) {
5042 fprintf(stderr, "Maximum connections must be greater than 0\n");
5043 return 1;
5044 }
5045 break;
5046 case 'h':
5047 usage();
5048 exit(EXIT_SUCCESS);
5049 case 'i':
5050 usage_license();
5051 exit(EXIT_SUCCESS);
5052 case 'V':
5053 printf(PACKAGE " " VERSION "\n");
5054 exit(EXIT_SUCCESS);
5055 case 'k':
5056 lock_memory = true;
5057 break;
5058 case 'v':
5059 settings.verbose++;
5060 break;
5061 case 'l':
5062 if (settings.inter != NULL) {
5063 if (strstr(settings.inter, optarg) != NULL) {
5064 break;
5065 }
5066 size_t len = strlen(settings.inter) + strlen(optarg) + 2;
5067 char *p = malloc(len);
5068 if (p == NULL) {
5069 fprintf(stderr, "Failed to allocate memory\n");
5070 return 1;
5071 }
5072 snprintf(p, len, "%s,%s", settings.inter, optarg);
5073 free(settings.inter);
5074 settings.inter = p;
5075 } else {
5076 settings.inter= strdup(optarg);
5077 }
5078 break;
5079 case 'd':
5080 do_daemonize = true;
5081 break;
5082 case 'r':
5083 maxcore = 1;
5084 break;
5085 case 'R':
5086 settings.reqs_per_event = atoi(optarg);
5087 if (settings.reqs_per_event <= 0) {
5088 fprintf(stderr, "Number of requests per event must be greater than 0\n");
5089 return 1;
5090 }
5091 break;
5092 case 'u':
5093 username = optarg;
5094 break;
5095 case 'P':
5096 pid_file = optarg;
5097 break;
5098 case 'e':
5099 settings.memory_file = optarg;
5100 break;
5101 case 'f':
5102 settings.factor = atof(optarg);
5103 if (settings.factor <= 1.0) {
5104 fprintf(stderr, "Factor must be greater than 1\n");
5105 return 1;
5106 }
5107 meta->slab_config = strdup(optarg);
5108 break;
5109 case 'n':
5110 settings.chunk_size = atoi(optarg);
5111 if (settings.chunk_size <= 0) {
5112 fprintf(stderr, "Chunk size must be greater than 0\n");
5113 return 1;
5114 }
5115 break;
5116 case 't':
5117 settings.num_threads = atoi(optarg);
5118 if (settings.num_threads <= 0) {
5119 fprintf(stderr, "Number of threads must be greater than 0\n");
5120 return 1;
5121 }
5122 /* There're other problems when you get above 64 threads.
5123 * In the future we should portably detect # of cores for the
5124 * default.
5125 */
5126 if (settings.num_threads > 64) {
5127 fprintf(stderr, "WARNING: Setting a high number of worker"
5128 "threads is not recommended.\n"
5129 " Set this value to the number of cores in"
5130 " your machine or less.\n");
5131 }
5132 break;
5133 case 'D':
5134 if (! optarg || ! optarg[0]) {
5135 fprintf(stderr, "No delimiter specified\n");
5136 return 1;
5137 }
5138 settings.prefix_delimiter = optarg[0];
5139 settings.detail_enabled = 1;
5140 break;
5141 case 'L' :
5142 if (enable_large_pages() == 0) {
5143 preallocate = true;
5144 } else {
5145 fprintf(stderr, "Cannot enable large pages on this system\n"
5146 "(There is no support as of this version)\n");
5147 return 1;
5148 }
5149 break;
5150 case 'C' :
5151 settings.use_cas = false;
5152 break;
5153 case 'b' :
5154 settings.backlog = atoi(optarg);
5155 break;
5156 case 'B':
5157 protocol_specified = true;
5158 if (strcmp(optarg, "auto") == 0) {
5159 settings.binding_protocol = negotiating_prot;
5160 } else if (strcmp(optarg, "binary") == 0) {
5161 settings.binding_protocol = binary_prot;
5162 } else if (strcmp(optarg, "ascii") == 0) {
5163 settings.binding_protocol = ascii_prot;
5164 } else {
5165 fprintf(stderr, "Invalid value for binding protocol: %s\n"
5166 " -- should be one of auto, binary, or ascii\n", optarg);
5167 exit(EX_USAGE);
5168 }
5169 break;
5170 case 'I':
5171 buf = strdup(optarg);
5172 unit = buf[strlen(buf)-1];
5173 if (unit == 'k' || unit == 'm' ||
5174 unit == 'K' || unit == 'M') {
5175 buf[strlen(buf)-1] = '\0';
5176 size_max = atoi(buf);
5177 if (unit == 'k' || unit == 'K')
5178 size_max *= 1024;
5179 if (unit == 'm' || unit == 'M')
5180 size_max *= 1024 * 1024;
5181 settings.item_size_max = size_max;
5182 } else {
5183 settings.item_size_max = atoi(buf);
5184 }
5185 free(buf);
5186 break;
5187 case 'S': /* set Sasl authentication to true. Default is false */
5188 #ifndef ENABLE_SASL
5189 fprintf(stderr, "This server is not built with SASL support.\n");
5190 exit(EX_USAGE);
5191 #endif
5192 settings.sasl = true;
5193 break;
5194 case 'F' :
5195 settings.flush_enabled = false;
5196 break;
5197 case 'X' :
5198 settings.dump_enabled = false;
5199 break;
5200 case 'W' :
5201 settings.watch_enabled = false;
5202 break;
5203 case 'Y' :
5204 // dupe the file path now just in case the options get mangled.
5205 settings.auth_file = strdup(optarg);
5206 break;
5207 case 'N':
5208 settings.num_napi_ids = atoi(optarg);
5209 if (settings.num_napi_ids <= 0) {
5210 fprintf(stderr, "Maximum number of NAPI IDs must be greater than 0\n");
5211 return 1;
5212 }
5213 break;
5214 case 'o': /* It's sub-opts time! */
5215 subopts_orig = subopts = strdup(optarg); /* getsubopt() changes the original args */
5216
5217 while (*subopts != '\0') {
5218 // BSD getsubopt (at least) has undefined behavior on -1, so
5219 // if we want to retry the getsubopt call in submodules we
5220 // need an extra layer of string copies.
5221 char *subopts_temp_o = NULL;
5222 char *subopts_temp = subopts_temp_o = strdup(subopts);
5223
5224 switch (getsubopt(&subopts, subopts_tokens, &subopts_value)) {
5225 case MAXCONNS_FAST:
5226 settings.maxconns_fast = true;
5227 break;
5228 case HASHPOWER_INIT:
5229 if (subopts_value == NULL) {
5230 fprintf(stderr, "Missing numeric argument for hashpower\n");
5231 return 1;
5232 }
5233 settings.hashpower_init = atoi(subopts_value);
5234 if (settings.hashpower_init < 12) {
5235 fprintf(stderr, "Initial hashtable multiplier of %d is too low\n",
5236 settings.hashpower_init);
5237 return 1;
5238 } else if (settings.hashpower_init > 32) {
5239 fprintf(stderr, "Initial hashtable multiplier of %d is too high\n"
5240 "Choose a value based on \"STAT hash_power_level\" from a running instance\n",
5241 settings.hashpower_init);
5242 return 1;
5243 }
5244 break;
5245 case NO_HASHEXPAND:
5246 start_assoc_maint = false;
5247 break;
5248 case SLAB_REASSIGN:
5249 settings.slab_reassign = true;
5250 break;
5251 case SLAB_AUTOMOVE:
5252 if (subopts_value == NULL) {
5253 settings.slab_automove = 1;
5254 break;
5255 }
5256 settings.slab_automove = atoi(subopts_value);
5257 if (settings.slab_automove < 0 || settings.slab_automove > 2) {
5258 fprintf(stderr, "slab_automove must be between 0 and 2\n");
5259 return 1;
5260 }
5261 break;
5262 case SLAB_AUTOMOVE_RATIO:
5263 if (subopts_value == NULL) {
5264 fprintf(stderr, "Missing slab_automove_ratio argument\n");
5265 return 1;
5266 }
5267 settings.slab_automove_ratio = atof(subopts_value);
5268 if (settings.slab_automove_ratio <= 0 || settings.slab_automove_ratio > 1) {
5269 fprintf(stderr, "slab_automove_ratio must be > 0 and < 1\n");
5270 return 1;
5271 }
5272 break;
5273 case SLAB_AUTOMOVE_WINDOW:
5274 if (subopts_value == NULL) {
5275 fprintf(stderr, "Missing slab_automove_window argument\n");
5276 return 1;
5277 }
5278 settings.slab_automove_window = atoi(subopts_value);
5279 if (settings.slab_automove_window < 3) {
5280 fprintf(stderr, "slab_automove_window must be > 2\n");
5281 return 1;
5282 }
5283 break;
5284 case TAIL_REPAIR_TIME:
5285 if (subopts_value == NULL) {
5286 fprintf(stderr, "Missing numeric argument for tail_repair_time\n");
5287 return 1;
5288 }
5289 settings.tail_repair_time = atoi(subopts_value);
5290 if (settings.tail_repair_time < 10) {
5291 fprintf(stderr, "Cannot set tail_repair_time to less than 10 seconds\n");
5292 return 1;
5293 }
5294 break;
5295 case HASH_ALGORITHM:
5296 if (subopts_value == NULL) {
5297 fprintf(stderr, "Missing hash_algorithm argument\n");
5298 return 1;
5299 };
5300 if (strcmp(subopts_value, "jenkins") == 0) {
5301 hash_type = JENKINS_HASH;
5302 } else if (strcmp(subopts_value, "murmur3") == 0) {
5303 hash_type = MURMUR3_HASH;
5304 } else if (strcmp(subopts_value, "xxh3") == 0) {
5305 hash_type = XXH3_HASH;
5306 } else {
5307 fprintf(stderr, "Unknown hash_algorithm option (jenkins, murmur3, xxh3)\n");
5308 return 1;
5309 }
5310 break;
5311 case LRU_CRAWLER:
5312 start_lru_crawler = true;
5313 break;
5314 case LRU_CRAWLER_SLEEP:
5315 if (subopts_value == NULL) {
5316 fprintf(stderr, "Missing lru_crawler_sleep value\n");
5317 return 1;
5318 }
5319 settings.lru_crawler_sleep = atoi(subopts_value);
5320 if (settings.lru_crawler_sleep > 1000000 || settings.lru_crawler_sleep < 0) {
5321 fprintf(stderr, "LRU crawler sleep must be between 0 and 1 second\n");
5322 return 1;
5323 }
5324 break;
5325 case LRU_CRAWLER_TOCRAWL:
5326 if (subopts_value == NULL) {
5327 fprintf(stderr, "Missing lru_crawler_tocrawl value\n");
5328 return 1;
5329 }
5330 if (!safe_strtoul(subopts_value, &tocrawl)) {
5331 fprintf(stderr, "lru_crawler_tocrawl takes a numeric 32bit value\n");
5332 return 1;
5333 }
5334 settings.lru_crawler_tocrawl = tocrawl;
5335 break;
5336 case LRU_MAINTAINER:
5337 start_lru_maintainer = true;
5338 settings.lru_segmented = true;
5339 break;
5340 case HOT_LRU_PCT:
5341 if (subopts_value == NULL) {
5342 fprintf(stderr, "Missing hot_lru_pct argument\n");
5343 return 1;
5344 }
5345 settings.hot_lru_pct = atoi(subopts_value);
5346 if (settings.hot_lru_pct < 1 || settings.hot_lru_pct >= 80) {
5347 fprintf(stderr, "hot_lru_pct must be > 1 and < 80\n");
5348 return 1;
5349 }
5350 break;
5351 case WARM_LRU_PCT:
5352 if (subopts_value == NULL) {
5353 fprintf(stderr, "Missing warm_lru_pct argument\n");
5354 return 1;
5355 }
5356 settings.warm_lru_pct = atoi(subopts_value);
5357 if (settings.warm_lru_pct < 1 || settings.warm_lru_pct >= 80) {
5358 fprintf(stderr, "warm_lru_pct must be > 1 and < 80\n");
5359 return 1;
5360 }
5361 break;
5362 case HOT_MAX_FACTOR:
5363 if (subopts_value == NULL) {
5364 fprintf(stderr, "Missing hot_max_factor argument\n");
5365 return 1;
5366 }
5367 settings.hot_max_factor = atof(subopts_value);
5368 if (settings.hot_max_factor <= 0) {
5369 fprintf(stderr, "hot_max_factor must be > 0\n");
5370 return 1;
5371 }
5372 break;
5373 case WARM_MAX_FACTOR:
5374 if (subopts_value == NULL) {
5375 fprintf(stderr, "Missing warm_max_factor argument\n");
5376 return 1;
5377 }
5378 settings.warm_max_factor = atof(subopts_value);
5379 if (settings.warm_max_factor <= 0) {
5380 fprintf(stderr, "warm_max_factor must be > 0\n");
5381 return 1;
5382 }
5383 break;
5384 case TEMPORARY_TTL:
5385 if (subopts_value == NULL) {
5386 fprintf(stderr, "Missing temporary_ttl argument\n");
5387 return 1;
5388 }
5389 settings.temp_lru = true;
5390 settings.temporary_ttl = atoi(subopts_value);
5391 break;
5392 case IDLE_TIMEOUT:
5393 if (subopts_value == NULL) {
5394 fprintf(stderr, "Missing numeric argument for idle_timeout\n");
5395 return 1;
5396 }
5397 settings.idle_timeout = atoi(subopts_value);
5398 break;
5399 case WATCHER_LOGBUF_SIZE:
5400 if (subopts_value == NULL) {
5401 fprintf(stderr, "Missing watcher_logbuf_size argument\n");
5402 return 1;
5403 }
5404 if (!safe_strtoul(subopts_value, &settings.logger_watcher_buf_size)) {
5405 fprintf(stderr, "could not parse argument to watcher_logbuf_size\n");
5406 return 1;
5407 }
5408 settings.logger_watcher_buf_size *= 1024; /* kilobytes */
5409 break;
5410 case WORKER_LOGBUF_SIZE:
5411 if (subopts_value == NULL) {
5412 fprintf(stderr, "Missing worker_logbuf_size argument\n");
5413 return 1;
5414 }
5415 if (!safe_strtoul(subopts_value, &settings.logger_buf_size)) {
5416 fprintf(stderr, "could not parse argument to worker_logbuf_size\n");
5417 return 1;
5418 }
5419 settings.logger_buf_size *= 1024; /* kilobytes */
5420 case SLAB_SIZES:
5421 slab_sizes_unparsed = strdup(subopts_value);
5422 break;
5423 case SLAB_CHUNK_MAX:
5424 if (subopts_value == NULL) {
5425 fprintf(stderr, "Missing slab_chunk_max argument\n");
5426 return 1;
5427 }
5428 if (!safe_strtol(subopts_value, &settings.slab_chunk_size_max)) {
5429 fprintf(stderr, "could not parse argument to slab_chunk_max\n");
5430 return 1;
5431 }
5432 if (settings.slab_chunk_size_max <= 0) {
5433 fprintf(stderr, "slab_chunk_max must be >= 0\n");
5434 return 1;
5435 }
5436 if (settings.slab_chunk_size_max > (1 << 10)) {
5437 fprintf(stderr, "slab_chunk_max must be specified in kilobytes.\n");
5438 return 1;
5439 }
5440 settings.slab_chunk_size_max *= (1 << 10);
5441 slab_chunk_size_changed = true;
5442 break;
5443 case TRACK_SIZES:
5444 item_stats_sizes_init();
5445 break;
5446 case NO_INLINE_ASCII_RESP:
5447 break;
5448 case INLINE_ASCII_RESP:
5449 break;
5450 case NO_CHUNKED_ITEMS:
5451 settings.slab_chunk_size_max = settings.slab_page_size;
5452 break;
5453 case NO_SLAB_REASSIGN:
5454 settings.slab_reassign = false;
5455 break;
5456 case NO_SLAB_AUTOMOVE:
5457 settings.slab_automove = 0;
5458 break;
5459 case NO_MAXCONNS_FAST:
5460 settings.maxconns_fast = false;
5461 break;
5462 case NO_LRU_CRAWLER:
5463 settings.lru_crawler = false;
5464 start_lru_crawler = false;
5465 break;
5466 case NO_LRU_MAINTAINER:
5467 start_lru_maintainer = false;
5468 settings.lru_segmented = false;
5469 break;
5470 #ifdef TLS
5471 case SSL_CERT:
5472 if (subopts_value == NULL) {
5473 fprintf(stderr, "Missing ssl_chain_cert argument\n");
5474 return 1;
5475 }
5476 settings.ssl_chain_cert = strdup(subopts_value);
5477 break;
5478 case SSL_KEY:
5479 if (subopts_value == NULL) {
5480 fprintf(stderr, "Missing ssl_key argument\n");
5481 return 1;
5482 }
5483 settings.ssl_key = strdup(subopts_value);
5484 break;
5485 case SSL_VERIFY_MODE:
5486 {
5487 if (subopts_value == NULL) {
5488 fprintf(stderr, "Missing ssl_verify_mode argument\n");
5489 return 1;
5490 }
5491 int verify = 0;
5492 if (!safe_strtol(subopts_value, &verify)) {
5493 fprintf(stderr, "could not parse argument to ssl_verify_mode\n");
5494 return 1;
5495 }
5496 if (!ssl_set_verify_mode(verify)) {
5497 fprintf(stderr, "Invalid ssl_verify_mode. Use help to see valid options.\n");
5498 return 1;
5499 }
5500 break;
5501 }
5502 case SSL_KEYFORM:
5503 if (subopts_value == NULL) {
5504 fprintf(stderr, "Missing ssl_keyformat argument\n");
5505 return 1;
5506 }
5507 if (!safe_strtol(subopts_value, &settings.ssl_keyformat)) {
5508 fprintf(stderr, "could not parse argument to ssl_keyformat\n");
5509 return 1;
5510 }
5511 break;
5512 case SSL_CIPHERS:
5513 if (subopts_value == NULL) {
5514 fprintf(stderr, "Missing ssl_ciphers argument\n");
5515 return 1;
5516 }
5517 settings.ssl_ciphers = strdup(subopts_value);
5518 break;
5519 case SSL_CA_CERT:
5520 if (subopts_value == NULL) {
5521 fprintf(stderr, "Missing ssl_ca_cert argument\n");
5522 return 1;
5523 }
5524 settings.ssl_ca_cert = strdup(subopts_value);
5525 break;
5526 case SSL_WBUF_SIZE:
5527 if (subopts_value == NULL) {
5528 fprintf(stderr, "Missing ssl_wbuf_size argument\n");
5529 return 1;
5530 }
5531 if (!safe_strtoul(subopts_value, &settings.ssl_wbuf_size)) {
5532 fprintf(stderr, "could not parse argument to ssl_wbuf_size\n");
5533 return 1;
5534 }
5535 settings.ssl_wbuf_size *= 1024; /* kilobytes */
5536 break;
5537 case SSL_SESSION_CACHE:
5538 settings.ssl_session_cache = true;
5539 break;
5540 case SSL_KERNEL_TLS:
5541 settings.ssl_kernel_tls = true;
5542 break;
5543 case SSL_MIN_VERSION: {
5544 int min_version;
5545 if (subopts_value == NULL) {
5546 fprintf(stderr, "Missing ssl_min_version argument\n");
5547 return 1;
5548 }
5549 if (!safe_strtol(subopts_value, &min_version)) {
5550 fprintf(stderr, "could not parse argument to ssl_min_version\n");
5551 return 1;
5552 }
5553 if (!ssl_set_min_version(min_version)) {
5554 fprintf(stderr, "Invalid ssl_min_version. Use help to see valid options.\n");
5555 return 1;
5556 }
5557 break;
5558 }
5559 #endif
5560 case MODERN:
5561 /* currently no new defaults */
5562 break;
5563 case NO_MODERN:
5564 if (!slab_chunk_size_changed) {
5565 settings.slab_chunk_size_max = settings.slab_page_size;
5566 }
5567 settings.slab_reassign = false;
5568 settings.slab_automove = 0;
5569 settings.maxconns_fast = false;
5570 settings.lru_segmented = false;
5571 hash_type = JENKINS_HASH;
5572 start_lru_crawler = false;
5573 start_lru_maintainer = false;
5574 break;
5575 case NO_DROP_PRIVILEGES:
5576 settings.drop_privileges = false;
5577 break;
5578 case DROP_PRIVILEGES:
5579 settings.drop_privileges = true;
5580 break;
5581 case RESP_OBJ_MEM_LIMIT:
5582 // TODO: Remove at some point in the future.
5583 fprintf(stderr, "DEPRECATED: resp_obj_mem_limit no longer used. See read_buf_mem_limit,\n");
5584 break;
5585 case READ_BUF_MEM_LIMIT:
5586 if (subopts_value == NULL) {
5587 fprintf(stderr, "Missing read_buf_mem_limit argument\n");
5588 return 1;
5589 }
5590 if (!safe_strtoul(subopts_value, &settings.read_buf_mem_limit)) {
5591 fprintf(stderr, "could not parse argument to read_buf_mem_limit\n");
5592 return 1;
5593 }
5594 settings.read_buf_mem_limit *= 1024 * 1024; /* megabytes */
5595 break;
5596 #ifdef PROXY
5597 case PROXY_CONFIG:
5598 if (subopts_value == NULL) {
5599 fprintf(stderr, "Missing proxy_config file argument\n");
5600 return 1;
5601 }
5602 if (protocol_specified) {
5603 fprintf(stderr, "Cannot specify a protocol with proxy mode enabled\n");
5604 return 1;
5605 }
5606 settings.proxy_startfile = strdup(subopts_value);
5607 settings.proxy_enabled = true;
5608 settings.binding_protocol = proxy_prot;
5609 protocol_specified = true;
5610 break;
5611 case PROXY_ARG:
5612 settings.proxy_startarg = strdup(subopts_value);
5613 break;
5614 case PROXY_URING:
5615 settings.proxy_uring = true;
5616 break;
5617 case PROXY_MEMPROFILE:
5618 settings.proxy_memprofile = true;
5619 break;
5620 #endif
5621 #ifdef MEMCACHED_DEBUG
5622 case RELAXED_PRIVILEGES:
5623 settings.relaxed_privileges = true;
5624 break;
5625 #endif
5626 #ifdef SOCK_COOKIE_ID
5627 case COOKIE_ID:
5628 (void)safe_strtoul(subopts_value, &settings.sock_cookie_id);
5629 break;
5630 #endif
5631 default:
5632 #ifdef EXTSTORE
5633 // TODO: differentiating response code.
5634 if (storage_read_config(storage_cf, &subopts_temp)) {
5635 return 1;
5636 }
5637 #else
5638 printf("Illegal suboption \"%s\"\n", subopts_temp);
5639 return 1;
5640 #endif
5641 } // switch
5642 if (subopts_temp_o) {
5643 free(subopts_temp_o);
5644 }
5645
5646 } // while
5647 free(subopts_orig);
5648 break;
5649 default:
5650 fprintf(stderr, "Illegal argument \"%c\"\n", c);
5651 return 1;
5652 }
5653 }
5654
5655 if (settings.num_napi_ids > settings.num_threads) {
5656 fprintf(stderr, "Number of napi_ids(%d) cannot be greater than number of threads(%d)\n",
5657 settings.num_napi_ids, settings.num_threads);
5658 exit(EX_USAGE);
5659 }
5660
5661 if (settings.item_size_max < ITEM_SIZE_MAX_LOWER_LIMIT) {
5662 fprintf(stderr, "Item max size cannot be less than 1024 bytes.\n");
5663 exit(EX_USAGE);
5664 }
5665 if (settings.item_size_max > (settings.maxbytes / 2)) {
5666 fprintf(stderr, "Cannot set item size limit higher than 1/2 of memory max.\n");
5667 exit(EX_USAGE);
5668 }
5669 if (settings.item_size_max > (ITEM_SIZE_MAX_UPPER_LIMIT)) {
5670 fprintf(stderr, "Cannot set item size limit higher than a gigabyte.\n");
5671 exit(EX_USAGE);
5672 }
5673 if (settings.item_size_max > 1024 * 1024) {
5674 if (!slab_chunk_size_changed) {
5675 // Ideal new default is 16k, but needs stitching.
5676 settings.slab_chunk_size_max = settings.slab_page_size / 2;
5677 }
5678 }
5679
5680 if (settings.slab_chunk_size_max > settings.item_size_max) {
5681 fprintf(stderr, "slab_chunk_max (bytes: %d) cannot be larger than -I (item_size_max %d)\n",
5682 settings.slab_chunk_size_max, settings.item_size_max);
5683 exit(EX_USAGE);
5684 }
5685
5686 if (settings.item_size_max % settings.slab_chunk_size_max != 0) {
5687 fprintf(stderr, "-I (item_size_max: %d) must be evenly divisible by slab_chunk_max (bytes: %d)\n",
5688 settings.item_size_max, settings.slab_chunk_size_max);
5689 exit(EX_USAGE);
5690 }
5691
5692 if (settings.slab_page_size % settings.slab_chunk_size_max != 0) {
5693 fprintf(stderr, "slab_chunk_max (bytes: %d) must divide evenly into %d (slab_page_size)\n",
5694 settings.slab_chunk_size_max, settings.slab_page_size);
5695 exit(EX_USAGE);
5696 }
5697 #ifdef EXTSTORE
5698 switch (storage_check_config(storage_cf)) {
5699 case 0:
5700 storage_enabled = true;
5701 break;
5702 case 1:
5703 exit(EX_USAGE);
5704 break;
5705 }
5706 #endif
5707 // Reserve this for the new default. If factor size hasn't changed, use
5708 // new default.
5709 /*if (settings.slab_chunk_size_max == 16384 && settings.factor == 1.25) {
5710 settings.factor = 1.08;
5711 }*/
5712
5713 if (slab_sizes_unparsed != NULL) {
5714 // want the unedited string for restart code.
5715 char *temp = strdup(slab_sizes_unparsed);
5716 if (_parse_slab_sizes(slab_sizes_unparsed, slab_sizes)) {
5717 use_slab_sizes = true;
5718 if (meta->slab_config) {
5719 free(meta->slab_config);
5720 }
5721 meta->slab_config = temp;
5722 } else {
5723 exit(EX_USAGE);
5724 }
5725 } else if (!meta->slab_config) {
5726 // using the default factor.
5727 meta->slab_config = "1.25";
5728 }
5729
5730 if (settings.hot_lru_pct + settings.warm_lru_pct > 80) {
5731 fprintf(stderr, "hot_lru_pct + warm_lru_pct cannot be more than 80%% combined\n");
5732 exit(EX_USAGE);
5733 }
5734
5735 if (settings.temp_lru && !start_lru_maintainer) {
5736 fprintf(stderr, "temporary_ttl requires lru_maintainer to be enabled\n");
5737 exit(EX_USAGE);
5738 }
5739
5740 if (hash_init(hash_type) != 0) {
5741 fprintf(stderr, "Failed to initialize hash_algorithm!\n");
5742 exit(EX_USAGE);
5743 }
5744
5745 /*
5746 * Use one workerthread to serve each UDP port if the user specified
5747 * multiple ports
5748 */
5749 if (settings.inter != NULL && strchr(settings.inter, ',')) {
5750 settings.num_threads_per_udp = 1;
5751 } else {
5752 settings.num_threads_per_udp = settings.num_threads;
5753 }
5754
5755 if (settings.sasl) {
5756 if (!protocol_specified) {
5757 settings.binding_protocol = binary_prot;
5758 } else {
5759 if (settings.binding_protocol != binary_prot) {
5760 fprintf(stderr, "ERROR: You cannot allow the ASCII protocol while using SASL.\n");
5761 exit(EX_USAGE);
5762 }
5763 }
5764
5765 if (settings.udpport) {
5766 fprintf(stderr, "ERROR: Cannot enable UDP while using binary SASL authentication.\n");
5767 exit(EX_USAGE);
5768 }
5769 }
5770
5771 if (settings.auth_file) {
5772 if (!protocol_specified) {
5773 settings.binding_protocol = ascii_prot;
5774 } else {
5775 if (settings.binding_protocol != ascii_prot) {
5776 fprintf(stderr, "ERROR: You cannot allow the BINARY protocol while using ascii authentication tokens.\n");
5777 exit(EX_USAGE);
5778 }
5779 }
5780 }
5781
5782 if (udp_specified && settings.udpport != 0 && !tcp_specified) {
5783 settings.port = settings.udpport;
5784 }
5785
5786 if (settings.port > 65535) {
5787 fprintf(stderr, "ERROR: Invalid port number %d.\n", settings.port);
5788 exit(EX_USAGE);
5789 }
5790
5791
5792 /*
5793 * Setup SSL if enabled
5794 */
5795 if (settings.ssl_enabled) {
5796 if (!settings.port) {
5797 fprintf(stderr, "ERROR: You cannot enable SSL without a TCP port.\n");
5798 exit(EX_USAGE);
5799 }
5800 // Initiate the SSL context.
5801 ssl_init();
5802 }
5803
5804 if (maxcore != 0) {
5805 struct rlimit rlim_new;
5806 /*
5807 * First try raising to infinity; if that fails, try bringing
5808 * the soft limit to the hard.
5809 */
5810 if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
5811 rlim_new.rlim_cur = rlim_new.rlim_max = RLIM_INFINITY;
5812 if (setrlimit(RLIMIT_CORE, &rlim_new)!= 0) {
5813 /* failed. try raising just to the old max */
5814 rlim_new.rlim_cur = rlim_new.rlim_max = rlim.rlim_max;
5815 (void)setrlimit(RLIMIT_CORE, &rlim_new);
5816 }
5817 }
5818 /*
5819 * getrlimit again to see what we ended up with. Only fail if
5820 * the soft limit ends up 0, because then no core files will be
5821 * created at all.
5822 */
5823
5824 if ((getrlimit(RLIMIT_CORE, &rlim) != 0) || rlim.rlim_cur == 0) {
5825 fprintf(stderr, "failed to ensure corefile creation\n");
5826 exit(EX_OSERR);
5827 }
5828 }
5829
5830 /*
5831 * If needed, increase rlimits to allow as many connections
5832 * as needed.
5833 */
5834
5835 if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
5836 fprintf(stderr, "failed to getrlimit number of files\n");
5837 exit(EX_OSERR);
5838 } else {
5839 rlim.rlim_cur = settings.maxconns;
5840 rlim.rlim_max = settings.maxconns;
5841 if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) {
5842 #ifndef MEMCACHED_DEBUG
5843 fprintf(stderr, "failed to set rlimit for open files. Try starting as root or requesting smaller maxconns value.\n");
5844 exit(EX_OSERR);
5845 #endif
5846 }
5847 }
5848
5849 /* lose root privileges if we have them */
5850 if (getuid() == 0 || geteuid() == 0) {
5851 if (username == 0 || *username == '\0') {
5852 fprintf(stderr, "can't run as root without the -u switch\n");
5853 exit(EX_USAGE);
5854 }
5855 if ((pw = getpwnam(username)) == 0) {
5856 fprintf(stderr, "can't find the user %s to switch to\n", username);
5857 exit(EX_NOUSER);
5858 }
5859 if (setgroups(0, NULL) < 0) {
5860 /* setgroups may fail with EPERM, indicating we are already in a
5861 * minimally-privileged state. In that case we continue. For all
5862 * other failure codes we exit.
5863 *
5864 * Note that errno is stored here because fprintf may change it.
5865 */
5866 bool should_exit = errno != EPERM;
5867 fprintf(stderr, "failed to drop supplementary groups: %s\n",
5868 strerror(errno));
5869 if (should_exit) {
5870 exit(EX_OSERR);
5871 }
5872 }
5873 if (setgid(pw->pw_gid) < 0 || setuid(pw->pw_uid) < 0) {
5874 fprintf(stderr, "failed to assume identity of user %s\n", username);
5875 exit(EX_OSERR);
5876 }
5877 }
5878
5879 /* Initialize Sasl if -S was specified */
5880 if (settings.sasl) {
5881 init_sasl();
5882 }
5883
5884 /* daemonize if requested */
5885 /* if we want to ensure our ability to dump core, don't chdir to / */
5886 if (do_daemonize) {
5887 if (daemonize(maxcore, settings.verbose) == -1) {
5888 fprintf(stderr, "failed to daemon() in order to daemonize\n");
5889 exit(EXIT_FAILURE);
5890 }
5891 }
5892
5893 /* lock paged memory if needed */
5894 if (lock_memory) {
5895 #ifdef HAVE_MLOCKALL
5896 int res = mlockall(MCL_CURRENT | MCL_FUTURE);
5897 if (res != 0) {
5898 fprintf(stderr, "warning: -k invalid, mlockall() failed: %s\n",
5899 strerror(errno));
5900 }
5901 #else
5902 fprintf(stderr, "warning: -k invalid, mlockall() not supported on this platform. proceeding without.\n");
5903 #endif
5904 }
5905
5906 /* initialize main thread libevent instance */
5907 #if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= 0x02000101
5908 /* If libevent version is larger/equal to 2.0.2-alpha, use newer version */
5909 struct event_config *ev_config;
5910 ev_config = event_config_new();
5911 event_config_set_flag(ev_config, EVENT_BASE_FLAG_NOLOCK);
5912 main_base = event_base_new_with_config(ev_config);
5913 event_config_free(ev_config);
5914 #else
5915 /* Otherwise, use older API */
5916 main_base = event_init();
5917 #endif
5918
5919 /* Load initial auth file if required */
5920 if (settings.auth_file) {
5921 if (settings.udpport) {
5922 fprintf(stderr, "Cannot use UDP with ascii authentication enabled (-U 0 to disable)\n");
5923 exit(EX_USAGE);
5924 }
5925
5926 switch (authfile_load(settings.auth_file)) {
5927 case AUTHFILE_STATFAIL:
5928 vperror("Could not stat authfile [%s], error %s", settings.auth_file
5929 , strerror(errno));
5930 exit(EXIT_FAILURE);
5931 break;
5932 case AUTHFILE_OPENFAIL:
5933 vperror("Could not open authfile [%s] for reading, error %s", settings.auth_file
5934 , strerror(errno));
5935 exit(EXIT_FAILURE);
5936 break;
5937 case AUTHFILE_OOM:
5938 fprintf(stderr, "Out of memory reading password file: %s", settings.auth_file);
5939 exit(EXIT_FAILURE);
5940 break;
5941 case AUTHFILE_MALFORMED:
5942 fprintf(stderr, "Authfile [%s] has a malformed entry. Should be 'user:password'", settings.auth_file);
5943 exit(EXIT_FAILURE);
5944 break;
5945 case AUTHFILE_OK:
5946 break;
5947 }
5948 }
5949
5950 /* initialize other stuff */
5951 stats_init();
5952 logger_init();
5953 logger_create(); // main process logger
5954 conn_init();
5955 bool reuse_mem = false;
5956 void *mem_base = NULL;
5957 bool prefill = false;
5958 if (settings.memory_file != NULL) {
5959 preallocate = true;
5960 // Easier to manage memory if we prefill the global pool when reusing.
5961 prefill = true;
5962 restart_register("main", _mc_meta_load_cb, _mc_meta_save_cb, meta);
5963 reuse_mem = restart_mmap_open(settings.maxbytes,
5964 settings.memory_file,
5965 &mem_base);
5966 // The "save" callback gets called when we're closing out the mmap,
5967 // but we don't know what the mmap_base is until after we call open.
5968 // So we pass the struct above but have to fill it in here so the
5969 // data's available during the save routine.
5970 meta->mmap_base = mem_base;
5971 // Also, the callbacks for load() run before _open returns, so we
5972 // should have the old base in 'meta' as of here.
5973 }
5974 // Initialize the hash table _after_ checking restart metadata.
5975 // We override the hash table start argument with what was live
5976 // previously, to avoid filling a huge set of items into a tiny hash
5977 // table.
5978 assoc_init(settings.hashpower_init);
5979 #ifdef EXTSTORE
5980 if (storage_enabled && reuse_mem) {
5981 fprintf(stderr, "[restart] memory restart with extstore not presently supported.\n");
5982 reuse_mem = false;
5983 }
5984 #endif
5985 slabs_init(settings.maxbytes, settings.factor, preallocate,
5986 use_slab_sizes ? slab_sizes : NULL, mem_base, reuse_mem);
5987 #ifdef EXTSTORE
5988 if (storage_enabled) {
5989 storage = storage_init(storage_cf);
5990 if (storage == NULL) {
5991 exit(EXIT_FAILURE);
5992 }
5993 ext_storage = storage;
5994 /* page mover algorithm for extstore needs memory prefilled */
5995 prefill = true;
5996 }
5997 #endif
5998
5999 if (settings.drop_privileges) {
6000 setup_privilege_violations_handler();
6001 }
6002
6003 if (prefill)
6004 slabs_prefill_global();
6005 /* In restartable mode and we've decided to issue a fixup on memory */
6006 if (settings.memory_file != NULL && reuse_mem) {
6007 mc_ptr_t old_base = meta->old_base;
6008 assert(old_base == meta->old_base);
6009
6010 // should've pulled in process_started from meta file.
6011 process_started = meta->process_started;
6012 // TODO: must be a more canonical way of serializing/deserializing
6013 // pointers? passing through uint64_t should work, and we're not
6014 // annotating the pointer with anything, but it's still slightly
6015 // insane.
6016 restart_fixup((void *)old_base);
6017 }
6018 /*
6019 * ignore SIGPIPE signals; we can use errno == EPIPE if we
6020 * need that information
6021 */
6022 if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) {
6023 perror("failed to ignore SIGPIPE; sigaction");
6024 exit(EX_OSERR);
6025 }
6026 /* start up worker threads if MT mode */
6027 #ifdef PROXY
6028 if (settings.proxy_enabled) {
6029 settings.proxy_ctx = proxy_init(settings.proxy_uring, settings.proxy_memprofile);
6030 }
6031 #endif
6032 #ifdef EXTSTORE
6033 slabs_set_storage(storage);
6034 memcached_thread_init(settings.num_threads, storage);
6035 init_lru_crawler(storage);
6036 #else
6037 memcached_thread_init(settings.num_threads, NULL);
6038 init_lru_crawler(NULL);
6039 #endif
6040
6041 #ifdef PROXY
6042 if (settings.proxy_enabled) {
6043 if (proxy_first_confload(settings.proxy_ctx) != 0) {
6044 exit(EXIT_FAILURE);
6045 }
6046 }
6047 #endif
6048
6049 if (start_assoc_maint && start_assoc_maintenance_thread() == -1) {
6050 exit(EXIT_FAILURE);
6051 }
6052 if (start_lru_crawler && start_item_crawler_thread() != 0) {
6053 fprintf(stderr, "Failed to enable LRU crawler thread\n");
6054 exit(EXIT_FAILURE);
6055 }
6056 #ifdef EXTSTORE
6057 if (storage && start_storage_compact_thread(storage) != 0) {
6058 fprintf(stderr, "Failed to start storage compaction thread\n");
6059 exit(EXIT_FAILURE);
6060 }
6061 if (storage && start_storage_write_thread(storage) != 0) {
6062 fprintf(stderr, "Failed to start storage writer thread\n");
6063 exit(EXIT_FAILURE);
6064 }
6065
6066 if (start_lru_maintainer && start_lru_maintainer_thread(storage) != 0) {
6067 #else
6068 if (start_lru_maintainer && start_lru_maintainer_thread(NULL) != 0) {
6069 #endif
6070 fprintf(stderr, "Failed to enable LRU maintainer thread\n");
6071 free(meta);
6072 return 1;
6073 }
6074
6075 if (settings.slab_reassign &&
6076 start_slab_maintenance_thread() == -1) {
6077 exit(EXIT_FAILURE);
6078 }
6079
6080 if (settings.idle_timeout && start_conn_timeout_thread() == -1) {
6081 exit(EXIT_FAILURE);
6082 }
6083
6084 /* initialise clock event */
6085 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
6086 {
6087 struct timespec ts;
6088 if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) {
6089 monotonic = true;
6090 monotonic_start = ts.tv_sec;
6091 // Monotonic clock needs special handling for restarts.
6092 // We get a start time at an arbitrary place, so we need to
6093 // restore the original time delta, which is always "now" - _start
6094 if (reuse_mem) {
6095 // the running timespan at stop time + the time we think we
6096 // were stopped.
6097 monotonic_start -= meta->current_time + meta->time_delta;
6098 } else {
6099 monotonic_start -= ITEM_UPDATE_INTERVAL + 2;
6100 }
6101 }
6102 }
6103 #endif
6104 clock_handler(0, 0, 0);
6105
6106 /* create unix mode sockets after dropping privileges */
6107 if (settings.socketpath != NULL) {
6108 errno = 0;
6109 if (server_socket_unix(settings.socketpath,settings.access)) {
6110 vperror("failed to listen on UNIX socket: %s", settings.socketpath);
6111 exit(EX_OSERR);
6112 }
6113 }
6114
6115 /* create the listening socket, bind it, and init */
6116 if (settings.socketpath == NULL) {
6117 const char *portnumber_filename = getenv("MEMCACHED_PORT_FILENAME");
6118 char *temp_portnumber_filename = NULL;
6119 size_t len;
6120 FILE *portnumber_file = NULL;
6121
6122 if (portnumber_filename != NULL) {
6123 len = strlen(portnumber_filename)+4+1;
6124 temp_portnumber_filename = malloc(len);
6125 if (temp_portnumber_filename == NULL) {
6126 vperror("Failed to allocate memory for portnumber file");
6127 exit(EX_OSERR);
6128 }
6129 snprintf(temp_portnumber_filename,
6130 len,
6131 "%s.lck", portnumber_filename);
6132
6133 portnumber_file = fopen(temp_portnumber_filename, "a");
6134 if (portnumber_file == NULL) {
6135 fprintf(stderr, "Failed to open \"%s\": %s\n",
6136 temp_portnumber_filename, strerror(errno));
6137 }
6138 }
6139
6140 errno = 0;
6141 if (settings.port && server_sockets(settings.port, tcp_transport,
6142 portnumber_file)) {
6143 if (settings.inter == NULL) {
6144 vperror("failed to listen on TCP port %d", settings.port);
6145 } else {
6146 vperror("failed to listen on one of interface(s) %s", settings.inter);
6147 }
6148 exit(EX_OSERR);
6149 }
6150
6151 /*
6152 * initialization order: first create the listening sockets
6153 * (may need root on low ports), then drop root if needed,
6154 * then daemonize if needed, then init libevent (in some cases
6155 * descriptors created by libevent wouldn't survive forking).
6156 */
6157
6158 /* create the UDP listening socket and bind it */
6159 errno = 0;
6160 if (settings.udpport && server_sockets(settings.udpport, udp_transport,
6161 portnumber_file)) {
6162 if (settings.inter == NULL) {
6163 vperror("failed to listen on UDP port %d", settings.udpport);
6164 } else {
6165 vperror("failed to listen on one of interface(s) %s", settings.inter);
6166 }
6167 exit(EX_OSERR);
6168 }
6169
6170 if (portnumber_file) {
6171 fclose(portnumber_file);
6172 rename(temp_portnumber_filename, portnumber_filename);
6173 }
6174 if (temp_portnumber_filename)
6175 free(temp_portnumber_filename);
6176 }
6177
6178 /* Give the sockets a moment to open. I know this is dumb, but the error
6179 * is only an advisory.
6180 */
6181 usleep(1000);
6182 if (stats_state.curr_conns + stats_state.reserved_fds >= settings.maxconns - 1) {
6183 fprintf(stderr, "Maxconns setting is too low, use -c to increase.\n");
6184 exit(EXIT_FAILURE);
6185 }
6186
6187 if (pid_file != NULL) {
6188 save_pid(pid_file);
6189 }
6190
6191 /* Drop privileges no longer needed */
6192 if (settings.drop_privileges) {
6193 drop_privileges();
6194 }
6195
6196 /* Initialize the uriencode lookup table. */
6197 uriencode_init();
6198
6199 /* enter the event loop */
6200 while (!stop_main_loop) {
6201 if (event_base_loop(main_base, EVLOOP_ONCE) != 0) {
6202 retval = EXIT_FAILURE;
6203 break;
6204 }
6205 }
6206
6207 switch (stop_main_loop) {
6208 case GRACE_STOP:
6209 fprintf(stderr, "Gracefully stopping\n");
6210 break;
6211 case EXIT_NORMALLY:
6212 // Don't need to print anything to STDERR for a normal shutdown except
6213 // if we want to.
6214
6215 if (settings.verbose) {
6216 fprintf(stderr, "Exiting normally\n");
6217 }
6218
6219 break;
6220 default:
6221 fprintf(stderr, "Exiting on error\n");
6222 break;
6223 }
6224
6225 if (stop_main_loop == GRACE_STOP) {
6226 stop_threads();
6227 if (settings.memory_file != NULL) {
6228 restart_mmap_close();
6229 }
6230 }
6231
6232 /* remove the PID file if we're a daemon */
6233 if (do_daemonize)
6234 remove_pidfile(pid_file);
6235 /* Clean up strdup() call for bind() address */
6236 if (settings.inter)
6237 free(settings.inter);
6238
6239 /* cleanup base */
6240 event_base_free(main_base);
6241
6242 free(meta);
6243
6244 return retval;
6245 }
6246