Lines Matching refs:it
25 static void item_link_q(item *it);
26 static void item_unlink_q(item *it);
94 item *it; member
104 static bool lru_bump_async(lru_bump_buf *b, item *it, uint32_t hv);
122 int item_is_flushed(item *it) { in item_is_flushed() argument
124 if (it->time <= oldest_live && oldest_live <= current_time) in item_is_flushed()
137 # define DEBUG_REFCNT(it,op) \
139 it, op, it->refcount, \
140 (it->it_flags & ITEM_LINKED) ? 'L' : ' ', \
141 (it->it_flags & ITEM_SLABBED) ? 'S' : ' ')
143 # define DEBUG_REFCNT(it,op) while(0) argument
168 item *it = NULL; in do_item_alloc_pull() local
181 it = slabs_alloc(ntotal, id, 0); in do_item_alloc_pull()
183 if (it == NULL) { in do_item_alloc_pull()
205 return it; in do_item_alloc_pull()
257 item *it = NULL; in do_item_alloc() local
293 it = do_item_alloc_pull(htotal, hdr_id); in do_item_alloc()
295 if (it != NULL) in do_item_alloc()
296 it->it_flags |= ITEM_CHUNKED; in do_item_alloc()
298 it = do_item_alloc_pull(ntotal, id); in do_item_alloc()
301 if (it == NULL) { in do_item_alloc()
308 assert(it->it_flags == 0 || it->it_flags == ITEM_CHUNKED); in do_item_alloc()
312 it->next = it->prev = 0; in do_item_alloc()
326 it->slabs_clsid = id; in do_item_alloc()
328 DEBUG_REFCNT(it, '*'); in do_item_alloc()
329 it->it_flags |= settings.use_cas ? ITEM_CAS : 0; in do_item_alloc()
330 it->it_flags |= nsuffix != 0 ? ITEM_CFLAGS : 0; in do_item_alloc()
331 it->nkey = nkey; in do_item_alloc()
332 it->nbytes = nbytes; in do_item_alloc()
333 memcpy(ITEM_key(it), key, nkey); in do_item_alloc()
334 it->exptime = exptime; in do_item_alloc()
336 memcpy(ITEM_suffix(it), &flags, sizeof(flags)); in do_item_alloc()
340 if (it->it_flags & ITEM_CHUNKED) { in do_item_alloc()
341 item_chunk *chunk = (item_chunk *) ITEM_schunk(it); in do_item_alloc()
347 chunk->head = it; in do_item_alloc()
350 it->h_next = 0; in do_item_alloc()
352 return it; in do_item_alloc()
355 void item_free(item *it) { in item_free() argument
356 size_t ntotal = ITEM_ntotal(it); in item_free()
358 assert((it->it_flags & ITEM_LINKED) == 0); in item_free()
359 assert(it != heads[it->slabs_clsid]); in item_free()
360 assert(it != tails[it->slabs_clsid]); in item_free()
361 assert(it->refcount == 0); in item_free()
364 clsid = ITEM_clsid(it); in item_free()
365 DEBUG_REFCNT(it, 'F'); in item_free()
366 slabs_free(it, ntotal, clsid); in item_free()
389 void do_item_link_fixup(item *it) { in do_item_link_fixup() argument
391 int ntotal = ITEM_ntotal(it); in do_item_link_fixup()
392 uint32_t hv = hash(ITEM_key(it), it->nkey); in do_item_link_fixup()
393 assoc_insert(it, hv); in do_item_link_fixup()
395 head = &heads[it->slabs_clsid]; in do_item_link_fixup()
396 tail = &tails[it->slabs_clsid]; in do_item_link_fixup()
397 if (it->prev == 0 && *head == 0) *head = it; in do_item_link_fixup()
398 if (it->next == 0 && *tail == 0) *tail = it; in do_item_link_fixup()
399 sizes[it->slabs_clsid]++; in do_item_link_fixup()
400 sizes_bytes[it->slabs_clsid] += ntotal; in do_item_link_fixup()
408 item_stats_sizes_add(it); in do_item_link_fixup()
413 static void do_item_link_q(item *it) { /* item is the new head */ in do_item_link_q() argument
415 assert((it->it_flags & ITEM_SLABBED) == 0); in do_item_link_q()
417 head = &heads[it->slabs_clsid]; in do_item_link_q()
418 tail = &tails[it->slabs_clsid]; in do_item_link_q()
419 assert(it != *head); in do_item_link_q()
421 it->prev = 0; in do_item_link_q()
422 it->next = *head; in do_item_link_q()
423 if (it->next) it->next->prev = it; in do_item_link_q()
424 *head = it; in do_item_link_q()
425 if (*tail == 0) *tail = it; in do_item_link_q()
426 sizes[it->slabs_clsid]++; in do_item_link_q()
428 if (it->it_flags & ITEM_HDR) { in do_item_link_q()
429 sizes_bytes[it->slabs_clsid] += (ITEM_ntotal(it) - it->nbytes) + sizeof(item_hdr); in do_item_link_q()
431 sizes_bytes[it->slabs_clsid] += ITEM_ntotal(it); in do_item_link_q()
434 sizes_bytes[it->slabs_clsid] += ITEM_ntotal(it); in do_item_link_q()
440 static void item_link_q(item *it) { in item_link_q() argument
441 pthread_mutex_lock(&lru_locks[it->slabs_clsid]); in item_link_q()
442 do_item_link_q(it); in item_link_q()
443 pthread_mutex_unlock(&lru_locks[it->slabs_clsid]); in item_link_q()
446 static void item_link_q_warm(item *it) { in item_link_q_warm() argument
447 pthread_mutex_lock(&lru_locks[it->slabs_clsid]); in item_link_q_warm()
448 do_item_link_q(it); in item_link_q_warm()
449 itemstats[it->slabs_clsid].moves_to_warm++; in item_link_q_warm()
450 pthread_mutex_unlock(&lru_locks[it->slabs_clsid]); in item_link_q_warm()
453 static void do_item_unlink_q(item *it) { in do_item_unlink_q() argument
455 head = &heads[it->slabs_clsid]; in do_item_unlink_q()
456 tail = &tails[it->slabs_clsid]; in do_item_unlink_q()
458 if (*head == it) { in do_item_unlink_q()
459 assert(it->prev == 0); in do_item_unlink_q()
460 *head = it->next; in do_item_unlink_q()
462 if (*tail == it) { in do_item_unlink_q()
463 assert(it->next == 0); in do_item_unlink_q()
464 *tail = it->prev; in do_item_unlink_q()
466 assert(it->next != it); in do_item_unlink_q()
467 assert(it->prev != it); in do_item_unlink_q()
469 if (it->next) it->next->prev = it->prev; in do_item_unlink_q()
470 if (it->prev) it->prev->next = it->next; in do_item_unlink_q()
471 sizes[it->slabs_clsid]--; in do_item_unlink_q()
473 if (it->it_flags & ITEM_HDR) { in do_item_unlink_q()
474 sizes_bytes[it->slabs_clsid] -= (ITEM_ntotal(it) - it->nbytes) + sizeof(item_hdr); in do_item_unlink_q()
476 sizes_bytes[it->slabs_clsid] -= ITEM_ntotal(it); in do_item_unlink_q()
479 sizes_bytes[it->slabs_clsid] -= ITEM_ntotal(it); in do_item_unlink_q()
485 static void item_unlink_q(item *it) { in item_unlink_q() argument
486 pthread_mutex_lock(&lru_locks[it->slabs_clsid]); in item_unlink_q()
487 do_item_unlink_q(it); in item_unlink_q()
488 pthread_mutex_unlock(&lru_locks[it->slabs_clsid]); in item_unlink_q()
491 int do_item_link(item *it, const uint32_t hv, const uint64_t cas) { in do_item_link() argument
492 MEMCACHED_ITEM_LINK(ITEM_key(it), it->nkey, it->nbytes); in do_item_link()
493 assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0); in do_item_link()
494 it->it_flags |= ITEM_LINKED; in do_item_link()
495 it->time = current_time; in do_item_link()
498 stats_state.curr_bytes += ITEM_ntotal(it); in do_item_link()
504 ITEM_set_cas(it, cas); in do_item_link()
505 assoc_insert(it, hv); in do_item_link()
506 item_link_q(it); in do_item_link()
507 refcount_incr(it); in do_item_link()
508 item_stats_sizes_add(it); in do_item_link()
513 void do_item_unlink(item *it, const uint32_t hv) { in do_item_unlink() argument
514 MEMCACHED_ITEM_UNLINK(ITEM_key(it), it->nkey, it->nbytes); in do_item_unlink()
515 if ((it->it_flags & ITEM_LINKED) != 0) { in do_item_unlink()
516 it->it_flags &= ~ITEM_LINKED; in do_item_unlink()
518 stats_state.curr_bytes -= ITEM_ntotal(it); in do_item_unlink()
521 item_stats_sizes_remove(it); in do_item_unlink()
522 assoc_delete(ITEM_key(it), it->nkey, hv); in do_item_unlink()
523 item_unlink_q(it); in do_item_unlink()
524 do_item_remove(it); in do_item_unlink()
529 void do_item_unlink_nolock(item *it, const uint32_t hv) { in do_item_unlink_nolock() argument
530 MEMCACHED_ITEM_UNLINK(ITEM_key(it), it->nkey, it->nbytes); in do_item_unlink_nolock()
531 if ((it->it_flags & ITEM_LINKED) != 0) { in do_item_unlink_nolock()
532 it->it_flags &= ~ITEM_LINKED; in do_item_unlink_nolock()
534 stats_state.curr_bytes -= ITEM_ntotal(it); in do_item_unlink_nolock()
537 item_stats_sizes_remove(it); in do_item_unlink_nolock()
538 assoc_delete(ITEM_key(it), it->nkey, hv); in do_item_unlink_nolock()
539 do_item_unlink_q(it); in do_item_unlink_nolock()
540 do_item_remove(it); in do_item_unlink_nolock()
544 void do_item_remove(item *it) { in do_item_remove() argument
545 MEMCACHED_ITEM_REMOVE(ITEM_key(it), it->nkey, it->nbytes); in do_item_remove()
546 assert((it->it_flags & ITEM_SLABBED) == 0); in do_item_remove()
547 assert(it->refcount > 0); in do_item_remove()
549 if (refcount_decr(it) == 0) { in do_item_remove()
550 item_free(it); in do_item_remove()
555 void do_item_update(item *it) { in do_item_update() argument
556 MEMCACHED_ITEM_UPDATE(ITEM_key(it), it->nkey, it->nbytes); in do_item_update()
560 assert((it->it_flags & ITEM_SLABBED) == 0); in do_item_update()
561 if ((it->it_flags & ITEM_LINKED) != 0) { in do_item_update()
562 if (ITEM_lruid(it) == COLD_LRU && (it->it_flags & ITEM_ACTIVE)) { in do_item_update()
563 it->time = current_time; in do_item_update()
564 item_unlink_q(it); in do_item_update()
565 it->slabs_clsid = ITEM_clsid(it); in do_item_update()
566 it->slabs_clsid |= WARM_LRU; in do_item_update()
567 it->it_flags &= ~ITEM_ACTIVE; in do_item_update()
568 item_link_q_warm(it); in do_item_update()
570 it->time = current_time; in do_item_update()
573 } else if (it->time < current_time - ITEM_UPDATE_INTERVAL) { in do_item_update()
574 assert((it->it_flags & ITEM_SLABBED) == 0); in do_item_update()
576 if ((it->it_flags & ITEM_LINKED) != 0) { in do_item_update()
577 it->time = current_time; in do_item_update()
578 item_unlink_q(it); in do_item_update()
579 item_link_q(it); in do_item_update()
584 int do_item_replace(item *it, item *new_it, const uint32_t hv, const uint64_t cas) { in do_item_replace() argument
585 MEMCACHED_ITEM_REPLACE(ITEM_key(it), it->nkey, it->nbytes, in do_item_replace()
587 assert((it->it_flags & ITEM_SLABBED) == 0); in do_item_replace()
589 do_item_unlink(it, hv); in do_item_replace()
649 item *it; in item_cachedump() local
658 it = heads[id]; in item_cachedump()
667 while (it != NULL && (limit == 0 || shown < limit)) { in item_cachedump()
668 assert(it->nkey <= KEY_MAX_LENGTH); in item_cachedump()
670 if ((it->nbytes == 0 && it->nkey == 0) || (it->it_flags & ITEM_KEY_BINARY)) { in item_cachedump()
671 it = it->next; in item_cachedump()
675 strncpy(key_temp, ITEM_key(it), it->nkey); in item_cachedump()
676 key_temp[it->nkey] = 0x00; /* terminate */ in item_cachedump()
678 key_temp, it->nbytes - 2, in item_cachedump()
679 it->exptime == 0 ? 0 : in item_cachedump()
680 (unsigned long long)it->exptime + process_started); in item_cachedump()
686 it = it->next; in item_cachedump()
935 void item_stats_sizes_add(item *it) { in item_stats_sizes_add() argument
938 int ntotal = ITEM_ntotal(it); in item_stats_sizes_add()
947 void item_stats_sizes_remove(item *it) { in item_stats_sizes_remove() argument
950 int ntotal = ITEM_ntotal(it); in item_stats_sizes_remove()
981 item *it = assoc_find(key, nkey, hv); in do_item_get() local
982 if (it != NULL) { in do_item_get()
983 refcount_incr(it); in do_item_get()
1009 if (it == NULL) { in do_item_get()
1019 if (it != NULL) { in do_item_get()
1021 if (item_is_flushed(it)) { in do_item_get()
1022 do_item_unlink(it, hv); in do_item_get()
1023 STORAGE_delete(t->storage, it); in do_item_get()
1024 do_item_remove(it); in do_item_get()
1025 it = NULL; in do_item_get()
1033 } else if (it->exptime != 0 && it->exptime <= current_time) { in do_item_get()
1034 do_item_unlink(it, hv); in do_item_get()
1035 STORAGE_delete(t->storage, it); in do_item_get()
1036 do_item_remove(it); in do_item_get()
1037 it = NULL; in do_item_get()
1047 do_item_bump(t, it, hv); in do_item_get()
1049 DEBUG_REFCNT(it, '+'); in do_item_get()
1057 nkey, (it) ? it->nbytes : 0, (it) ? ITEM_clsid(it) : 0, t->cur_sfd); in do_item_get()
1059 return it; in do_item_get()
1065 void do_item_bump(LIBEVENT_THREAD *t, item *it, const uint32_t hv) { in do_item_bump() argument
1073 if ((it->it_flags & ITEM_ACTIVE) == 0) { in do_item_bump()
1074 if ((it->it_flags & ITEM_FETCHED) == 0) { in do_item_bump()
1075 it->it_flags |= ITEM_FETCHED; in do_item_bump()
1077 it->it_flags |= ITEM_ACTIVE; in do_item_bump()
1078 if (ITEM_lruid(it) != COLD_LRU) { in do_item_bump()
1079 it->time = current_time; // only need to bump time. in do_item_bump()
1080 } else if (!lru_bump_async(t->lru_bump_buf, it, hv)) { in do_item_bump()
1082 it->it_flags &= ~ITEM_ACTIVE; in do_item_bump()
1087 it->it_flags |= ITEM_FETCHED; in do_item_bump()
1088 do_item_update(it); in do_item_bump()
1094 item *it = do_item_get(key, nkey, hv, t, DO_UPDATE); in do_item_touch() local
1095 if (it != NULL) { in do_item_touch()
1096 it->exptime = exptime; in do_item_touch()
1098 return it; in do_item_touch()
1108 item *it = NULL; in lru_pull_tail() local
1204 it = search; in lru_pull_tail()
1211 it = search; in lru_pull_tail()
1216 it = search; in lru_pull_tail()
1220 it = search; /* No matter what, we're stopping */ in lru_pull_tail()
1245 ret_it->it = it; in lru_pull_tail()
1257 it = search; /* Kill the loop. Parent only interested in reclaims */ in lru_pull_tail()
1260 if (it != NULL) in lru_pull_tail()
1266 if (it != NULL) { in lru_pull_tail()
1268 it->slabs_clsid = ITEM_clsid(it); in lru_pull_tail()
1269 it->slabs_clsid |= move_to_lru; in lru_pull_tail()
1270 item_link_q(it); in lru_pull_tail()
1273 do_item_remove(it); in lru_pull_tail()
1314 static bool lru_bump_async(lru_bump_buf *b, item *it, uint32_t hv) { in lru_bump_async() argument
1316 refcount_incr(it); in lru_bump_async()
1320 be->it = it; in lru_bump_async()
1331 refcount_decr(it); in lru_bump_async()
1363 do_item_update(be->it); in lru_maintainer_bumps()
1364 do_item_remove(be->it); in lru_maintainer_bumps()
1751 void do_item_linktail_q(item *it) { /* item is the new tail */ in do_item_linktail_q() argument
1753 assert(it->it_flags == 1); in do_item_linktail_q()
1754 assert(it->nbytes == 0); in do_item_linktail_q()
1756 head = &heads[it->slabs_clsid]; in do_item_linktail_q()
1757 tail = &tails[it->slabs_clsid]; in do_item_linktail_q()
1759 assert(it != *tail); in do_item_linktail_q()
1761 it->prev = *tail; in do_item_linktail_q()
1762 it->next = 0; in do_item_linktail_q()
1763 if (it->prev) { in do_item_linktail_q()
1764 assert(it->prev->next == 0); in do_item_linktail_q()
1765 it->prev->next = it; in do_item_linktail_q()
1767 *tail = it; in do_item_linktail_q()
1768 if (*head == 0) *head = it; in do_item_linktail_q()
1772 void do_item_unlinktail_q(item *it) { in do_item_unlinktail_q() argument
1774 head = &heads[it->slabs_clsid]; in do_item_unlinktail_q()
1775 tail = &tails[it->slabs_clsid]; in do_item_unlinktail_q()
1777 if (*head == it) { in do_item_unlinktail_q()
1778 assert(it->prev == 0); in do_item_unlinktail_q()
1779 *head = it->next; in do_item_unlinktail_q()
1781 if (*tail == it) { in do_item_unlinktail_q()
1782 assert(it->next == 0); in do_item_unlinktail_q()
1783 *tail = it->prev; in do_item_unlinktail_q()
1785 assert(it->next != it); in do_item_unlinktail_q()
1786 assert(it->prev != it); in do_item_unlinktail_q()
1788 if (it->next) it->next->prev = it->prev; in do_item_unlinktail_q()
1789 if (it->prev) it->prev->next = it->next; in do_item_unlinktail_q()
1795 item *do_item_crawl_q(item *it) { in do_item_crawl_q() argument
1797 assert(it->it_flags == 1); in do_item_crawl_q()
1798 assert(it->nbytes == 0); in do_item_crawl_q()
1799 head = &heads[it->slabs_clsid]; in do_item_crawl_q()
1800 tail = &tails[it->slabs_clsid]; in do_item_crawl_q()
1803 if (it->prev == 0) { in do_item_crawl_q()
1804 assert(*head == it); in do_item_crawl_q()
1805 if (it->next) { in do_item_crawl_q()
1806 *head = it->next; in do_item_crawl_q()
1807 assert(it->next->prev == it); in do_item_crawl_q()
1808 it->next->prev = 0; in do_item_crawl_q()
1815 assert(it->prev != it); in do_item_crawl_q()
1816 if (it->prev) { in do_item_crawl_q()
1817 if (*head == it->prev) { in do_item_crawl_q()
1819 *head = it; in do_item_crawl_q()
1821 if (*tail == it) { in do_item_crawl_q()
1823 *tail = it->prev; in do_item_crawl_q()
1825 assert(it->next != it); in do_item_crawl_q()
1826 if (it->next) { in do_item_crawl_q()
1827 assert(it->prev->next == it); in do_item_crawl_q()
1828 it->prev->next = it->next; in do_item_crawl_q()
1829 it->next->prev = it->prev; in do_item_crawl_q()
1832 it->prev->next = 0; in do_item_crawl_q()
1835 it->next = it->prev; in do_item_crawl_q()
1836 it->prev = it->next->prev; in do_item_crawl_q()
1837 it->next->prev = it; in do_item_crawl_q()
1839 if (it->prev) { in do_item_crawl_q()
1840 it->prev->next = it; in do_item_crawl_q()
1843 assert(it->next != it); in do_item_crawl_q()
1844 assert(it->prev != it); in do_item_crawl_q()
1846 return it->next; /* success */ in do_item_crawl_q()