1 #include "redis.h" 2 #include "lzf.h" /* LZF compression library */ 3 4 #include <math.h> 5 #include <sys/types.h> 6 #include <sys/time.h> 7 #include <sys/resource.h> 8 #include <sys/wait.h> 9 #include <arpa/inet.h> 10 #include <sys/stat.h> 11 12 /* Convenience wrapper around fwrite, that returns the number of bytes written 13 * to the file instead of the number of objects (see fwrite(3)) and -1 in the 14 * case of an error. It also supports a NULL *fp to skip writing altogether 15 * instead of writing to /dev/null. */ 16 static int rdbWriteRaw(FILE *fp, void *p, size_t len) { 17 if (fp != NULL && fwrite(p,len,1,fp) == 0) return -1; 18 return len; 19 } 20 21 int rdbSaveType(FILE *fp, unsigned char type) { 22 return rdbWriteRaw(fp,&type,1); 23 } 24 25 int rdbSaveTime(FILE *fp, time_t t) { 26 int32_t t32 = (int32_t) t; 27 return rdbWriteRaw(fp,&t32,4); 28 } 29 30 /* check rdbLoadLen() comments for more info */ 31 int rdbSaveLen(FILE *fp, uint32_t len) { 32 unsigned char buf[2]; 33 int nwritten; 34 35 if (len < (1<<6)) { 36 /* Save a 6 bit len */ 37 buf[0] = (len&0xFF)|(REDIS_RDB_6BITLEN<<6); 38 if (rdbWriteRaw(fp,buf,1) == -1) return -1; 39 nwritten = 1; 40 } else if (len < (1<<14)) { 41 /* Save a 14 bit len */ 42 buf[0] = ((len>>8)&0xFF)|(REDIS_RDB_14BITLEN<<6); 43 buf[1] = len&0xFF; 44 if (rdbWriteRaw(fp,buf,2) == -1) return -1; 45 nwritten = 2; 46 } else { 47 /* Save a 32 bit len */ 48 buf[0] = (REDIS_RDB_32BITLEN<<6); 49 if (rdbWriteRaw(fp,buf,1) == -1) return -1; 50 len = htonl(len); 51 if (rdbWriteRaw(fp,&len,4) == -1) return -1; 52 nwritten = 1+4; 53 } 54 return nwritten; 55 } 56 57 /* Encode 'value' as an integer if possible (if integer will fit the 58 * supported range). If the function sucessful encoded the integer 59 * then the (up to 5 bytes) encoded representation is written in the 60 * string pointed by 'enc' and the length is returned. Otherwise 61 * 0 is returned. */ 62 int rdbEncodeInteger(long long value, unsigned char *enc) { 63 /* Finally check if it fits in our ranges */ 64 if (value >= -(1<<7) && value <= (1<<7)-1) { 65 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT8; 66 enc[1] = value&0xFF; 67 return 2; 68 } else if (value >= -(1<<15) && value <= (1<<15)-1) { 69 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT16; 70 enc[1] = value&0xFF; 71 enc[2] = (value>>8)&0xFF; 72 return 3; 73 } else if (value >= -((long long)1<<31) && value <= ((long long)1<<31)-1) { 74 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT32; 75 enc[1] = value&0xFF; 76 enc[2] = (value>>8)&0xFF; 77 enc[3] = (value>>16)&0xFF; 78 enc[4] = (value>>24)&0xFF; 79 return 5; 80 } else { 81 return 0; 82 } 83 } 84 85 /* String objects in the form "2391" "-100" without any space and with a 86 * range of values that can fit in an 8, 16 or 32 bit signed value can be 87 * encoded as integers to save space */ 88 int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) { 89 long long value; 90 char *endptr, buf[32]; 91 92 /* Check if it's possible to encode this value as a number */ 93 value = strtoll(s, &endptr, 10); 94 if (endptr[0] != '\0') return 0; 95 ll2string(buf,32,value); 96 97 /* If the number converted back into a string is not identical 98 * then it's not possible to encode the string as integer */ 99 if (strlen(buf) != len || memcmp(buf,s,len)) return 0; 100 101 return rdbEncodeInteger(value,enc); 102 } 103 104 int rdbSaveLzfStringObject(FILE *fp, unsigned char *s, size_t len) { 105 size_t comprlen, outlen; 106 unsigned char byte; 107 int n, nwritten = 0; 108 void *out; 109 110 /* We require at least four bytes compression for this to be worth it */ 111 if (len <= 4) return 0; 112 outlen = len-4; 113 if ((out = zmalloc(outlen+1)) == NULL) return 0; 114 comprlen = lzf_compress(s, len, out, outlen); 115 if (comprlen == 0) { 116 zfree(out); 117 return 0; 118 } 119 /* Data compressed! Let's save it on disk */ 120 byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF; 121 if ((n = rdbWriteRaw(fp,&byte,1)) == -1) goto writeerr; 122 nwritten += n; 123 124 if ((n = rdbSaveLen(fp,comprlen)) == -1) goto writeerr; 125 nwritten += n; 126 127 if ((n = rdbSaveLen(fp,len)) == -1) goto writeerr; 128 nwritten += n; 129 130 if ((n = rdbWriteRaw(fp,out,comprlen)) == -1) goto writeerr; 131 nwritten += n; 132 133 zfree(out); 134 return nwritten; 135 136 writeerr: 137 zfree(out); 138 return -1; 139 } 140 141 /* Save a string objet as [len][data] on disk. If the object is a string 142 * representation of an integer value we try to save it in a special form */ 143 int rdbSaveRawString(FILE *fp, unsigned char *s, size_t len) { 144 int enclen; 145 int n, nwritten = 0; 146 147 /* Try integer encoding */ 148 if (len <= 11) { 149 unsigned char buf[5]; 150 if ((enclen = rdbTryIntegerEncoding((char*)s,len,buf)) > 0) { 151 if (rdbWriteRaw(fp,buf,enclen) == -1) return -1; 152 return enclen; 153 } 154 } 155 156 /* Try LZF compression - under 20 bytes it's unable to compress even 157 * aaaaaaaaaaaaaaaaaa so skip it */ 158 if (server.rdbcompression && len > 20) { 159 n = rdbSaveLzfStringObject(fp,s,len); 160 if (n == -1) return -1; 161 if (n > 0) return n; 162 /* Return value of 0 means data can't be compressed, save the old way */ 163 } 164 165 /* Store verbatim */ 166 if ((n = rdbSaveLen(fp,len)) == -1) return -1; 167 nwritten += n; 168 if (len > 0) { 169 if (rdbWriteRaw(fp,s,len) == -1) return -1; 170 nwritten += len; 171 } 172 return nwritten; 173 } 174 175 /* Save a long long value as either an encoded string or a string. */ 176 int rdbSaveLongLongAsStringObject(FILE *fp, long long value) { 177 unsigned char buf[32]; 178 int n, nwritten = 0; 179 int enclen = rdbEncodeInteger(value,buf); 180 if (enclen > 0) { 181 return rdbWriteRaw(fp,buf,enclen); 182 } else { 183 /* Encode as string */ 184 enclen = ll2string((char*)buf,32,value); 185 redisAssert(enclen < 32); 186 if ((n = rdbSaveLen(fp,enclen)) == -1) return -1; 187 nwritten += n; 188 if ((n = rdbWriteRaw(fp,buf,enclen)) == -1) return -1; 189 nwritten += n; 190 } 191 return nwritten; 192 } 193 194 /* Like rdbSaveStringObjectRaw() but handle encoded objects */ 195 int rdbSaveStringObject(FILE *fp, robj *obj) { 196 /* Avoid to decode the object, then encode it again, if the 197 * object is alrady integer encoded. */ 198 if (obj->encoding == REDIS_ENCODING_INT) { 199 return rdbSaveLongLongAsStringObject(fp,(long)obj->ptr); 200 } else { 201 redisAssert(obj->encoding == REDIS_ENCODING_RAW); 202 return rdbSaveRawString(fp,obj->ptr,sdslen(obj->ptr)); 203 } 204 } 205 206 /* Save a double value. Doubles are saved as strings prefixed by an unsigned 207 * 8 bit integer specifing the length of the representation. 208 * This 8 bit integer has special values in order to specify the following 209 * conditions: 210 * 253: not a number 211 * 254: + inf 212 * 255: - inf 213 */ 214 int rdbSaveDoubleValue(FILE *fp, double val) { 215 unsigned char buf[128]; 216 int len; 217 218 if (isnan(val)) { 219 buf[0] = 253; 220 len = 1; 221 } else if (!isfinite(val)) { 222 len = 1; 223 buf[0] = (val < 0) ? 255 : 254; 224 } else { 225 #if (DBL_MANT_DIG >= 52) && (LLONG_MAX == 0x7fffffffffffffffLL) 226 /* Check if the float is in a safe range to be casted into a 227 * long long. We are assuming that long long is 64 bit here. 228 * Also we are assuming that there are no implementations around where 229 * double has precision < 52 bit. 230 * 231 * Under this assumptions we test if a double is inside an interval 232 * where casting to long long is safe. Then using two castings we 233 * make sure the decimal part is zero. If all this is true we use 234 * integer printing function that is much faster. */ 235 double min = -4503599627370495; /* (2^52)-1 */ 236 double max = 4503599627370496; /* -(2^52) */ 237 if (val > min && val < max && val == ((double)((long long)val))) 238 ll2string((char*)buf+1,sizeof(buf),(long long)val); 239 else 240 #endif 241 snprintf((char*)buf+1,sizeof(buf)-1,"%.17g",val); 242 buf[0] = strlen((char*)buf+1); 243 len = buf[0]+1; 244 } 245 return rdbWriteRaw(fp,buf,len); 246 } 247 248 /* Save a Redis object. Returns -1 on error, 0 on success. */ 249 int rdbSaveObject(FILE *fp, robj *o) { 250 int n, nwritten = 0; 251 252 if (o->type == REDIS_STRING) { 253 /* Save a string value */ 254 if ((n = rdbSaveStringObject(fp,o)) == -1) return -1; 255 nwritten += n; 256 } else if (o->type == REDIS_LIST) { 257 /* Save a list value */ 258 if (o->encoding == REDIS_ENCODING_ZIPLIST) { 259 size_t l = ziplistBlobLen((unsigned char*)o->ptr); 260 261 if ((n = rdbSaveRawString(fp,o->ptr,l)) == -1) return -1; 262 nwritten += n; 263 } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { 264 list *list = o->ptr; 265 listIter li; 266 listNode *ln; 267 268 if ((n = rdbSaveLen(fp,listLength(list))) == -1) return -1; 269 nwritten += n; 270 271 listRewind(list,&li); 272 while((ln = listNext(&li))) { 273 robj *eleobj = listNodeValue(ln); 274 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; 275 nwritten += n; 276 } 277 } else { 278 redisPanic("Unknown list encoding"); 279 } 280 } else if (o->type == REDIS_SET) { 281 /* Save a set value */ 282 if (o->encoding == REDIS_ENCODING_HT) { 283 dict *set = o->ptr; 284 dictIterator *di = dictGetIterator(set); 285 dictEntry *de; 286 287 if ((n = rdbSaveLen(fp,dictSize(set))) == -1) return -1; 288 nwritten += n; 289 290 while((de = dictNext(di)) != NULL) { 291 robj *eleobj = dictGetEntryKey(de); 292 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; 293 nwritten += n; 294 } 295 dictReleaseIterator(di); 296 } else if (o->encoding == REDIS_ENCODING_INTSET) { 297 size_t l = intsetBlobLen((intset*)o->ptr); 298 299 if ((n = rdbSaveRawString(fp,o->ptr,l)) == -1) return -1; 300 nwritten += n; 301 } else { 302 redisPanic("Unknown set encoding"); 303 } 304 } else if (o->type == REDIS_ZSET) { 305 /* Save a sorted set value */ 306 if (o->encoding == REDIS_ENCODING_ZIPLIST) { 307 size_t l = ziplistBlobLen((unsigned char*)o->ptr); 308 309 if ((n = rdbSaveRawString(fp,o->ptr,l)) == -1) return -1; 310 nwritten += n; 311 } else if (o->encoding == REDIS_ENCODING_SKIPLIST) { 312 zset *zs = o->ptr; 313 dictIterator *di = dictGetIterator(zs->dict); 314 dictEntry *de; 315 316 if ((n = rdbSaveLen(fp,dictSize(zs->dict))) == -1) return -1; 317 nwritten += n; 318 319 while((de = dictNext(di)) != NULL) { 320 robj *eleobj = dictGetEntryKey(de); 321 double *score = dictGetEntryVal(de); 322 323 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; 324 nwritten += n; 325 if ((n = rdbSaveDoubleValue(fp,*score)) == -1) return -1; 326 nwritten += n; 327 } 328 dictReleaseIterator(di); 329 } else { 330 redisPanic("Unknown sorted set encoding"); 331 } 332 } else if (o->type == REDIS_HASH) { 333 /* Save a hash value */ 334 if (o->encoding == REDIS_ENCODING_ZIPMAP) { 335 size_t l = zipmapBlobLen((unsigned char*)o->ptr); 336 337 if ((n = rdbSaveRawString(fp,o->ptr,l)) == -1) return -1; 338 nwritten += n; 339 } else { 340 dictIterator *di = dictGetIterator(o->ptr); 341 dictEntry *de; 342 343 if ((n = rdbSaveLen(fp,dictSize((dict*)o->ptr))) == -1) return -1; 344 nwritten += n; 345 346 while((de = dictNext(di)) != NULL) { 347 robj *key = dictGetEntryKey(de); 348 robj *val = dictGetEntryVal(de); 349 350 if ((n = rdbSaveStringObject(fp,key)) == -1) return -1; 351 nwritten += n; 352 if ((n = rdbSaveStringObject(fp,val)) == -1) return -1; 353 nwritten += n; 354 } 355 dictReleaseIterator(di); 356 } 357 } else { 358 redisPanic("Unknown object type"); 359 } 360 return nwritten; 361 } 362 363 /* Return the length the object will have on disk if saved with 364 * the rdbSaveObject() function. Currently we use a trick to get 365 * this length with very little changes to the code. In the future 366 * we could switch to a faster solution. */ 367 off_t rdbSavedObjectLen(robj *o) { 368 int len = rdbSaveObject(NULL,o); 369 redisAssert(len != -1); 370 return len; 371 } 372 373 /* Save a key-value pair, with expire time, type, key, value. 374 * On error -1 is returned. 375 * On success if the key was actaully saved 1 is returned, otherwise 0 376 * is returned (the key was already expired). */ 377 int rdbSaveKeyValuePair(FILE *fp, robj *key, robj *val, 378 time_t expiretime, time_t now) 379 { 380 int vtype; 381 382 /* Save the expire time */ 383 if (expiretime != -1) { 384 /* If this key is already expired skip it */ 385 if (expiretime < now) return 0; 386 if (rdbSaveType(fp,REDIS_EXPIRETIME) == -1) return -1; 387 if (rdbSaveTime(fp,expiretime) == -1) return -1; 388 } 389 /* Fix the object type if needed, to support saving zipmaps, ziplists, 390 * and intsets, directly as blobs of bytes: they are already serialized. */ 391 vtype = val->type; 392 if (vtype == REDIS_HASH && val->encoding == REDIS_ENCODING_ZIPMAP) 393 vtype = REDIS_HASH_ZIPMAP; 394 else if (vtype == REDIS_LIST && val->encoding == REDIS_ENCODING_ZIPLIST) 395 vtype = REDIS_LIST_ZIPLIST; 396 else if (vtype == REDIS_SET && val->encoding == REDIS_ENCODING_INTSET) 397 vtype = REDIS_SET_INTSET; 398 else if (vtype == REDIS_ZSET && val->encoding == REDIS_ENCODING_ZIPLIST) 399 vtype = REDIS_ZSET_ZIPLIST; 400 /* Save type, key, value */ 401 if (rdbSaveType(fp,vtype) == -1) return -1; 402 if (rdbSaveStringObject(fp,key) == -1) return -1; 403 if (rdbSaveObject(fp,val) == -1) return -1; 404 return 1; 405 } 406 407 /* Save the DB on disk. Return REDIS_ERR on error, REDIS_OK on success */ 408 int rdbSave(char *filename) { 409 dictIterator *di = NULL; 410 dictEntry *de; 411 FILE *fp; 412 char tmpfile[256]; 413 int j; 414 time_t now = time(NULL); 415 416 snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid()); 417 fp = fopen(tmpfile,"w"); 418 if (!fp) { 419 redisLog(REDIS_WARNING, "Failed opening .rdb for saving: %s", 420 strerror(errno)); 421 return REDIS_ERR; 422 } 423 if (fwrite("REDIS0002",9,1,fp) == 0) goto werr; 424 for (j = 0; j < server.dbnum; j++) { 425 redisDb *db = server.db+j; 426 dict *d = db->dict; 427 if (dictSize(d) == 0) continue; 428 di = dictGetSafeIterator(d); 429 if (!di) { 430 fclose(fp); 431 return REDIS_ERR; 432 } 433 434 /* Write the SELECT DB opcode */ 435 if (rdbSaveType(fp,REDIS_SELECTDB) == -1) goto werr; 436 if (rdbSaveLen(fp,j) == -1) goto werr; 437 438 /* Iterate this DB writing every entry */ 439 while((de = dictNext(di)) != NULL) { 440 sds keystr = dictGetEntryKey(de); 441 robj key, *o = dictGetEntryVal(de); 442 time_t expire; 443 444 initStaticStringObject(key,keystr); 445 expire = getExpire(db,&key); 446 if (rdbSaveKeyValuePair(fp,&key,o,expire,now) == -1) goto werr; 447 } 448 dictReleaseIterator(di); 449 } 450 /* EOF opcode */ 451 if (rdbSaveType(fp,REDIS_EOF) == -1) goto werr; 452 453 /* Make sure data will not remain on the OS's output buffers */ 454 fflush(fp); 455 fsync(fileno(fp)); 456 fclose(fp); 457 458 /* Use RENAME to make sure the DB file is changed atomically only 459 * if the generate DB file is ok. */ 460 if (rename(tmpfile,filename) == -1) { 461 redisLog(REDIS_WARNING,"Error moving temp DB file on the final destination: %s", strerror(errno)); 462 unlink(tmpfile); 463 return REDIS_ERR; 464 } 465 redisLog(REDIS_NOTICE,"DB saved on disk"); 466 server.dirty = 0; 467 server.lastsave = time(NULL); 468 return REDIS_OK; 469 470 werr: 471 fclose(fp); 472 unlink(tmpfile); 473 redisLog(REDIS_WARNING,"Write error saving DB on disk: %s", strerror(errno)); 474 if (di) dictReleaseIterator(di); 475 return REDIS_ERR; 476 } 477 478 int rdbSaveBackground(char *filename) { 479 pid_t childpid; 480 long long start; 481 482 if (server.bgsavechildpid != -1) return REDIS_ERR; 483 484 server.dirty_before_bgsave = server.dirty; 485 486 start = ustime(); 487 if ((childpid = fork()) == 0) { 488 int retval; 489 490 /* Child */ 491 if (server.ipfd > 0) close(server.ipfd); 492 if (server.sofd > 0) close(server.sofd); 493 retval = rdbSave(filename); 494 _exit((retval == REDIS_OK) ? 0 : 1); 495 } else { 496 /* Parent */ 497 server.stat_fork_time = ustime()-start; 498 if (childpid == -1) { 499 redisLog(REDIS_WARNING,"Can't save in background: fork: %s", 500 strerror(errno)); 501 return REDIS_ERR; 502 } 503 redisLog(REDIS_NOTICE,"Background saving started by pid %d",childpid); 504 server.bgsavechildpid = childpid; 505 updateDictResizePolicy(); 506 return REDIS_OK; 507 } 508 return REDIS_OK; /* unreached */ 509 } 510 511 void rdbRemoveTempFile(pid_t childpid) { 512 char tmpfile[256]; 513 514 snprintf(tmpfile,256,"temp-%d.rdb", (int) childpid); 515 unlink(tmpfile); 516 } 517 518 int rdbLoadType(FILE *fp) { 519 unsigned char type; 520 if (fread(&type,1,1,fp) == 0) return -1; 521 return type; 522 } 523 524 time_t rdbLoadTime(FILE *fp) { 525 int32_t t32; 526 if (fread(&t32,4,1,fp) == 0) return -1; 527 return (time_t) t32; 528 } 529 530 /* Load an encoded length from the DB, see the REDIS_RDB_* defines on the top 531 * of this file for a description of how this are stored on disk. 532 * 533 * isencoded is set to 1 if the readed length is not actually a length but 534 * an "encoding type", check the above comments for more info */ 535 uint32_t rdbLoadLen(FILE *fp, int *isencoded) { 536 unsigned char buf[2]; 537 uint32_t len; 538 int type; 539 540 if (isencoded) *isencoded = 0; 541 if (fread(buf,1,1,fp) == 0) return REDIS_RDB_LENERR; 542 type = (buf[0]&0xC0)>>6; 543 if (type == REDIS_RDB_6BITLEN) { 544 /* Read a 6 bit len */ 545 return buf[0]&0x3F; 546 } else if (type == REDIS_RDB_ENCVAL) { 547 /* Read a 6 bit len encoding type */ 548 if (isencoded) *isencoded = 1; 549 return buf[0]&0x3F; 550 } else if (type == REDIS_RDB_14BITLEN) { 551 /* Read a 14 bit len */ 552 if (fread(buf+1,1,1,fp) == 0) return REDIS_RDB_LENERR; 553 return ((buf[0]&0x3F)<<8)|buf[1]; 554 } else { 555 /* Read a 32 bit len */ 556 if (fread(&len,4,1,fp) == 0) return REDIS_RDB_LENERR; 557 return ntohl(len); 558 } 559 } 560 561 /* Load an integer-encoded object from file 'fp', with the specified 562 * encoding type 'enctype'. If encode is true the function may return 563 * an integer-encoded object as reply, otherwise the returned object 564 * will always be encoded as a raw string. */ 565 robj *rdbLoadIntegerObject(FILE *fp, int enctype, int encode) { 566 unsigned char enc[4]; 567 long long val; 568 569 if (enctype == REDIS_RDB_ENC_INT8) { 570 if (fread(enc,1,1,fp) == 0) return NULL; 571 val = (signed char)enc[0]; 572 } else if (enctype == REDIS_RDB_ENC_INT16) { 573 uint16_t v; 574 if (fread(enc,2,1,fp) == 0) return NULL; 575 v = enc[0]|(enc[1]<<8); 576 val = (int16_t)v; 577 } else if (enctype == REDIS_RDB_ENC_INT32) { 578 uint32_t v; 579 if (fread(enc,4,1,fp) == 0) return NULL; 580 v = enc[0]|(enc[1]<<8)|(enc[2]<<16)|(enc[3]<<24); 581 val = (int32_t)v; 582 } else { 583 val = 0; /* anti-warning */ 584 redisPanic("Unknown RDB integer encoding type"); 585 } 586 if (encode) 587 return createStringObjectFromLongLong(val); 588 else 589 return createObject(REDIS_STRING,sdsfromlonglong(val)); 590 } 591 592 robj *rdbLoadLzfStringObject(FILE*fp) { 593 unsigned int len, clen; 594 unsigned char *c = NULL; 595 sds val = NULL; 596 597 if ((clen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 598 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 599 if ((c = zmalloc(clen)) == NULL) goto err; 600 if ((val = sdsnewlen(NULL,len)) == NULL) goto err; 601 if (fread(c,clen,1,fp) == 0) goto err; 602 if (lzf_decompress(c,clen,val,len) == 0) goto err; 603 zfree(c); 604 return createObject(REDIS_STRING,val); 605 err: 606 zfree(c); 607 sdsfree(val); 608 return NULL; 609 } 610 611 robj *rdbGenericLoadStringObject(FILE*fp, int encode) { 612 int isencoded; 613 uint32_t len; 614 sds val; 615 616 len = rdbLoadLen(fp,&isencoded); 617 if (isencoded) { 618 switch(len) { 619 case REDIS_RDB_ENC_INT8: 620 case REDIS_RDB_ENC_INT16: 621 case REDIS_RDB_ENC_INT32: 622 return rdbLoadIntegerObject(fp,len,encode); 623 case REDIS_RDB_ENC_LZF: 624 return rdbLoadLzfStringObject(fp); 625 default: 626 redisPanic("Unknown RDB encoding type"); 627 } 628 } 629 630 if (len == REDIS_RDB_LENERR) return NULL; 631 val = sdsnewlen(NULL,len); 632 if (len && fread(val,len,1,fp) == 0) { 633 sdsfree(val); 634 return NULL; 635 } 636 return createObject(REDIS_STRING,val); 637 } 638 639 robj *rdbLoadStringObject(FILE *fp) { 640 return rdbGenericLoadStringObject(fp,0); 641 } 642 643 robj *rdbLoadEncodedStringObject(FILE *fp) { 644 return rdbGenericLoadStringObject(fp,1); 645 } 646 647 /* For information about double serialization check rdbSaveDoubleValue() */ 648 int rdbLoadDoubleValue(FILE *fp, double *val) { 649 char buf[128]; 650 unsigned char len; 651 652 if (fread(&len,1,1,fp) == 0) return -1; 653 switch(len) { 654 case 255: *val = R_NegInf; return 0; 655 case 254: *val = R_PosInf; return 0; 656 case 253: *val = R_Nan; return 0; 657 default: 658 if (fread(buf,len,1,fp) == 0) return -1; 659 buf[len] = '\0'; 660 sscanf(buf, "%lg", val); 661 return 0; 662 } 663 } 664 665 /* Load a Redis object of the specified type from the specified file. 666 * On success a newly allocated object is returned, otherwise NULL. */ 667 robj *rdbLoadObject(int type, FILE *fp) { 668 robj *o, *ele, *dec; 669 size_t len; 670 unsigned int i; 671 672 redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",type,ftell(fp)); 673 if (type == REDIS_STRING) { 674 /* Read string value */ 675 if ((o = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 676 o = tryObjectEncoding(o); 677 } else if (type == REDIS_LIST) { 678 /* Read list value */ 679 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 680 681 /* Use a real list when there are too many entries */ 682 if (len > server.list_max_ziplist_entries) { 683 o = createListObject(); 684 } else { 685 o = createZiplistObject(); 686 } 687 688 /* Load every single element of the list */ 689 while(len--) { 690 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 691 692 /* If we are using a ziplist and the value is too big, convert 693 * the object to a real list. */ 694 if (o->encoding == REDIS_ENCODING_ZIPLIST && 695 ele->encoding == REDIS_ENCODING_RAW && 696 sdslen(ele->ptr) > server.list_max_ziplist_value) 697 listTypeConvert(o,REDIS_ENCODING_LINKEDLIST); 698 699 if (o->encoding == REDIS_ENCODING_ZIPLIST) { 700 dec = getDecodedObject(ele); 701 o->ptr = ziplistPush(o->ptr,dec->ptr,sdslen(dec->ptr),REDIS_TAIL); 702 decrRefCount(dec); 703 decrRefCount(ele); 704 } else { 705 ele = tryObjectEncoding(ele); 706 listAddNodeTail(o->ptr,ele); 707 } 708 } 709 } else if (type == REDIS_SET) { 710 /* Read list/set value */ 711 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 712 713 /* Use a regular set when there are too many entries. */ 714 if (len > server.set_max_intset_entries) { 715 o = createSetObject(); 716 /* It's faster to expand the dict to the right size asap in order 717 * to avoid rehashing */ 718 if (len > DICT_HT_INITIAL_SIZE) 719 dictExpand(o->ptr,len); 720 } else { 721 o = createIntsetObject(); 722 } 723 724 /* Load every single element of the list/set */ 725 for (i = 0; i < len; i++) { 726 long long llval; 727 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 728 ele = tryObjectEncoding(ele); 729 730 if (o->encoding == REDIS_ENCODING_INTSET) { 731 /* Fetch integer value from element */ 732 if (isObjectRepresentableAsLongLong(ele,&llval) == REDIS_OK) { 733 o->ptr = intsetAdd(o->ptr,llval,NULL); 734 } else { 735 setTypeConvert(o,REDIS_ENCODING_HT); 736 dictExpand(o->ptr,len); 737 } 738 } 739 740 /* This will also be called when the set was just converted 741 * to regular hashtable encoded set */ 742 if (o->encoding == REDIS_ENCODING_HT) { 743 dictAdd((dict*)o->ptr,ele,NULL); 744 } else { 745 decrRefCount(ele); 746 } 747 } 748 } else if (type == REDIS_ZSET) { 749 /* Read list/set value */ 750 size_t zsetlen; 751 size_t maxelelen = 0; 752 zset *zs; 753 754 if ((zsetlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 755 o = createZsetObject(); 756 zs = o->ptr; 757 758 /* Load every single element of the list/set */ 759 while(zsetlen--) { 760 robj *ele; 761 double score; 762 zskiplistNode *znode; 763 764 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 765 ele = tryObjectEncoding(ele); 766 if (rdbLoadDoubleValue(fp,&score) == -1) return NULL; 767 768 /* Don't care about integer-encoded strings. */ 769 if (ele->encoding == REDIS_ENCODING_RAW && 770 sdslen(ele->ptr) > maxelelen) 771 maxelelen = sdslen(ele->ptr); 772 773 znode = zslInsert(zs->zsl,score,ele); 774 dictAdd(zs->dict,ele,&znode->score); 775 incrRefCount(ele); /* added to skiplist */ 776 } 777 778 /* Convert *after* loading, since sorted sets are not stored ordered. */ 779 if (zsetLength(o) <= server.zset_max_ziplist_entries && 780 maxelelen <= server.zset_max_ziplist_value) 781 zsetConvert(o,REDIS_ENCODING_ZIPLIST); 782 } else if (type == REDIS_HASH) { 783 size_t hashlen; 784 785 if ((hashlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 786 o = createHashObject(); 787 /* Too many entries? Use an hash table. */ 788 if (hashlen > server.hash_max_zipmap_entries) 789 convertToRealHash(o); 790 /* Load every key/value, then set it into the zipmap or hash 791 * table, as needed. */ 792 while(hashlen--) { 793 robj *key, *val; 794 795 if ((key = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 796 if ((val = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 797 /* If we are using a zipmap and there are too big values 798 * the object is converted to real hash table encoding. */ 799 if (o->encoding != REDIS_ENCODING_HT && 800 ((key->encoding == REDIS_ENCODING_RAW && 801 sdslen(key->ptr) > server.hash_max_zipmap_value) || 802 (val->encoding == REDIS_ENCODING_RAW && 803 sdslen(val->ptr) > server.hash_max_zipmap_value))) 804 { 805 convertToRealHash(o); 806 } 807 808 if (o->encoding == REDIS_ENCODING_ZIPMAP) { 809 unsigned char *zm = o->ptr; 810 robj *deckey, *decval; 811 812 /* We need raw string objects to add them to the zipmap */ 813 deckey = getDecodedObject(key); 814 decval = getDecodedObject(val); 815 zm = zipmapSet(zm,deckey->ptr,sdslen(deckey->ptr), 816 decval->ptr,sdslen(decval->ptr),NULL); 817 o->ptr = zm; 818 decrRefCount(deckey); 819 decrRefCount(decval); 820 decrRefCount(key); 821 decrRefCount(val); 822 } else { 823 key = tryObjectEncoding(key); 824 val = tryObjectEncoding(val); 825 dictAdd((dict*)o->ptr,key,val); 826 } 827 } 828 } else if (type == REDIS_HASH_ZIPMAP || 829 type == REDIS_LIST_ZIPLIST || 830 type == REDIS_SET_INTSET || 831 type == REDIS_ZSET_ZIPLIST) 832 { 833 robj *aux = rdbLoadStringObject(fp); 834 835 if (aux == NULL) return NULL; 836 o = createObject(REDIS_STRING,NULL); /* string is just placeholder */ 837 o->ptr = zmalloc(sdslen(aux->ptr)); 838 memcpy(o->ptr,aux->ptr,sdslen(aux->ptr)); 839 decrRefCount(aux); 840 841 /* Fix the object encoding, and make sure to convert the encoded 842 * data type into the base type if accordingly to the current 843 * configuration there are too many elements in the encoded data 844 * type. Note that we only check the length and not max element 845 * size as this is an O(N) scan. Eventually everything will get 846 * converted. */ 847 switch(type) { 848 case REDIS_HASH_ZIPMAP: 849 o->type = REDIS_HASH; 850 o->encoding = REDIS_ENCODING_ZIPMAP; 851 if (zipmapLen(o->ptr) > server.hash_max_zipmap_entries) 852 convertToRealHash(o); 853 break; 854 case REDIS_LIST_ZIPLIST: 855 o->type = REDIS_LIST; 856 o->encoding = REDIS_ENCODING_ZIPLIST; 857 if (ziplistLen(o->ptr) > server.list_max_ziplist_entries) 858 listTypeConvert(o,REDIS_ENCODING_LINKEDLIST); 859 break; 860 case REDIS_SET_INTSET: 861 o->type = REDIS_SET; 862 o->encoding = REDIS_ENCODING_INTSET; 863 if (intsetLen(o->ptr) > server.set_max_intset_entries) 864 setTypeConvert(o,REDIS_ENCODING_HT); 865 break; 866 case REDIS_ZSET_ZIPLIST: 867 o->type = REDIS_ZSET; 868 o->encoding = REDIS_ENCODING_ZIPLIST; 869 if (zsetLength(o) > server.zset_max_ziplist_entries) 870 zsetConvert(o,REDIS_ENCODING_SKIPLIST); 871 break; 872 default: 873 redisPanic("Unknown encoding"); 874 break; 875 } 876 } else { 877 redisPanic("Unknown object type"); 878 } 879 return o; 880 } 881 882 /* Mark that we are loading in the global state and setup the fields 883 * needed to provide loading stats. */ 884 void startLoading(FILE *fp) { 885 struct stat sb; 886 887 /* Load the DB */ 888 server.loading = 1; 889 server.loading_start_time = time(NULL); 890 if (fstat(fileno(fp), &sb) == -1) { 891 server.loading_total_bytes = 1; /* just to avoid division by zero */ 892 } else { 893 server.loading_total_bytes = sb.st_size; 894 } 895 } 896 897 /* Refresh the loading progress info */ 898 void loadingProgress(off_t pos) { 899 server.loading_loaded_bytes = pos; 900 } 901 902 /* Loading finished */ 903 void stopLoading(void) { 904 server.loading = 0; 905 } 906 907 int rdbLoad(char *filename) { 908 FILE *fp; 909 uint32_t dbid; 910 int type, rdbver; 911 redisDb *db = server.db+0; 912 char buf[1024]; 913 time_t expiretime, now = time(NULL); 914 long loops = 0; 915 916 fp = fopen(filename,"r"); 917 if (!fp) return REDIS_ERR; 918 if (fread(buf,9,1,fp) == 0) goto eoferr; 919 buf[9] = '\0'; 920 if (memcmp(buf,"REDIS",5) != 0) { 921 fclose(fp); 922 redisLog(REDIS_WARNING,"Wrong signature trying to load DB from file"); 923 return REDIS_ERR; 924 } 925 rdbver = atoi(buf+5); 926 if (rdbver < 1 || rdbver > 2) { 927 fclose(fp); 928 redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver); 929 return REDIS_ERR; 930 } 931 932 startLoading(fp); 933 while(1) { 934 robj *key, *val; 935 expiretime = -1; 936 937 /* Serve the clients from time to time */ 938 if (!(loops++ % 1000)) { 939 loadingProgress(ftello(fp)); 940 aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); 941 } 942 943 /* Read type. */ 944 if ((type = rdbLoadType(fp)) == -1) goto eoferr; 945 if (type == REDIS_EXPIRETIME) { 946 if ((expiretime = rdbLoadTime(fp)) == -1) goto eoferr; 947 /* We read the time so we need to read the object type again */ 948 if ((type = rdbLoadType(fp)) == -1) goto eoferr; 949 } 950 if (type == REDIS_EOF) break; 951 /* Handle SELECT DB opcode as a special case */ 952 if (type == REDIS_SELECTDB) { 953 if ((dbid = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) 954 goto eoferr; 955 if (dbid >= (unsigned)server.dbnum) { 956 redisLog(REDIS_WARNING,"FATAL: Data file was created with a Redis server configured to handle more than %d databases. Exiting\n", server.dbnum); 957 exit(1); 958 } 959 db = server.db+dbid; 960 continue; 961 } 962 /* Read key */ 963 if ((key = rdbLoadStringObject(fp)) == NULL) goto eoferr; 964 /* Read value */ 965 if ((val = rdbLoadObject(type,fp)) == NULL) goto eoferr; 966 /* Check if the key already expired */ 967 if (expiretime != -1 && expiretime < now) { 968 decrRefCount(key); 969 decrRefCount(val); 970 continue; 971 } 972 /* Add the new object in the hash table */ 973 dbAdd(db,key,val); 974 975 /* Set the expire time if needed */ 976 if (expiretime != -1) setExpire(db,key,expiretime); 977 978 decrRefCount(key); 979 } 980 fclose(fp); 981 stopLoading(); 982 return REDIS_OK; 983 984 eoferr: /* unexpected end of file is handled here with a fatal exit */ 985 redisLog(REDIS_WARNING,"Short read or OOM loading DB. Unrecoverable error, aborting now."); 986 exit(1); 987 return REDIS_ERR; /* Just to avoid warning */ 988 } 989 990 /* A background saving child (BGSAVE) terminated its work. Handle this. */ 991 void backgroundSaveDoneHandler(int exitcode, int bysignal) { 992 if (!bysignal && exitcode == 0) { 993 redisLog(REDIS_NOTICE, 994 "Background saving terminated with success"); 995 server.dirty = server.dirty - server.dirty_before_bgsave; 996 server.lastsave = time(NULL); 997 } else if (!bysignal && exitcode != 0) { 998 redisLog(REDIS_WARNING, "Background saving error"); 999 } else { 1000 redisLog(REDIS_WARNING, 1001 "Background saving terminated by signal %d", bysignal); 1002 rdbRemoveTempFile(server.bgsavechildpid); 1003 } 1004 server.bgsavechildpid = -1; 1005 /* Possibly there are slaves waiting for a BGSAVE in order to be served 1006 * (the first stage of SYNC is a bulk transfer of dump.rdb) */ 1007 updateSlavesWaitingBgsave(exitcode == 0 ? REDIS_OK : REDIS_ERR); 1008 } 1009 1010 void saveCommand(redisClient *c) { 1011 if (server.bgsavechildpid != -1) { 1012 addReplyError(c,"Background save already in progress"); 1013 return; 1014 } 1015 if (rdbSave(server.dbfilename) == REDIS_OK) { 1016 addReply(c,shared.ok); 1017 } else { 1018 addReply(c,shared.err); 1019 } 1020 } 1021 1022 void bgsaveCommand(redisClient *c) { 1023 if (server.bgsavechildpid != -1) { 1024 addReplyError(c,"Background save already in progress"); 1025 } else if (server.bgrewritechildpid != -1) { 1026 addReplyError(c,"Can't BGSAVE while AOF log rewriting is in progress"); 1027 } else if (rdbSaveBackground(server.dbfilename) == REDIS_OK) { 1028 addReplyStatus(c,"Background saving started"); 1029 } else { 1030 addReply(c,shared.err); 1031 } 1032 } 1033