1 #include "redis.h" 2 #include "lzf.h" /* LZF compression library */ 3 4 #include <math.h> 5 #include <sys/types.h> 6 #include <sys/time.h> 7 #include <sys/resource.h> 8 #include <sys/wait.h> 9 #include <arpa/inet.h> 10 #include <sys/stat.h> 11 12 /* Convenience wrapper around fwrite, that returns the number of bytes written 13 * to the file instead of the number of objects (see fwrite(3)) and -1 in the 14 * case of an error. It also supports a NULL *fp to skip writing altogether 15 * instead of writing to /dev/null. */ 16 static int rdbWriteRaw(FILE *fp, void *p, size_t len) { 17 if (fp != NULL && fwrite(p,len,1,fp) == 0) return -1; 18 return len; 19 } 20 21 int rdbSaveType(FILE *fp, unsigned char type) { 22 return rdbWriteRaw(fp,&type,1); 23 } 24 25 int rdbSaveTime(FILE *fp, time_t t) { 26 int32_t t32 = (int32_t) t; 27 return rdbWriteRaw(fp,&t32,4); 28 } 29 30 /* check rdbLoadLen() comments for more info */ 31 int rdbSaveLen(FILE *fp, uint32_t len) { 32 unsigned char buf[2]; 33 int nwritten; 34 35 if (len < (1<<6)) { 36 /* Save a 6 bit len */ 37 buf[0] = (len&0xFF)|(REDIS_RDB_6BITLEN<<6); 38 if (rdbWriteRaw(fp,buf,1) == -1) return -1; 39 nwritten = 1; 40 } else if (len < (1<<14)) { 41 /* Save a 14 bit len */ 42 buf[0] = ((len>>8)&0xFF)|(REDIS_RDB_14BITLEN<<6); 43 buf[1] = len&0xFF; 44 if (rdbWriteRaw(fp,buf,2) == -1) return -1; 45 nwritten = 2; 46 } else { 47 /* Save a 32 bit len */ 48 buf[0] = (REDIS_RDB_32BITLEN<<6); 49 if (rdbWriteRaw(fp,buf,1) == -1) return -1; 50 len = htonl(len); 51 if (rdbWriteRaw(fp,&len,4) == -1) return -1; 52 nwritten = 1+4; 53 } 54 return nwritten; 55 } 56 57 /* Encode 'value' as an integer if possible (if integer will fit the 58 * supported range). If the function sucessful encoded the integer 59 * then the (up to 5 bytes) encoded representation is written in the 60 * string pointed by 'enc' and the length is returned. Otherwise 61 * 0 is returned. */ 62 int rdbEncodeInteger(long long value, unsigned char *enc) { 63 /* Finally check if it fits in our ranges */ 64 if (value >= -(1<<7) && value <= (1<<7)-1) { 65 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT8; 66 enc[1] = value&0xFF; 67 return 2; 68 } else if (value >= -(1<<15) && value <= (1<<15)-1) { 69 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT16; 70 enc[1] = value&0xFF; 71 enc[2] = (value>>8)&0xFF; 72 return 3; 73 } else if (value >= -((long long)1<<31) && value <= ((long long)1<<31)-1) { 74 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT32; 75 enc[1] = value&0xFF; 76 enc[2] = (value>>8)&0xFF; 77 enc[3] = (value>>16)&0xFF; 78 enc[4] = (value>>24)&0xFF; 79 return 5; 80 } else { 81 return 0; 82 } 83 } 84 85 /* String objects in the form "2391" "-100" without any space and with a 86 * range of values that can fit in an 8, 16 or 32 bit signed value can be 87 * encoded as integers to save space */ 88 int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) { 89 long long value; 90 char *endptr, buf[32]; 91 92 /* Check if it's possible to encode this value as a number */ 93 value = strtoll(s, &endptr, 10); 94 if (endptr[0] != '\0') return 0; 95 ll2string(buf,32,value); 96 97 /* If the number converted back into a string is not identical 98 * then it's not possible to encode the string as integer */ 99 if (strlen(buf) != len || memcmp(buf,s,len)) return 0; 100 101 return rdbEncodeInteger(value,enc); 102 } 103 104 int rdbSaveLzfStringObject(FILE *fp, unsigned char *s, size_t len) { 105 size_t comprlen, outlen; 106 unsigned char byte; 107 int n, nwritten = 0; 108 void *out; 109 110 /* We require at least four bytes compression for this to be worth it */ 111 if (len <= 4) return 0; 112 outlen = len-4; 113 if ((out = zmalloc(outlen+1)) == NULL) return 0; 114 comprlen = lzf_compress(s, len, out, outlen); 115 if (comprlen == 0) { 116 zfree(out); 117 return 0; 118 } 119 /* Data compressed! Let's save it on disk */ 120 byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF; 121 if ((n = rdbWriteRaw(fp,&byte,1)) == -1) goto writeerr; 122 nwritten += n; 123 124 if ((n = rdbSaveLen(fp,comprlen)) == -1) goto writeerr; 125 nwritten += n; 126 127 if ((n = rdbSaveLen(fp,len)) == -1) goto writeerr; 128 nwritten += n; 129 130 if ((n = rdbWriteRaw(fp,out,comprlen)) == -1) goto writeerr; 131 nwritten += n; 132 133 zfree(out); 134 return nwritten; 135 136 writeerr: 137 zfree(out); 138 return -1; 139 } 140 141 /* Save a string objet as [len][data] on disk. If the object is a string 142 * representation of an integer value we try to safe it in a special form */ 143 int rdbSaveRawString(FILE *fp, unsigned char *s, size_t len) { 144 int enclen; 145 int n, nwritten = 0; 146 147 /* Try integer encoding */ 148 if (len <= 11) { 149 unsigned char buf[5]; 150 if ((enclen = rdbTryIntegerEncoding((char*)s,len,buf)) > 0) { 151 if (rdbWriteRaw(fp,buf,enclen) == -1) return -1; 152 return enclen; 153 } 154 } 155 156 /* Try LZF compression - under 20 bytes it's unable to compress even 157 * aaaaaaaaaaaaaaaaaa so skip it */ 158 if (server.rdbcompression && len > 20) { 159 n = rdbSaveLzfStringObject(fp,s,len); 160 if (n == -1) return -1; 161 if (n > 0) return n; 162 /* Return value of 0 means data can't be compressed, save the old way */ 163 } 164 165 /* Store verbatim */ 166 if ((n = rdbSaveLen(fp,len)) == -1) return -1; 167 nwritten += n; 168 if (len > 0) { 169 if (rdbWriteRaw(fp,s,len) == -1) return -1; 170 nwritten += len; 171 } 172 return nwritten; 173 } 174 175 /* Save a long long value as either an encoded string or a string. */ 176 int rdbSaveLongLongAsStringObject(FILE *fp, long long value) { 177 unsigned char buf[32]; 178 int n, nwritten = 0; 179 int enclen = rdbEncodeInteger(value,buf); 180 if (enclen > 0) { 181 return rdbWriteRaw(fp,buf,enclen); 182 } else { 183 /* Encode as string */ 184 enclen = ll2string((char*)buf,32,value); 185 redisAssert(enclen < 32); 186 if ((n = rdbSaveLen(fp,enclen)) == -1) return -1; 187 nwritten += n; 188 if ((n = rdbWriteRaw(fp,buf,enclen)) == -1) return -1; 189 nwritten += n; 190 } 191 return nwritten; 192 } 193 194 /* Like rdbSaveStringObjectRaw() but handle encoded objects */ 195 int rdbSaveStringObject(FILE *fp, robj *obj) { 196 /* Avoid to decode the object, then encode it again, if the 197 * object is alrady integer encoded. */ 198 if (obj->encoding == REDIS_ENCODING_INT) { 199 return rdbSaveLongLongAsStringObject(fp,(long)obj->ptr); 200 } else { 201 redisAssert(obj->encoding == REDIS_ENCODING_RAW); 202 return rdbSaveRawString(fp,obj->ptr,sdslen(obj->ptr)); 203 } 204 } 205 206 /* Save a double value. Doubles are saved as strings prefixed by an unsigned 207 * 8 bit integer specifing the length of the representation. 208 * This 8 bit integer has special values in order to specify the following 209 * conditions: 210 * 253: not a number 211 * 254: + inf 212 * 255: - inf 213 */ 214 int rdbSaveDoubleValue(FILE *fp, double val) { 215 unsigned char buf[128]; 216 int len; 217 218 if (isnan(val)) { 219 buf[0] = 253; 220 len = 1; 221 } else if (!isfinite(val)) { 222 len = 1; 223 buf[0] = (val < 0) ? 255 : 254; 224 } else { 225 #if (DBL_MANT_DIG >= 52) && (LLONG_MAX == 0x7fffffffffffffffLL) 226 /* Check if the float is in a safe range to be casted into a 227 * long long. We are assuming that long long is 64 bit here. 228 * Also we are assuming that there are no implementations around where 229 * double has precision < 52 bit. 230 * 231 * Under this assumptions we test if a double is inside an interval 232 * where casting to long long is safe. Then using two castings we 233 * make sure the decimal part is zero. If all this is true we use 234 * integer printing function that is much faster. */ 235 double min = -4503599627370495; /* (2^52)-1 */ 236 double max = 4503599627370496; /* -(2^52) */ 237 if (val > min && val < max && val == ((double)((long long)val))) 238 ll2string((char*)buf+1,sizeof(buf),(long long)val); 239 else 240 #endif 241 snprintf((char*)buf+1,sizeof(buf)-1,"%.17g",val); 242 buf[0] = strlen((char*)buf+1); 243 len = buf[0]+1; 244 } 245 return rdbWriteRaw(fp,buf,len); 246 } 247 248 /* Save a Redis object. */ 249 int rdbSaveObject(FILE *fp, robj *o) { 250 int n, nwritten = 0; 251 252 if (o->type == REDIS_STRING) { 253 /* Save a string value */ 254 if ((n = rdbSaveStringObject(fp,o)) == -1) return -1; 255 nwritten += n; 256 } else if (o->type == REDIS_LIST) { 257 /* Save a list value */ 258 if (o->encoding == REDIS_ENCODING_ZIPLIST) { 259 unsigned char *p; 260 unsigned char *vstr; 261 unsigned int vlen; 262 long long vlong; 263 264 if ((n = rdbSaveLen(fp,ziplistLen(o->ptr))) == -1) return -1; 265 nwritten += n; 266 267 p = ziplistIndex(o->ptr,0); 268 while(ziplistGet(p,&vstr,&vlen,&vlong)) { 269 if (vstr) { 270 if ((n = rdbSaveRawString(fp,vstr,vlen)) == -1) 271 return -1; 272 nwritten += n; 273 } else { 274 if ((n = rdbSaveLongLongAsStringObject(fp,vlong)) == -1) 275 return -1; 276 nwritten += n; 277 } 278 p = ziplistNext(o->ptr,p); 279 } 280 } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { 281 list *list = o->ptr; 282 listIter li; 283 listNode *ln; 284 285 if ((n = rdbSaveLen(fp,listLength(list))) == -1) return -1; 286 nwritten += n; 287 288 listRewind(list,&li); 289 while((ln = listNext(&li))) { 290 robj *eleobj = listNodeValue(ln); 291 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; 292 nwritten += n; 293 } 294 } else { 295 redisPanic("Unknown list encoding"); 296 } 297 } else if (o->type == REDIS_SET) { 298 /* Save a set value */ 299 if (o->encoding == REDIS_ENCODING_HT) { 300 dict *set = o->ptr; 301 dictIterator *di = dictGetIterator(set); 302 dictEntry *de; 303 304 if ((n = rdbSaveLen(fp,dictSize(set))) == -1) return -1; 305 nwritten += n; 306 307 while((de = dictNext(di)) != NULL) { 308 robj *eleobj = dictGetEntryKey(de); 309 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; 310 nwritten += n; 311 } 312 dictReleaseIterator(di); 313 } else if (o->encoding == REDIS_ENCODING_INTSET) { 314 intset *is = o->ptr; 315 int64_t llval; 316 int i = 0; 317 318 if ((n = rdbSaveLen(fp,intsetLen(is))) == -1) return -1; 319 nwritten += n; 320 321 while(intsetGet(is,i++,&llval)) { 322 if ((n = rdbSaveLongLongAsStringObject(fp,llval)) == -1) return -1; 323 nwritten += n; 324 } 325 } else { 326 redisPanic("Unknown set encoding"); 327 } 328 } else if (o->type == REDIS_ZSET) { 329 /* Save a set value */ 330 zset *zs = o->ptr; 331 dictIterator *di = dictGetIterator(zs->dict); 332 dictEntry *de; 333 334 if ((n = rdbSaveLen(fp,dictSize(zs->dict))) == -1) return -1; 335 nwritten += n; 336 337 while((de = dictNext(di)) != NULL) { 338 robj *eleobj = dictGetEntryKey(de); 339 double *score = dictGetEntryVal(de); 340 341 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; 342 nwritten += n; 343 if ((n = rdbSaveDoubleValue(fp,*score)) == -1) return -1; 344 nwritten += n; 345 } 346 dictReleaseIterator(di); 347 } else if (o->type == REDIS_HASH) { 348 /* Save a hash value */ 349 if (o->encoding == REDIS_ENCODING_ZIPMAP) { 350 unsigned char *p = zipmapRewind(o->ptr); 351 unsigned int count = zipmapLen(o->ptr); 352 unsigned char *key, *val; 353 unsigned int klen, vlen; 354 355 if ((n = rdbSaveLen(fp,count)) == -1) return -1; 356 nwritten += n; 357 358 while((p = zipmapNext(p,&key,&klen,&val,&vlen)) != NULL) { 359 if ((n = rdbSaveRawString(fp,key,klen)) == -1) return -1; 360 nwritten += n; 361 if ((n = rdbSaveRawString(fp,val,vlen)) == -1) return -1; 362 nwritten += n; 363 } 364 } else { 365 dictIterator *di = dictGetIterator(o->ptr); 366 dictEntry *de; 367 368 if ((n = rdbSaveLen(fp,dictSize((dict*)o->ptr))) == -1) return -1; 369 nwritten += n; 370 371 while((de = dictNext(di)) != NULL) { 372 robj *key = dictGetEntryKey(de); 373 robj *val = dictGetEntryVal(de); 374 375 if ((n = rdbSaveStringObject(fp,key)) == -1) return -1; 376 nwritten += n; 377 if ((n = rdbSaveStringObject(fp,val)) == -1) return -1; 378 nwritten += n; 379 } 380 dictReleaseIterator(di); 381 } 382 } else { 383 redisPanic("Unknown object type"); 384 } 385 return nwritten; 386 } 387 388 /* Return the length the object will have on disk if saved with 389 * the rdbSaveObject() function. Currently we use a trick to get 390 * this length with very little changes to the code. In the future 391 * we could switch to a faster solution. */ 392 off_t rdbSavedObjectLen(robj *o) { 393 int len = rdbSaveObject(NULL,o); 394 redisAssert(len != -1); 395 return len; 396 } 397 398 /* Save a key-value pair, with expire time, type, key, value. 399 * On error -1 is returned. 400 * On success if the key was actaully saved 1 is returned, otherwise 0 401 * is returned (the key was already expired). */ 402 int rdbSaveKeyValuePair(FILE *fp, redisDb *db, robj *key, robj *val, 403 time_t now) 404 { 405 time_t expiretime; 406 407 expiretime = getExpire(db,key); 408 409 /* Save the expire time */ 410 if (expiretime != -1) { 411 /* If this key is already expired skip it */ 412 if (expiretime < now) return 0; 413 if (rdbSaveType(fp,REDIS_EXPIRETIME) == -1) return -1; 414 if (rdbSaveTime(fp,expiretime) == -1) return -1; 415 } 416 /* Save type, key, value */ 417 if (rdbSaveType(fp,val->type) == -1) return -1; 418 if (rdbSaveStringObject(fp,key) == -1) return -1; 419 if (rdbSaveObject(fp,val) == -1) return -1; 420 return 1; 421 } 422 423 /* Save the DB on disk. Return REDIS_ERR on error, REDIS_OK on success */ 424 int rdbSave(char *filename) { 425 dictIterator *di = NULL; 426 dictEntry *de; 427 FILE *fp; 428 char tmpfile[256]; 429 int j; 430 time_t now = time(NULL); 431 432 /* FIXME: implement .rdb save for disk store properly */ 433 redisAssert(server.ds_enabled == 0); 434 435 snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid()); 436 fp = fopen(tmpfile,"w"); 437 if (!fp) { 438 redisLog(REDIS_WARNING, "Failed saving the DB: %s", strerror(errno)); 439 return REDIS_ERR; 440 } 441 if (fwrite("REDIS0001",9,1,fp) == 0) goto werr; 442 for (j = 0; j < server.dbnum; j++) { 443 redisDb *db = server.db+j; 444 dict *d = db->dict; 445 if (dictSize(d) == 0) continue; 446 di = dictGetIterator(d); 447 if (!di) { 448 fclose(fp); 449 return REDIS_ERR; 450 } 451 452 /* Write the SELECT DB opcode */ 453 if (rdbSaveType(fp,REDIS_SELECTDB) == -1) goto werr; 454 if (rdbSaveLen(fp,j) == -1) goto werr; 455 456 /* Iterate this DB writing every entry */ 457 while((de = dictNext(di)) != NULL) { 458 sds keystr = dictGetEntryKey(de); 459 robj key, *o = dictGetEntryVal(de); 460 461 initStaticStringObject(key,keystr); 462 if (rdbSaveKeyValuePair(fp,db,&key,o,now) == -1) goto werr; 463 } 464 dictReleaseIterator(di); 465 } 466 /* EOF opcode */ 467 if (rdbSaveType(fp,REDIS_EOF) == -1) goto werr; 468 469 /* Make sure data will not remain on the OS's output buffers */ 470 fflush(fp); 471 fsync(fileno(fp)); 472 fclose(fp); 473 474 /* Use RENAME to make sure the DB file is changed atomically only 475 * if the generate DB file is ok. */ 476 if (rename(tmpfile,filename) == -1) { 477 redisLog(REDIS_WARNING,"Error moving temp DB file on the final destination: %s", strerror(errno)); 478 unlink(tmpfile); 479 return REDIS_ERR; 480 } 481 redisLog(REDIS_NOTICE,"DB saved on disk"); 482 server.dirty = 0; 483 server.lastsave = time(NULL); 484 return REDIS_OK; 485 486 werr: 487 fclose(fp); 488 unlink(tmpfile); 489 redisLog(REDIS_WARNING,"Write error saving DB on disk: %s", strerror(errno)); 490 if (di) dictReleaseIterator(di); 491 return REDIS_ERR; 492 } 493 494 int rdbSaveBackground(char *filename) { 495 pid_t childpid; 496 497 if (server.bgsavechildpid != -1) return REDIS_ERR; 498 redisAssert(server.ds_enabled == 0); 499 server.dirty_before_bgsave = server.dirty; 500 if ((childpid = fork()) == 0) { 501 /* Child */ 502 if (server.ipfd > 0) close(server.ipfd); 503 if (server.sofd > 0) close(server.sofd); 504 if (rdbSave(filename) == REDIS_OK) { 505 _exit(0); 506 } else { 507 _exit(1); 508 } 509 } else { 510 /* Parent */ 511 if (childpid == -1) { 512 redisLog(REDIS_WARNING,"Can't save in background: fork: %s", 513 strerror(errno)); 514 return REDIS_ERR; 515 } 516 redisLog(REDIS_NOTICE,"Background saving started by pid %d",childpid); 517 server.bgsavechildpid = childpid; 518 updateDictResizePolicy(); 519 return REDIS_OK; 520 } 521 return REDIS_OK; /* unreached */ 522 } 523 524 void rdbRemoveTempFile(pid_t childpid) { 525 char tmpfile[256]; 526 527 snprintf(tmpfile,256,"temp-%d.rdb", (int) childpid); 528 unlink(tmpfile); 529 } 530 531 int rdbLoadType(FILE *fp) { 532 unsigned char type; 533 if (fread(&type,1,1,fp) == 0) return -1; 534 return type; 535 } 536 537 time_t rdbLoadTime(FILE *fp) { 538 int32_t t32; 539 if (fread(&t32,4,1,fp) == 0) return -1; 540 return (time_t) t32; 541 } 542 543 /* Load an encoded length from the DB, see the REDIS_RDB_* defines on the top 544 * of this file for a description of how this are stored on disk. 545 * 546 * isencoded is set to 1 if the readed length is not actually a length but 547 * an "encoding type", check the above comments for more info */ 548 uint32_t rdbLoadLen(FILE *fp, int *isencoded) { 549 unsigned char buf[2]; 550 uint32_t len; 551 int type; 552 553 if (isencoded) *isencoded = 0; 554 if (fread(buf,1,1,fp) == 0) return REDIS_RDB_LENERR; 555 type = (buf[0]&0xC0)>>6; 556 if (type == REDIS_RDB_6BITLEN) { 557 /* Read a 6 bit len */ 558 return buf[0]&0x3F; 559 } else if (type == REDIS_RDB_ENCVAL) { 560 /* Read a 6 bit len encoding type */ 561 if (isencoded) *isencoded = 1; 562 return buf[0]&0x3F; 563 } else if (type == REDIS_RDB_14BITLEN) { 564 /* Read a 14 bit len */ 565 if (fread(buf+1,1,1,fp) == 0) return REDIS_RDB_LENERR; 566 return ((buf[0]&0x3F)<<8)|buf[1]; 567 } else { 568 /* Read a 32 bit len */ 569 if (fread(&len,4,1,fp) == 0) return REDIS_RDB_LENERR; 570 return ntohl(len); 571 } 572 } 573 574 /* Load an integer-encoded object from file 'fp', with the specified 575 * encoding type 'enctype'. If encode is true the function may return 576 * an integer-encoded object as reply, otherwise the returned object 577 * will always be encoded as a raw string. */ 578 robj *rdbLoadIntegerObject(FILE *fp, int enctype, int encode) { 579 unsigned char enc[4]; 580 long long val; 581 582 if (enctype == REDIS_RDB_ENC_INT8) { 583 if (fread(enc,1,1,fp) == 0) return NULL; 584 val = (signed char)enc[0]; 585 } else if (enctype == REDIS_RDB_ENC_INT16) { 586 uint16_t v; 587 if (fread(enc,2,1,fp) == 0) return NULL; 588 v = enc[0]|(enc[1]<<8); 589 val = (int16_t)v; 590 } else if (enctype == REDIS_RDB_ENC_INT32) { 591 uint32_t v; 592 if (fread(enc,4,1,fp) == 0) return NULL; 593 v = enc[0]|(enc[1]<<8)|(enc[2]<<16)|(enc[3]<<24); 594 val = (int32_t)v; 595 } else { 596 val = 0; /* anti-warning */ 597 redisPanic("Unknown RDB integer encoding type"); 598 } 599 if (encode) 600 return createStringObjectFromLongLong(val); 601 else 602 return createObject(REDIS_STRING,sdsfromlonglong(val)); 603 } 604 605 robj *rdbLoadLzfStringObject(FILE*fp) { 606 unsigned int len, clen; 607 unsigned char *c = NULL; 608 sds val = NULL; 609 610 if ((clen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 611 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 612 if ((c = zmalloc(clen)) == NULL) goto err; 613 if ((val = sdsnewlen(NULL,len)) == NULL) goto err; 614 if (fread(c,clen,1,fp) == 0) goto err; 615 if (lzf_decompress(c,clen,val,len) == 0) goto err; 616 zfree(c); 617 return createObject(REDIS_STRING,val); 618 err: 619 zfree(c); 620 sdsfree(val); 621 return NULL; 622 } 623 624 robj *rdbGenericLoadStringObject(FILE*fp, int encode) { 625 int isencoded; 626 uint32_t len; 627 sds val; 628 629 len = rdbLoadLen(fp,&isencoded); 630 if (isencoded) { 631 switch(len) { 632 case REDIS_RDB_ENC_INT8: 633 case REDIS_RDB_ENC_INT16: 634 case REDIS_RDB_ENC_INT32: 635 return rdbLoadIntegerObject(fp,len,encode); 636 case REDIS_RDB_ENC_LZF: 637 return rdbLoadLzfStringObject(fp); 638 default: 639 redisPanic("Unknown RDB encoding type"); 640 } 641 } 642 643 if (len == REDIS_RDB_LENERR) return NULL; 644 val = sdsnewlen(NULL,len); 645 if (len && fread(val,len,1,fp) == 0) { 646 sdsfree(val); 647 return NULL; 648 } 649 return createObject(REDIS_STRING,val); 650 } 651 652 robj *rdbLoadStringObject(FILE *fp) { 653 return rdbGenericLoadStringObject(fp,0); 654 } 655 656 robj *rdbLoadEncodedStringObject(FILE *fp) { 657 return rdbGenericLoadStringObject(fp,1); 658 } 659 660 /* For information about double serialization check rdbSaveDoubleValue() */ 661 int rdbLoadDoubleValue(FILE *fp, double *val) { 662 char buf[128]; 663 unsigned char len; 664 665 if (fread(&len,1,1,fp) == 0) return -1; 666 switch(len) { 667 case 255: *val = R_NegInf; return 0; 668 case 254: *val = R_PosInf; return 0; 669 case 253: *val = R_Nan; return 0; 670 default: 671 if (fread(buf,len,1,fp) == 0) return -1; 672 buf[len] = '\0'; 673 sscanf(buf, "%lg", val); 674 return 0; 675 } 676 } 677 678 /* Load a Redis object of the specified type from the specified file. 679 * On success a newly allocated object is returned, otherwise NULL. */ 680 robj *rdbLoadObject(int type, FILE *fp) { 681 robj *o, *ele, *dec; 682 size_t len; 683 unsigned int i; 684 685 redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",type,ftell(fp)); 686 if (type == REDIS_STRING) { 687 /* Read string value */ 688 if ((o = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 689 o = tryObjectEncoding(o); 690 } else if (type == REDIS_LIST) { 691 /* Read list value */ 692 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 693 694 /* Use a real list when there are too many entries */ 695 if (len > server.list_max_ziplist_entries) { 696 o = createListObject(); 697 } else { 698 o = createZiplistObject(); 699 } 700 701 /* Load every single element of the list */ 702 while(len--) { 703 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 704 705 /* If we are using a ziplist and the value is too big, convert 706 * the object to a real list. */ 707 if (o->encoding == REDIS_ENCODING_ZIPLIST && 708 ele->encoding == REDIS_ENCODING_RAW && 709 sdslen(ele->ptr) > server.list_max_ziplist_value) 710 listTypeConvert(o,REDIS_ENCODING_LINKEDLIST); 711 712 if (o->encoding == REDIS_ENCODING_ZIPLIST) { 713 dec = getDecodedObject(ele); 714 o->ptr = ziplistPush(o->ptr,dec->ptr,sdslen(dec->ptr),REDIS_TAIL); 715 decrRefCount(dec); 716 decrRefCount(ele); 717 } else { 718 ele = tryObjectEncoding(ele); 719 listAddNodeTail(o->ptr,ele); 720 } 721 } 722 } else if (type == REDIS_SET) { 723 /* Read list/set value */ 724 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 725 726 /* Use a regular set when there are too many entries. */ 727 if (len > server.set_max_intset_entries) { 728 o = createSetObject(); 729 /* It's faster to expand the dict to the right size asap in order 730 * to avoid rehashing */ 731 if (len > DICT_HT_INITIAL_SIZE) 732 dictExpand(o->ptr,len); 733 } else { 734 o = createIntsetObject(); 735 } 736 737 /* Load every single element of the list/set */ 738 for (i = 0; i < len; i++) { 739 long long llval; 740 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 741 ele = tryObjectEncoding(ele); 742 743 if (o->encoding == REDIS_ENCODING_INTSET) { 744 /* Fetch integer value from element */ 745 if (isObjectRepresentableAsLongLong(ele,&llval) == REDIS_OK) { 746 o->ptr = intsetAdd(o->ptr,llval,NULL); 747 } else { 748 setTypeConvert(o,REDIS_ENCODING_HT); 749 dictExpand(o->ptr,len); 750 } 751 } 752 753 /* This will also be called when the set was just converted 754 * to regular hashtable encoded set */ 755 if (o->encoding == REDIS_ENCODING_HT) { 756 dictAdd((dict*)o->ptr,ele,NULL); 757 } else { 758 decrRefCount(ele); 759 } 760 } 761 } else if (type == REDIS_ZSET) { 762 /* Read list/set value */ 763 size_t zsetlen; 764 zset *zs; 765 766 if ((zsetlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 767 o = createZsetObject(); 768 zs = o->ptr; 769 /* Load every single element of the list/set */ 770 while(zsetlen--) { 771 robj *ele; 772 double score; 773 zskiplistNode *znode; 774 775 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 776 ele = tryObjectEncoding(ele); 777 if (rdbLoadDoubleValue(fp,&score) == -1) return NULL; 778 znode = zslInsert(zs->zsl,score,ele); 779 dictAdd(zs->dict,ele,&znode->score); 780 incrRefCount(ele); /* added to skiplist */ 781 } 782 } else if (type == REDIS_HASH) { 783 size_t hashlen; 784 785 if ((hashlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 786 o = createHashObject(); 787 /* Too many entries? Use an hash table. */ 788 if (hashlen > server.hash_max_zipmap_entries) 789 convertToRealHash(o); 790 /* Load every key/value, then set it into the zipmap or hash 791 * table, as needed. */ 792 while(hashlen--) { 793 robj *key, *val; 794 795 if ((key = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 796 if ((val = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 797 /* If we are using a zipmap and there are too big values 798 * the object is converted to real hash table encoding. */ 799 if (o->encoding != REDIS_ENCODING_HT && 800 ((key->encoding == REDIS_ENCODING_RAW && 801 sdslen(key->ptr) > server.hash_max_zipmap_value) || 802 (val->encoding == REDIS_ENCODING_RAW && 803 sdslen(val->ptr) > server.hash_max_zipmap_value))) 804 { 805 convertToRealHash(o); 806 } 807 808 if (o->encoding == REDIS_ENCODING_ZIPMAP) { 809 unsigned char *zm = o->ptr; 810 robj *deckey, *decval; 811 812 /* We need raw string objects to add them to the zipmap */ 813 deckey = getDecodedObject(key); 814 decval = getDecodedObject(val); 815 zm = zipmapSet(zm,deckey->ptr,sdslen(deckey->ptr), 816 decval->ptr,sdslen(decval->ptr),NULL); 817 o->ptr = zm; 818 decrRefCount(deckey); 819 decrRefCount(decval); 820 decrRefCount(key); 821 decrRefCount(val); 822 } else { 823 key = tryObjectEncoding(key); 824 val = tryObjectEncoding(val); 825 dictAdd((dict*)o->ptr,key,val); 826 } 827 } 828 } else { 829 redisPanic("Unknown object type"); 830 } 831 return o; 832 } 833 834 /* Mark that we are loading in the global state and setup the fields 835 * needed to provide loading stats. */ 836 void startLoading(FILE *fp) { 837 struct stat sb; 838 839 /* Load the DB */ 840 server.loading = 1; 841 server.loading_start_time = time(NULL); 842 if (fstat(fileno(fp), &sb) == -1) { 843 server.loading_total_bytes = 1; /* just to avoid division by zero */ 844 } else { 845 server.loading_total_bytes = sb.st_size; 846 } 847 } 848 849 /* Refresh the loading progress info */ 850 void loadingProgress(off_t pos) { 851 server.loading_loaded_bytes = pos; 852 } 853 854 /* Loading finished */ 855 void stopLoading(void) { 856 server.loading = 0; 857 } 858 859 int rdbLoad(char *filename) { 860 FILE *fp; 861 uint32_t dbid; 862 int type, retval, rdbver; 863 redisDb *db = server.db+0; 864 char buf[1024]; 865 time_t expiretime, now = time(NULL); 866 long loops = 0; 867 868 fp = fopen(filename,"r"); 869 if (!fp) return REDIS_ERR; 870 if (fread(buf,9,1,fp) == 0) goto eoferr; 871 buf[9] = '\0'; 872 if (memcmp(buf,"REDIS",5) != 0) { 873 fclose(fp); 874 redisLog(REDIS_WARNING,"Wrong signature trying to load DB from file"); 875 return REDIS_ERR; 876 } 877 rdbver = atoi(buf+5); 878 if (rdbver != 1) { 879 fclose(fp); 880 redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver); 881 return REDIS_ERR; 882 } 883 884 startLoading(fp); 885 while(1) { 886 robj *key, *val; 887 expiretime = -1; 888 889 /* Serve the clients from time to time */ 890 if (!(loops++ % 1000)) { 891 loadingProgress(ftello(fp)); 892 aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); 893 } 894 895 /* Read type. */ 896 if ((type = rdbLoadType(fp)) == -1) goto eoferr; 897 if (type == REDIS_EXPIRETIME) { 898 if ((expiretime = rdbLoadTime(fp)) == -1) goto eoferr; 899 /* We read the time so we need to read the object type again */ 900 if ((type = rdbLoadType(fp)) == -1) goto eoferr; 901 } 902 if (type == REDIS_EOF) break; 903 /* Handle SELECT DB opcode as a special case */ 904 if (type == REDIS_SELECTDB) { 905 if ((dbid = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) 906 goto eoferr; 907 if (dbid >= (unsigned)server.dbnum) { 908 redisLog(REDIS_WARNING,"FATAL: Data file was created with a Redis server configured to handle more than %d databases. Exiting\n", server.dbnum); 909 exit(1); 910 } 911 db = server.db+dbid; 912 continue; 913 } 914 /* Read key */ 915 if ((key = rdbLoadStringObject(fp)) == NULL) goto eoferr; 916 /* Read value */ 917 if ((val = rdbLoadObject(type,fp)) == NULL) goto eoferr; 918 /* Check if the key already expired */ 919 if (expiretime != -1 && expiretime < now) { 920 decrRefCount(key); 921 decrRefCount(val); 922 continue; 923 } 924 /* Add the new object in the hash table */ 925 retval = dbAdd(db,key,val); 926 if (retval == REDIS_ERR) { 927 redisLog(REDIS_WARNING,"Loading DB, duplicated key (%s) found! Unrecoverable error, exiting now.", key->ptr); 928 exit(1); 929 } 930 /* Set the expire time if needed */ 931 if (expiretime != -1) setExpire(db,key,expiretime); 932 933 decrRefCount(key); 934 } 935 fclose(fp); 936 stopLoading(); 937 return REDIS_OK; 938 939 eoferr: /* unexpected end of file is handled here with a fatal exit */ 940 redisLog(REDIS_WARNING,"Short read or OOM loading DB. Unrecoverable error, aborting now."); 941 exit(1); 942 return REDIS_ERR; /* Just to avoid warning */ 943 } 944 945 /* A background saving child (BGSAVE) terminated its work. Handle this. */ 946 void backgroundSaveDoneHandler(int statloc) { 947 int exitcode = WEXITSTATUS(statloc); 948 int bysignal = WIFSIGNALED(statloc); 949 950 if (!bysignal && exitcode == 0) { 951 redisLog(REDIS_NOTICE, 952 "Background saving terminated with success"); 953 server.dirty = server.dirty - server.dirty_before_bgsave; 954 server.lastsave = time(NULL); 955 } else if (!bysignal && exitcode != 0) { 956 redisLog(REDIS_WARNING, "Background saving error"); 957 } else { 958 redisLog(REDIS_WARNING, 959 "Background saving terminated by signal %d", WTERMSIG(statloc)); 960 rdbRemoveTempFile(server.bgsavechildpid); 961 } 962 server.bgsavechildpid = -1; 963 /* Possibly there are slaves waiting for a BGSAVE in order to be served 964 * (the first stage of SYNC is a bulk transfer of dump.rdb) */ 965 updateSlavesWaitingBgsave(exitcode == 0 ? REDIS_OK : REDIS_ERR); 966 } 967