1 #include "redis.h" 2 #include "lzf.h" /* LZF compression library */ 3 4 #include <math.h> 5 #include <sys/types.h> 6 #include <sys/time.h> 7 #include <sys/resource.h> 8 #include <sys/wait.h> 9 #include <arpa/inet.h> 10 #include <sys/stat.h> 11 12 /* Convenience wrapper around fwrite, that returns the number of bytes written 13 * to the file instead of the number of objects (see fwrite(3)) and -1 in the 14 * case of an error. It also supports a NULL *fp to skip writing altogether 15 * instead of writing to /dev/null. */ 16 static int rdbWriteRaw(FILE *fp, void *p, size_t len) { 17 if (fp != NULL && fwrite(p,len,1,fp) == 0) return -1; 18 return len; 19 } 20 21 int rdbSaveType(FILE *fp, unsigned char type) { 22 return rdbWriteRaw(fp,&type,1); 23 } 24 25 int rdbSaveTime(FILE *fp, time_t t) { 26 int32_t t32 = (int32_t) t; 27 return rdbWriteRaw(fp,&t32,4); 28 } 29 30 /* check rdbLoadLen() comments for more info */ 31 int rdbSaveLen(FILE *fp, uint32_t len) { 32 unsigned char buf[2]; 33 int nwritten; 34 35 if (len < (1<<6)) { 36 /* Save a 6 bit len */ 37 buf[0] = (len&0xFF)|(REDIS_RDB_6BITLEN<<6); 38 if (rdbWriteRaw(fp,buf,1) == -1) return -1; 39 nwritten = 1; 40 } else if (len < (1<<14)) { 41 /* Save a 14 bit len */ 42 buf[0] = ((len>>8)&0xFF)|(REDIS_RDB_14BITLEN<<6); 43 buf[1] = len&0xFF; 44 if (rdbWriteRaw(fp,buf,2) == -1) return -1; 45 nwritten = 2; 46 } else { 47 /* Save a 32 bit len */ 48 buf[0] = (REDIS_RDB_32BITLEN<<6); 49 if (rdbWriteRaw(fp,buf,1) == -1) return -1; 50 len = htonl(len); 51 if (rdbWriteRaw(fp,&len,4) == -1) return -1; 52 nwritten = 1+4; 53 } 54 return nwritten; 55 } 56 57 /* Encode 'value' as an integer if possible (if integer will fit the 58 * supported range). If the function sucessful encoded the integer 59 * then the (up to 5 bytes) encoded representation is written in the 60 * string pointed by 'enc' and the length is returned. Otherwise 61 * 0 is returned. */ 62 int rdbEncodeInteger(long long value, unsigned char *enc) { 63 /* Finally check if it fits in our ranges */ 64 if (value >= -(1<<7) && value <= (1<<7)-1) { 65 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT8; 66 enc[1] = value&0xFF; 67 return 2; 68 } else if (value >= -(1<<15) && value <= (1<<15)-1) { 69 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT16; 70 enc[1] = value&0xFF; 71 enc[2] = (value>>8)&0xFF; 72 return 3; 73 } else if (value >= -((long long)1<<31) && value <= ((long long)1<<31)-1) { 74 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT32; 75 enc[1] = value&0xFF; 76 enc[2] = (value>>8)&0xFF; 77 enc[3] = (value>>16)&0xFF; 78 enc[4] = (value>>24)&0xFF; 79 return 5; 80 } else { 81 return 0; 82 } 83 } 84 85 /* String objects in the form "2391" "-100" without any space and with a 86 * range of values that can fit in an 8, 16 or 32 bit signed value can be 87 * encoded as integers to save space */ 88 int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) { 89 long long value; 90 char *endptr, buf[32]; 91 92 /* Check if it's possible to encode this value as a number */ 93 value = strtoll(s, &endptr, 10); 94 if (endptr[0] != '\0') return 0; 95 ll2string(buf,32,value); 96 97 /* If the number converted back into a string is not identical 98 * then it's not possible to encode the string as integer */ 99 if (strlen(buf) != len || memcmp(buf,s,len)) return 0; 100 101 return rdbEncodeInteger(value,enc); 102 } 103 104 int rdbSaveLzfStringObject(FILE *fp, unsigned char *s, size_t len) { 105 size_t comprlen, outlen; 106 unsigned char byte; 107 int n, nwritten = 0; 108 void *out; 109 110 /* We require at least four bytes compression for this to be worth it */ 111 if (len <= 4) return 0; 112 outlen = len-4; 113 if ((out = zmalloc(outlen+1)) == NULL) return 0; 114 comprlen = lzf_compress(s, len, out, outlen); 115 if (comprlen == 0) { 116 zfree(out); 117 return 0; 118 } 119 /* Data compressed! Let's save it on disk */ 120 byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF; 121 if ((n = rdbWriteRaw(fp,&byte,1)) == -1) goto writeerr; 122 nwritten += n; 123 124 if ((n = rdbSaveLen(fp,comprlen)) == -1) goto writeerr; 125 nwritten += n; 126 127 if ((n = rdbSaveLen(fp,len)) == -1) goto writeerr; 128 nwritten += n; 129 130 if ((n = rdbWriteRaw(fp,out,comprlen)) == -1) goto writeerr; 131 nwritten += n; 132 133 zfree(out); 134 return nwritten; 135 136 writeerr: 137 zfree(out); 138 return -1; 139 } 140 141 /* Save a string objet as [len][data] on disk. If the object is a string 142 * representation of an integer value we try to safe it in a special form */ 143 int rdbSaveRawString(FILE *fp, unsigned char *s, size_t len) { 144 int enclen; 145 int n, nwritten = 0; 146 147 /* Try integer encoding */ 148 if (len <= 11) { 149 unsigned char buf[5]; 150 if ((enclen = rdbTryIntegerEncoding((char*)s,len,buf)) > 0) { 151 if (rdbWriteRaw(fp,buf,enclen) == -1) return -1; 152 return enclen; 153 } 154 } 155 156 /* Try LZF compression - under 20 bytes it's unable to compress even 157 * aaaaaaaaaaaaaaaaaa so skip it */ 158 if (server.rdbcompression && len > 20) { 159 n = rdbSaveLzfStringObject(fp,s,len); 160 if (n == -1) return -1; 161 if (n > 0) return n; 162 /* Return value of 0 means data can't be compressed, save the old way */ 163 } 164 165 /* Store verbatim */ 166 if ((n = rdbSaveLen(fp,len)) == -1) return -1; 167 nwritten += n; 168 if (len > 0) { 169 if (rdbWriteRaw(fp,s,len) == -1) return -1; 170 nwritten += len; 171 } 172 return nwritten; 173 } 174 175 /* Save a long long value as either an encoded string or a string. */ 176 int rdbSaveLongLongAsStringObject(FILE *fp, long long value) { 177 unsigned char buf[32]; 178 int n, nwritten = 0; 179 int enclen = rdbEncodeInteger(value,buf); 180 if (enclen > 0) { 181 return rdbWriteRaw(fp,buf,enclen); 182 } else { 183 /* Encode as string */ 184 enclen = ll2string((char*)buf,32,value); 185 redisAssert(enclen < 32); 186 if ((n = rdbSaveLen(fp,enclen)) == -1) return -1; 187 nwritten += n; 188 if ((n = rdbWriteRaw(fp,buf,enclen)) == -1) return -1; 189 nwritten += n; 190 } 191 return nwritten; 192 } 193 194 /* Like rdbSaveStringObjectRaw() but handle encoded objects */ 195 int rdbSaveStringObject(FILE *fp, robj *obj) { 196 /* Avoid to decode the object, then encode it again, if the 197 * object is alrady integer encoded. */ 198 if (obj->encoding == REDIS_ENCODING_INT) { 199 return rdbSaveLongLongAsStringObject(fp,(long)obj->ptr); 200 } else { 201 redisAssert(obj->encoding == REDIS_ENCODING_RAW); 202 return rdbSaveRawString(fp,obj->ptr,sdslen(obj->ptr)); 203 } 204 } 205 206 /* Save a double value. Doubles are saved as strings prefixed by an unsigned 207 * 8 bit integer specifing the length of the representation. 208 * This 8 bit integer has special values in order to specify the following 209 * conditions: 210 * 253: not a number 211 * 254: + inf 212 * 255: - inf 213 */ 214 int rdbSaveDoubleValue(FILE *fp, double val) { 215 unsigned char buf[128]; 216 int len; 217 218 if (isnan(val)) { 219 buf[0] = 253; 220 len = 1; 221 } else if (!isfinite(val)) { 222 len = 1; 223 buf[0] = (val < 0) ? 255 : 254; 224 } else { 225 #if (DBL_MANT_DIG >= 52) && (LLONG_MAX == 0x7fffffffffffffffLL) 226 /* Check if the float is in a safe range to be casted into a 227 * long long. We are assuming that long long is 64 bit here. 228 * Also we are assuming that there are no implementations around where 229 * double has precision < 52 bit. 230 * 231 * Under this assumptions we test if a double is inside an interval 232 * where casting to long long is safe. Then using two castings we 233 * make sure the decimal part is zero. If all this is true we use 234 * integer printing function that is much faster. */ 235 double min = -4503599627370495; /* (2^52)-1 */ 236 double max = 4503599627370496; /* -(2^52) */ 237 if (val > min && val < max && val == ((double)((long long)val))) 238 ll2string((char*)buf+1,sizeof(buf),(long long)val); 239 else 240 #endif 241 snprintf((char*)buf+1,sizeof(buf)-1,"%.17g",val); 242 buf[0] = strlen((char*)buf+1); 243 len = buf[0]+1; 244 } 245 return rdbWriteRaw(fp,buf,len); 246 } 247 248 /* Save a Redis object. */ 249 int rdbSaveObject(FILE *fp, robj *o) { 250 int n, nwritten = 0; 251 252 if (o->type == REDIS_STRING) { 253 /* Save a string value */ 254 if ((n = rdbSaveStringObject(fp,o)) == -1) return -1; 255 nwritten += n; 256 } else if (o->type == REDIS_LIST) { 257 /* Save a list value */ 258 if (o->encoding == REDIS_ENCODING_ZIPLIST) { 259 unsigned char *p; 260 unsigned char *vstr; 261 unsigned int vlen; 262 long long vlong; 263 264 if ((n = rdbSaveLen(fp,ziplistLen(o->ptr))) == -1) return -1; 265 nwritten += n; 266 267 p = ziplistIndex(o->ptr,0); 268 while(ziplistGet(p,&vstr,&vlen,&vlong)) { 269 if (vstr) { 270 if ((n = rdbSaveRawString(fp,vstr,vlen)) == -1) 271 return -1; 272 nwritten += n; 273 } else { 274 if ((n = rdbSaveLongLongAsStringObject(fp,vlong)) == -1) 275 return -1; 276 nwritten += n; 277 } 278 p = ziplistNext(o->ptr,p); 279 } 280 } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { 281 list *list = o->ptr; 282 listIter li; 283 listNode *ln; 284 285 if ((n = rdbSaveLen(fp,listLength(list))) == -1) return -1; 286 nwritten += n; 287 288 listRewind(list,&li); 289 while((ln = listNext(&li))) { 290 robj *eleobj = listNodeValue(ln); 291 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; 292 nwritten += n; 293 } 294 } else { 295 redisPanic("Unknown list encoding"); 296 } 297 } else if (o->type == REDIS_SET) { 298 /* Save a set value */ 299 if (o->encoding == REDIS_ENCODING_HT) { 300 dict *set = o->ptr; 301 dictIterator *di = dictGetIterator(set); 302 dictEntry *de; 303 304 if ((n = rdbSaveLen(fp,dictSize(set))) == -1) return -1; 305 nwritten += n; 306 307 while((de = dictNext(di)) != NULL) { 308 robj *eleobj = dictGetEntryKey(de); 309 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; 310 nwritten += n; 311 } 312 dictReleaseIterator(di); 313 } else if (o->encoding == REDIS_ENCODING_INTSET) { 314 intset *is = o->ptr; 315 int64_t llval; 316 int i = 0; 317 318 if ((n = rdbSaveLen(fp,intsetLen(is))) == -1) return -1; 319 nwritten += n; 320 321 while(intsetGet(is,i++,&llval)) { 322 if ((n = rdbSaveLongLongAsStringObject(fp,llval)) == -1) return -1; 323 nwritten += n; 324 } 325 } else { 326 redisPanic("Unknown set encoding"); 327 } 328 } else if (o->type == REDIS_ZSET) { 329 /* Save a set value */ 330 zset *zs = o->ptr; 331 dictIterator *di = dictGetIterator(zs->dict); 332 dictEntry *de; 333 334 if ((n = rdbSaveLen(fp,dictSize(zs->dict))) == -1) return -1; 335 nwritten += n; 336 337 while((de = dictNext(di)) != NULL) { 338 robj *eleobj = dictGetEntryKey(de); 339 double *score = dictGetEntryVal(de); 340 341 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; 342 nwritten += n; 343 if ((n = rdbSaveDoubleValue(fp,*score)) == -1) return -1; 344 nwritten += n; 345 } 346 dictReleaseIterator(di); 347 } else if (o->type == REDIS_HASH) { 348 /* Save a hash value */ 349 if (o->encoding == REDIS_ENCODING_ZIPMAP) { 350 unsigned char *p = zipmapRewind(o->ptr); 351 unsigned int count = zipmapLen(o->ptr); 352 unsigned char *key, *val; 353 unsigned int klen, vlen; 354 355 if ((n = rdbSaveLen(fp,count)) == -1) return -1; 356 nwritten += n; 357 358 while((p = zipmapNext(p,&key,&klen,&val,&vlen)) != NULL) { 359 if ((n = rdbSaveRawString(fp,key,klen)) == -1) return -1; 360 nwritten += n; 361 if ((n = rdbSaveRawString(fp,val,vlen)) == -1) return -1; 362 nwritten += n; 363 } 364 } else { 365 dictIterator *di = dictGetIterator(o->ptr); 366 dictEntry *de; 367 368 if ((n = rdbSaveLen(fp,dictSize((dict*)o->ptr))) == -1) return -1; 369 nwritten += n; 370 371 while((de = dictNext(di)) != NULL) { 372 robj *key = dictGetEntryKey(de); 373 robj *val = dictGetEntryVal(de); 374 375 if ((n = rdbSaveStringObject(fp,key)) == -1) return -1; 376 nwritten += n; 377 if ((n = rdbSaveStringObject(fp,val)) == -1) return -1; 378 nwritten += n; 379 } 380 dictReleaseIterator(di); 381 } 382 } else { 383 redisPanic("Unknown object type"); 384 } 385 return nwritten; 386 } 387 388 /* Return the length the object will have on disk if saved with 389 * the rdbSaveObject() function. Currently we use a trick to get 390 * this length with very little changes to the code. In the future 391 * we could switch to a faster solution. */ 392 off_t rdbSavedObjectLen(robj *o) { 393 int len = rdbSaveObject(NULL,o); 394 redisAssert(len != -1); 395 return len; 396 } 397 398 /* Save a key-value pair, with expire time, type, key, value. 399 * On error -1 is returned. 400 * On success if the key was actaully saved 1 is returned, otherwise 0 401 * is returned (the key was already expired). */ 402 int rdbSaveKeyValuePair(FILE *fp, redisDb *db, robj *key, robj *val, 403 time_t now) 404 { 405 time_t expiretime; 406 407 expiretime = getExpire(db,key); 408 409 /* Save the expire time */ 410 if (expiretime != -1) { 411 /* If this key is already expired skip it */ 412 if (expiretime < now) return 0; 413 if (rdbSaveType(fp,REDIS_EXPIRETIME) == -1) return -1; 414 if (rdbSaveTime(fp,expiretime) == -1) return -1; 415 } 416 /* Save type, key, value */ 417 if (rdbSaveType(fp,val->type) == -1) return -1; 418 if (rdbSaveStringObject(fp,key) == -1) return -1; 419 if (rdbSaveObject(fp,val) == -1) return -1; 420 return 1; 421 } 422 423 /* Save the DB on disk. Return REDIS_ERR on error, REDIS_OK on success */ 424 int rdbSave(char *filename) { 425 dictIterator *di = NULL; 426 dictEntry *de; 427 FILE *fp; 428 char tmpfile[256]; 429 int j; 430 time_t now = time(NULL); 431 432 if (server.ds_enabled) { 433 cacheForcePointInTime(); 434 return dsRdbSave(filename); 435 } 436 437 snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid()); 438 fp = fopen(tmpfile,"w"); 439 if (!fp) { 440 redisLog(REDIS_WARNING, "Failed opening .rdb for saving: %s", 441 strerror(errno)); 442 return REDIS_ERR; 443 } 444 if (fwrite("REDIS0001",9,1,fp) == 0) goto werr; 445 for (j = 0; j < server.dbnum; j++) { 446 redisDb *db = server.db+j; 447 dict *d = db->dict; 448 if (dictSize(d) == 0) continue; 449 di = dictGetIterator(d); 450 if (!di) { 451 fclose(fp); 452 return REDIS_ERR; 453 } 454 455 /* Write the SELECT DB opcode */ 456 if (rdbSaveType(fp,REDIS_SELECTDB) == -1) goto werr; 457 if (rdbSaveLen(fp,j) == -1) goto werr; 458 459 /* Iterate this DB writing every entry */ 460 while((de = dictNext(di)) != NULL) { 461 sds keystr = dictGetEntryKey(de); 462 robj key, *o = dictGetEntryVal(de); 463 464 initStaticStringObject(key,keystr); 465 if (rdbSaveKeyValuePair(fp,db,&key,o,now) == -1) goto werr; 466 } 467 dictReleaseIterator(di); 468 } 469 /* EOF opcode */ 470 if (rdbSaveType(fp,REDIS_EOF) == -1) goto werr; 471 472 /* Make sure data will not remain on the OS's output buffers */ 473 fflush(fp); 474 fsync(fileno(fp)); 475 fclose(fp); 476 477 /* Use RENAME to make sure the DB file is changed atomically only 478 * if the generate DB file is ok. */ 479 if (rename(tmpfile,filename) == -1) { 480 redisLog(REDIS_WARNING,"Error moving temp DB file on the final destination: %s", strerror(errno)); 481 unlink(tmpfile); 482 return REDIS_ERR; 483 } 484 redisLog(REDIS_NOTICE,"DB saved on disk"); 485 server.dirty = 0; 486 server.lastsave = time(NULL); 487 return REDIS_OK; 488 489 werr: 490 fclose(fp); 491 unlink(tmpfile); 492 redisLog(REDIS_WARNING,"Write error saving DB on disk: %s", strerror(errno)); 493 if (di) dictReleaseIterator(di); 494 return REDIS_ERR; 495 } 496 497 int rdbSaveBackground(char *filename) { 498 pid_t childpid; 499 500 if (server.bgsavechildpid != -1 || 501 server.bgsavethread != (pthread_t) -1) return REDIS_ERR; 502 503 server.dirty_before_bgsave = server.dirty; 504 505 if (server.ds_enabled) { 506 cacheForcePointInTime(); 507 return dsRdbSaveBackground(filename); 508 } 509 510 if ((childpid = fork()) == 0) { 511 int retval; 512 513 /* Child */ 514 if (server.ipfd > 0) close(server.ipfd); 515 if (server.sofd > 0) close(server.sofd); 516 retval = rdbSave(filename); 517 _exit((retval == REDIS_OK) ? 0 : 1); 518 } else { 519 /* Parent */ 520 if (childpid == -1) { 521 redisLog(REDIS_WARNING,"Can't save in background: fork: %s", 522 strerror(errno)); 523 return REDIS_ERR; 524 } 525 redisLog(REDIS_NOTICE,"Background saving started by pid %d",childpid); 526 server.bgsavechildpid = childpid; 527 updateDictResizePolicy(); 528 return REDIS_OK; 529 } 530 return REDIS_OK; /* unreached */ 531 } 532 533 void rdbRemoveTempFile(pid_t childpid) { 534 char tmpfile[256]; 535 536 snprintf(tmpfile,256,"temp-%d.rdb", (int) childpid); 537 unlink(tmpfile); 538 } 539 540 int rdbLoadType(FILE *fp) { 541 unsigned char type; 542 if (fread(&type,1,1,fp) == 0) return -1; 543 return type; 544 } 545 546 time_t rdbLoadTime(FILE *fp) { 547 int32_t t32; 548 if (fread(&t32,4,1,fp) == 0) return -1; 549 return (time_t) t32; 550 } 551 552 /* Load an encoded length from the DB, see the REDIS_RDB_* defines on the top 553 * of this file for a description of how this are stored on disk. 554 * 555 * isencoded is set to 1 if the readed length is not actually a length but 556 * an "encoding type", check the above comments for more info */ 557 uint32_t rdbLoadLen(FILE *fp, int *isencoded) { 558 unsigned char buf[2]; 559 uint32_t len; 560 int type; 561 562 if (isencoded) *isencoded = 0; 563 if (fread(buf,1,1,fp) == 0) return REDIS_RDB_LENERR; 564 type = (buf[0]&0xC0)>>6; 565 if (type == REDIS_RDB_6BITLEN) { 566 /* Read a 6 bit len */ 567 return buf[0]&0x3F; 568 } else if (type == REDIS_RDB_ENCVAL) { 569 /* Read a 6 bit len encoding type */ 570 if (isencoded) *isencoded = 1; 571 return buf[0]&0x3F; 572 } else if (type == REDIS_RDB_14BITLEN) { 573 /* Read a 14 bit len */ 574 if (fread(buf+1,1,1,fp) == 0) return REDIS_RDB_LENERR; 575 return ((buf[0]&0x3F)<<8)|buf[1]; 576 } else { 577 /* Read a 32 bit len */ 578 if (fread(&len,4,1,fp) == 0) return REDIS_RDB_LENERR; 579 return ntohl(len); 580 } 581 } 582 583 /* Load an integer-encoded object from file 'fp', with the specified 584 * encoding type 'enctype'. If encode is true the function may return 585 * an integer-encoded object as reply, otherwise the returned object 586 * will always be encoded as a raw string. */ 587 robj *rdbLoadIntegerObject(FILE *fp, int enctype, int encode) { 588 unsigned char enc[4]; 589 long long val; 590 591 if (enctype == REDIS_RDB_ENC_INT8) { 592 if (fread(enc,1,1,fp) == 0) return NULL; 593 val = (signed char)enc[0]; 594 } else if (enctype == REDIS_RDB_ENC_INT16) { 595 uint16_t v; 596 if (fread(enc,2,1,fp) == 0) return NULL; 597 v = enc[0]|(enc[1]<<8); 598 val = (int16_t)v; 599 } else if (enctype == REDIS_RDB_ENC_INT32) { 600 uint32_t v; 601 if (fread(enc,4,1,fp) == 0) return NULL; 602 v = enc[0]|(enc[1]<<8)|(enc[2]<<16)|(enc[3]<<24); 603 val = (int32_t)v; 604 } else { 605 val = 0; /* anti-warning */ 606 redisPanic("Unknown RDB integer encoding type"); 607 } 608 if (encode) 609 return createStringObjectFromLongLong(val); 610 else 611 return createObject(REDIS_STRING,sdsfromlonglong(val)); 612 } 613 614 robj *rdbLoadLzfStringObject(FILE*fp) { 615 unsigned int len, clen; 616 unsigned char *c = NULL; 617 sds val = NULL; 618 619 if ((clen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 620 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 621 if ((c = zmalloc(clen)) == NULL) goto err; 622 if ((val = sdsnewlen(NULL,len)) == NULL) goto err; 623 if (fread(c,clen,1,fp) == 0) goto err; 624 if (lzf_decompress(c,clen,val,len) == 0) goto err; 625 zfree(c); 626 return createObject(REDIS_STRING,val); 627 err: 628 zfree(c); 629 sdsfree(val); 630 return NULL; 631 } 632 633 robj *rdbGenericLoadStringObject(FILE*fp, int encode) { 634 int isencoded; 635 uint32_t len; 636 sds val; 637 638 len = rdbLoadLen(fp,&isencoded); 639 if (isencoded) { 640 switch(len) { 641 case REDIS_RDB_ENC_INT8: 642 case REDIS_RDB_ENC_INT16: 643 case REDIS_RDB_ENC_INT32: 644 return rdbLoadIntegerObject(fp,len,encode); 645 case REDIS_RDB_ENC_LZF: 646 return rdbLoadLzfStringObject(fp); 647 default: 648 redisPanic("Unknown RDB encoding type"); 649 } 650 } 651 652 if (len == REDIS_RDB_LENERR) return NULL; 653 val = sdsnewlen(NULL,len); 654 if (len && fread(val,len,1,fp) == 0) { 655 sdsfree(val); 656 return NULL; 657 } 658 return createObject(REDIS_STRING,val); 659 } 660 661 robj *rdbLoadStringObject(FILE *fp) { 662 return rdbGenericLoadStringObject(fp,0); 663 } 664 665 robj *rdbLoadEncodedStringObject(FILE *fp) { 666 return rdbGenericLoadStringObject(fp,1); 667 } 668 669 /* For information about double serialization check rdbSaveDoubleValue() */ 670 int rdbLoadDoubleValue(FILE *fp, double *val) { 671 char buf[128]; 672 unsigned char len; 673 674 if (fread(&len,1,1,fp) == 0) return -1; 675 switch(len) { 676 case 255: *val = R_NegInf; return 0; 677 case 254: *val = R_PosInf; return 0; 678 case 253: *val = R_Nan; return 0; 679 default: 680 if (fread(buf,len,1,fp) == 0) return -1; 681 buf[len] = '\0'; 682 sscanf(buf, "%lg", val); 683 return 0; 684 } 685 } 686 687 /* Load a Redis object of the specified type from the specified file. 688 * On success a newly allocated object is returned, otherwise NULL. */ 689 robj *rdbLoadObject(int type, FILE *fp) { 690 robj *o, *ele, *dec; 691 size_t len; 692 unsigned int i; 693 694 redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",type,ftell(fp)); 695 if (type == REDIS_STRING) { 696 /* Read string value */ 697 if ((o = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 698 o = tryObjectEncoding(o); 699 } else if (type == REDIS_LIST) { 700 /* Read list value */ 701 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 702 703 /* Use a real list when there are too many entries */ 704 if (len > server.list_max_ziplist_entries) { 705 o = createListObject(); 706 } else { 707 o = createZiplistObject(); 708 } 709 710 /* Load every single element of the list */ 711 while(len--) { 712 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 713 714 /* If we are using a ziplist and the value is too big, convert 715 * the object to a real list. */ 716 if (o->encoding == REDIS_ENCODING_ZIPLIST && 717 ele->encoding == REDIS_ENCODING_RAW && 718 sdslen(ele->ptr) > server.list_max_ziplist_value) 719 listTypeConvert(o,REDIS_ENCODING_LINKEDLIST); 720 721 if (o->encoding == REDIS_ENCODING_ZIPLIST) { 722 dec = getDecodedObject(ele); 723 o->ptr = ziplistPush(o->ptr,dec->ptr,sdslen(dec->ptr),REDIS_TAIL); 724 decrRefCount(dec); 725 decrRefCount(ele); 726 } else { 727 ele = tryObjectEncoding(ele); 728 listAddNodeTail(o->ptr,ele); 729 } 730 } 731 } else if (type == REDIS_SET) { 732 /* Read list/set value */ 733 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 734 735 /* Use a regular set when there are too many entries. */ 736 if (len > server.set_max_intset_entries) { 737 o = createSetObject(); 738 /* It's faster to expand the dict to the right size asap in order 739 * to avoid rehashing */ 740 if (len > DICT_HT_INITIAL_SIZE) 741 dictExpand(o->ptr,len); 742 } else { 743 o = createIntsetObject(); 744 } 745 746 /* Load every single element of the list/set */ 747 for (i = 0; i < len; i++) { 748 long long llval; 749 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 750 ele = tryObjectEncoding(ele); 751 752 if (o->encoding == REDIS_ENCODING_INTSET) { 753 /* Fetch integer value from element */ 754 if (isObjectRepresentableAsLongLong(ele,&llval) == REDIS_OK) { 755 o->ptr = intsetAdd(o->ptr,llval,NULL); 756 } else { 757 setTypeConvert(o,REDIS_ENCODING_HT); 758 dictExpand(o->ptr,len); 759 } 760 } 761 762 /* This will also be called when the set was just converted 763 * to regular hashtable encoded set */ 764 if (o->encoding == REDIS_ENCODING_HT) { 765 dictAdd((dict*)o->ptr,ele,NULL); 766 } else { 767 decrRefCount(ele); 768 } 769 } 770 } else if (type == REDIS_ZSET) { 771 /* Read list/set value */ 772 size_t zsetlen; 773 zset *zs; 774 775 if ((zsetlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 776 o = createZsetObject(); 777 zs = o->ptr; 778 /* Load every single element of the list/set */ 779 while(zsetlen--) { 780 robj *ele; 781 double score; 782 zskiplistNode *znode; 783 784 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 785 ele = tryObjectEncoding(ele); 786 if (rdbLoadDoubleValue(fp,&score) == -1) return NULL; 787 znode = zslInsert(zs->zsl,score,ele); 788 dictAdd(zs->dict,ele,&znode->score); 789 incrRefCount(ele); /* added to skiplist */ 790 } 791 } else if (type == REDIS_HASH) { 792 size_t hashlen; 793 794 if ((hashlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; 795 o = createHashObject(); 796 /* Too many entries? Use an hash table. */ 797 if (hashlen > server.hash_max_zipmap_entries) 798 convertToRealHash(o); 799 /* Load every key/value, then set it into the zipmap or hash 800 * table, as needed. */ 801 while(hashlen--) { 802 robj *key, *val; 803 804 if ((key = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 805 if ((val = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; 806 /* If we are using a zipmap and there are too big values 807 * the object is converted to real hash table encoding. */ 808 if (o->encoding != REDIS_ENCODING_HT && 809 ((key->encoding == REDIS_ENCODING_RAW && 810 sdslen(key->ptr) > server.hash_max_zipmap_value) || 811 (val->encoding == REDIS_ENCODING_RAW && 812 sdslen(val->ptr) > server.hash_max_zipmap_value))) 813 { 814 convertToRealHash(o); 815 } 816 817 if (o->encoding == REDIS_ENCODING_ZIPMAP) { 818 unsigned char *zm = o->ptr; 819 robj *deckey, *decval; 820 821 /* We need raw string objects to add them to the zipmap */ 822 deckey = getDecodedObject(key); 823 decval = getDecodedObject(val); 824 zm = zipmapSet(zm,deckey->ptr,sdslen(deckey->ptr), 825 decval->ptr,sdslen(decval->ptr),NULL); 826 o->ptr = zm; 827 decrRefCount(deckey); 828 decrRefCount(decval); 829 decrRefCount(key); 830 decrRefCount(val); 831 } else { 832 key = tryObjectEncoding(key); 833 val = tryObjectEncoding(val); 834 dictAdd((dict*)o->ptr,key,val); 835 } 836 } 837 } else { 838 redisPanic("Unknown object type"); 839 } 840 return o; 841 } 842 843 /* Mark that we are loading in the global state and setup the fields 844 * needed to provide loading stats. */ 845 void startLoading(FILE *fp) { 846 struct stat sb; 847 848 /* Load the DB */ 849 server.loading = 1; 850 server.loading_start_time = time(NULL); 851 if (fstat(fileno(fp), &sb) == -1) { 852 server.loading_total_bytes = 1; /* just to avoid division by zero */ 853 } else { 854 server.loading_total_bytes = sb.st_size; 855 } 856 } 857 858 /* Refresh the loading progress info */ 859 void loadingProgress(off_t pos) { 860 server.loading_loaded_bytes = pos; 861 } 862 863 /* Loading finished */ 864 void stopLoading(void) { 865 server.loading = 0; 866 } 867 868 int rdbLoad(char *filename) { 869 FILE *fp; 870 uint32_t dbid; 871 int type, retval, rdbver; 872 redisDb *db = server.db+0; 873 char buf[1024]; 874 time_t expiretime, now = time(NULL); 875 long loops = 0; 876 877 fp = fopen(filename,"r"); 878 if (!fp) return REDIS_ERR; 879 if (fread(buf,9,1,fp) == 0) goto eoferr; 880 buf[9] = '\0'; 881 if (memcmp(buf,"REDIS",5) != 0) { 882 fclose(fp); 883 redisLog(REDIS_WARNING,"Wrong signature trying to load DB from file"); 884 return REDIS_ERR; 885 } 886 rdbver = atoi(buf+5); 887 if (rdbver != 1) { 888 fclose(fp); 889 redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver); 890 return REDIS_ERR; 891 } 892 893 startLoading(fp); 894 while(1) { 895 robj *key, *val; 896 expiretime = -1; 897 898 /* Serve the clients from time to time */ 899 if (!(loops++ % 1000)) { 900 loadingProgress(ftello(fp)); 901 aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); 902 } 903 904 /* Read type. */ 905 if ((type = rdbLoadType(fp)) == -1) goto eoferr; 906 if (type == REDIS_EXPIRETIME) { 907 if ((expiretime = rdbLoadTime(fp)) == -1) goto eoferr; 908 /* We read the time so we need to read the object type again */ 909 if ((type = rdbLoadType(fp)) == -1) goto eoferr; 910 } 911 if (type == REDIS_EOF) break; 912 /* Handle SELECT DB opcode as a special case */ 913 if (type == REDIS_SELECTDB) { 914 if ((dbid = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) 915 goto eoferr; 916 if (dbid >= (unsigned)server.dbnum) { 917 redisLog(REDIS_WARNING,"FATAL: Data file was created with a Redis server configured to handle more than %d databases. Exiting\n", server.dbnum); 918 exit(1); 919 } 920 db = server.db+dbid; 921 continue; 922 } 923 /* Read key */ 924 if ((key = rdbLoadStringObject(fp)) == NULL) goto eoferr; 925 /* Read value */ 926 if ((val = rdbLoadObject(type,fp)) == NULL) goto eoferr; 927 /* Check if the key already expired */ 928 if (expiretime != -1 && expiretime < now) { 929 decrRefCount(key); 930 decrRefCount(val); 931 continue; 932 } 933 /* Add the new object in the hash table */ 934 retval = dbAdd(db,key,val); 935 if (retval == REDIS_ERR) { 936 redisLog(REDIS_WARNING,"Loading DB, duplicated key (%s) found! Unrecoverable error, exiting now.", key->ptr); 937 exit(1); 938 } 939 /* Set the expire time if needed */ 940 if (expiretime != -1) setExpire(db,key,expiretime); 941 942 decrRefCount(key); 943 } 944 fclose(fp); 945 stopLoading(); 946 return REDIS_OK; 947 948 eoferr: /* unexpected end of file is handled here with a fatal exit */ 949 redisLog(REDIS_WARNING,"Short read or OOM loading DB. Unrecoverable error, aborting now."); 950 exit(1); 951 return REDIS_ERR; /* Just to avoid warning */ 952 } 953 954 /* A background saving child (BGSAVE) terminated its work. Handle this. */ 955 void backgroundSaveDoneHandler(int exitcode, int bysignal) { 956 if (!bysignal && exitcode == 0) { 957 redisLog(REDIS_NOTICE, 958 "Background saving terminated with success"); 959 server.dirty = server.dirty - server.dirty_before_bgsave; 960 server.lastsave = time(NULL); 961 } else if (!bysignal && exitcode != 0) { 962 redisLog(REDIS_WARNING, "Background saving error"); 963 } else { 964 redisLog(REDIS_WARNING, 965 "Background saving terminated by signal %d", bysignal); 966 rdbRemoveTempFile(server.bgsavechildpid); 967 } 968 server.bgsavechildpid = -1; 969 server.bgsavethread = (pthread_t) -1; 970 server.bgsavethread_state = REDIS_BGSAVE_THREAD_UNACTIVE; 971 /* Possibly there are slaves waiting for a BGSAVE in order to be served 972 * (the first stage of SYNC is a bulk transfer of dump.rdb) */ 973 updateSlavesWaitingBgsave(exitcode == 0 ? REDIS_OK : REDIS_ERR); 974 } 975 976 void saveCommand(redisClient *c) { 977 if (server.bgsavechildpid != -1 || server.bgsavethread != (pthread_t)-1) { 978 addReplyError(c,"Background save already in progress"); 979 return; 980 } 981 if (rdbSave(server.dbfilename) == REDIS_OK) { 982 addReply(c,shared.ok); 983 } else { 984 addReply(c,shared.err); 985 } 986 } 987 988 void bgsaveCommand(redisClient *c) { 989 if (server.bgsavechildpid != -1 || server.bgsavethread != (pthread_t)-1) { 990 addReplyError(c,"Background save already in progress"); 991 return; 992 } 993 if (rdbSaveBackground(server.dbfilename) == REDIS_OK) { 994 addReplyStatus(c,"Background saving started"); 995 } else { 996 addReply(c,shared.err); 997 } 998 } 999