1 /*
2 * cache-memcache.c: memcached caching for Subversion
3 *
4 * ====================================================================
5 * Licensed to the Apache Software Foundation (ASF) under one
6 * or more contributor license agreements. See the NOTICE file
7 * distributed with this work for additional information
8 * regarding copyright ownership. The ASF licenses this file
9 * to you under the Apache License, Version 2.0 (the
10 * "License"); you may not use this file except in compliance
11 * with the License. You may obtain a copy of the License at
12 *
13 * http://www.apache.org/licenses/LICENSE-2.0
14 *
15 * Unless required by applicable law or agreed to in writing,
16 * software distributed under the License is distributed on an
17 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
18 * KIND, either express or implied. See the License for the
19 * specific language governing permissions and limitations
20 * under the License.
21 * ====================================================================
22 */
23
24 #include <apr_md5.h>
25
26 #include "svn_pools.h"
27 #include "svn_base64.h"
28 #include "svn_path.h"
29
30 #include "svn_private_config.h"
31 #include "private/svn_cache.h"
32 #include "private/svn_dep_compat.h"
33
34 #include "cache.h"
35
36 #ifdef SVN_HAVE_MEMCACHE
37
38 #include <apr_memcache.h>
39
40 /* A note on thread safety:
41
42 The apr_memcache_t object does its own mutex handling, and nothing
43 else in memcache_t is ever modified, so this implementation should
44 be fully thread-safe.
45 */
46
47 /* The (internal) cache object. */
48 typedef struct memcache_t {
49 /* The memcached server set we're using. */
50 apr_memcache_t *memcache;
51
52 /* A prefix used to differentiate our data from any other data in
53 * the memcached (URI-encoded). */
54 const char *prefix;
55
56 /* The size of the key: either a fixed number of bytes or
57 * APR_HASH_KEY_STRING. */
58 apr_ssize_t klen;
59
60
61 /* Used to marshal values in and out of the cache. */
62 svn_cache__serialize_func_t serialize_func;
63 svn_cache__deserialize_func_t deserialize_func;
64 } memcache_t;
65
66 /* The wrapper around apr_memcache_t. */
67 struct svn_memcache_t {
68 apr_memcache_t *c;
69 };
70
71
72 /* The memcached protocol says the maximum key length is 250. Let's
73 just say 249, to be safe. */
74 #define MAX_MEMCACHED_KEY_LEN 249
75 #define MEMCACHED_KEY_UNHASHED_LEN (MAX_MEMCACHED_KEY_LEN - \
76 2 * APR_MD5_DIGESTSIZE)
77
78
79 /* Set *MC_KEY to a memcache key for the given key KEY for CACHE, allocated
80 in POOL. */
81 static svn_error_t *
build_key(const char ** mc_key,memcache_t * cache,const void * raw_key,apr_pool_t * pool)82 build_key(const char **mc_key,
83 memcache_t *cache,
84 const void *raw_key,
85 apr_pool_t *pool)
86 {
87 const char *encoded_suffix;
88 const char *long_key;
89 apr_size_t long_key_len;
90
91 if (cache->klen == APR_HASH_KEY_STRING)
92 encoded_suffix = svn_path_uri_encode(raw_key, pool);
93 else
94 {
95 const svn_string_t *raw = svn_string_ncreate(raw_key, cache->klen, pool);
96 const svn_string_t *encoded = svn_base64_encode_string2(raw, FALSE,
97 pool);
98 encoded_suffix = encoded->data;
99 }
100
101 long_key = apr_pstrcat(pool, "SVN:", cache->prefix, ":", encoded_suffix,
102 SVN_VA_NULL);
103 long_key_len = strlen(long_key);
104
105 /* We don't want to have a key that's too big. If it was going to
106 be too big, we MD5 the entire string, then replace the last bit
107 with the checksum. Note that APR_MD5_DIGESTSIZE is for the pure
108 binary digest; we have to double that when we convert to hex.
109
110 Every key we use will either be at most
111 MEMCACHED_KEY_UNHASHED_LEN bytes long, or be exactly
112 MAX_MEMCACHED_KEY_LEN bytes long. */
113 if (long_key_len > MEMCACHED_KEY_UNHASHED_LEN)
114 {
115 svn_checksum_t *checksum;
116 SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, long_key, long_key_len,
117 pool));
118
119 long_key = apr_pstrcat(pool,
120 apr_pstrmemdup(pool, long_key,
121 MEMCACHED_KEY_UNHASHED_LEN),
122 svn_checksum_to_cstring_display(checksum, pool),
123 SVN_VA_NULL);
124 }
125
126 *mc_key = long_key;
127 return SVN_NO_ERROR;
128 }
129
130 /* Core functionality of our getter functions: fetch DATA from the memcached
131 * given by CACHE_VOID and identified by KEY. Indicate success in FOUND and
132 * use a tempoary sub-pool of POOL for allocations.
133 */
134 static svn_error_t *
memcache_internal_get(char ** data,apr_size_t * size,svn_boolean_t * found,void * cache_void,const void * key,apr_pool_t * pool)135 memcache_internal_get(char **data,
136 apr_size_t *size,
137 svn_boolean_t *found,
138 void *cache_void,
139 const void *key,
140 apr_pool_t *pool)
141 {
142 memcache_t *cache = cache_void;
143 apr_status_t apr_err;
144 const char *mc_key;
145 apr_pool_t *subpool;
146
147 if (key == NULL)
148 {
149 *found = FALSE;
150 return SVN_NO_ERROR;
151 }
152
153 subpool = svn_pool_create(pool);
154 SVN_ERR(build_key(&mc_key, cache, key, subpool));
155
156 apr_err = apr_memcache_getp(cache->memcache,
157 pool,
158 mc_key,
159 data,
160 size,
161 NULL /* ignore flags */);
162 if (apr_err == APR_NOTFOUND)
163 {
164 *found = FALSE;
165 svn_pool_destroy(subpool);
166 return SVN_NO_ERROR;
167 }
168 else if (apr_err != APR_SUCCESS || !*data)
169 return svn_error_wrap_apr(apr_err,
170 _("Unknown memcached error while reading"));
171
172 *found = TRUE;
173
174 svn_pool_destroy(subpool);
175 return SVN_NO_ERROR;
176 }
177
178
179 static svn_error_t *
memcache_get(void ** value_p,svn_boolean_t * found,void * cache_void,const void * key,apr_pool_t * result_pool)180 memcache_get(void **value_p,
181 svn_boolean_t *found,
182 void *cache_void,
183 const void *key,
184 apr_pool_t *result_pool)
185 {
186 memcache_t *cache = cache_void;
187 char *data;
188 apr_size_t data_len;
189 SVN_ERR(memcache_internal_get(&data,
190 &data_len,
191 found,
192 cache_void,
193 key,
194 result_pool));
195
196 /* If we found it, de-serialize it. */
197 if (*found)
198 {
199 if (cache->deserialize_func)
200 {
201 SVN_ERR((cache->deserialize_func)(value_p, data, data_len,
202 result_pool));
203 }
204 else
205 {
206 svn_stringbuf_t *value = svn_stringbuf_create_empty(result_pool);
207 value->data = data;
208 value->blocksize = data_len;
209 value->len = data_len - 1; /* account for trailing NUL */
210 *value_p = value;
211 }
212 }
213
214 return SVN_NO_ERROR;
215 }
216
217 /* Implement vtable.has_key in terms of the getter.
218 */
219 static svn_error_t *
memcache_has_key(svn_boolean_t * found,void * cache_void,const void * key,apr_pool_t * scratch_pool)220 memcache_has_key(svn_boolean_t *found,
221 void *cache_void,
222 const void *key,
223 apr_pool_t *scratch_pool)
224 {
225 char *data;
226 apr_size_t data_len;
227 SVN_ERR(memcache_internal_get(&data,
228 &data_len,
229 found,
230 cache_void,
231 key,
232 scratch_pool));
233
234 return SVN_NO_ERROR;
235 }
236
237 /* Core functionality of our setter functions: store LENGH bytes of DATA
238 * to be identified by KEY in the memcached given by CACHE_VOID. Use POOL
239 * for temporary allocations.
240 */
241 static svn_error_t *
memcache_internal_set(void * cache_void,const void * key,const char * data,apr_size_t len,apr_pool_t * scratch_pool)242 memcache_internal_set(void *cache_void,
243 const void *key,
244 const char *data,
245 apr_size_t len,
246 apr_pool_t *scratch_pool)
247 {
248 memcache_t *cache = cache_void;
249 const char *mc_key;
250 apr_status_t apr_err;
251
252 SVN_ERR(build_key(&mc_key, cache, key, scratch_pool));
253 apr_err = apr_memcache_set(cache->memcache, mc_key, (char *)data, len, 0, 0);
254
255 /* ### Maybe write failures should be ignored (but logged)? */
256 if (apr_err != APR_SUCCESS)
257 return svn_error_wrap_apr(apr_err,
258 _("Unknown memcached error while writing"));
259
260 return SVN_NO_ERROR;
261 }
262
263
264 static svn_error_t *
memcache_set(void * cache_void,const void * key,void * value,apr_pool_t * scratch_pool)265 memcache_set(void *cache_void,
266 const void *key,
267 void *value,
268 apr_pool_t *scratch_pool)
269 {
270 memcache_t *cache = cache_void;
271 apr_pool_t *subpool = svn_pool_create(scratch_pool);
272 void *data;
273 apr_size_t data_len;
274 svn_error_t *err;
275
276 if (key == NULL)
277 return SVN_NO_ERROR;
278
279 if (cache->serialize_func)
280 {
281 SVN_ERR((cache->serialize_func)(&data, &data_len, value, subpool));
282 }
283 else
284 {
285 svn_stringbuf_t *value_str = value;
286 data = value_str->data;
287 data_len = value_str->len + 1; /* copy trailing NUL */
288 }
289
290 err = memcache_internal_set(cache_void, key, data, data_len, subpool);
291
292 svn_pool_destroy(subpool);
293 return err;
294 }
295
296 static svn_error_t *
memcache_get_partial(void ** value_p,svn_boolean_t * found,void * cache_void,const void * key,svn_cache__partial_getter_func_t func,void * baton,apr_pool_t * result_pool)297 memcache_get_partial(void **value_p,
298 svn_boolean_t *found,
299 void *cache_void,
300 const void *key,
301 svn_cache__partial_getter_func_t func,
302 void *baton,
303 apr_pool_t *result_pool)
304 {
305 svn_error_t *err = SVN_NO_ERROR;
306
307 char *data;
308 apr_size_t size;
309 SVN_ERR(memcache_internal_get(&data,
310 &size,
311 found,
312 cache_void,
313 key,
314 result_pool));
315
316 /* If we found it, de-serialize it. */
317 return *found
318 ? func(value_p, data, size, baton, result_pool)
319 : err;
320 }
321
322
323 static svn_error_t *
memcache_set_partial(void * cache_void,const void * key,svn_cache__partial_setter_func_t func,void * baton,apr_pool_t * scratch_pool)324 memcache_set_partial(void *cache_void,
325 const void *key,
326 svn_cache__partial_setter_func_t func,
327 void *baton,
328 apr_pool_t *scratch_pool)
329 {
330 svn_error_t *err = SVN_NO_ERROR;
331
332 void *data;
333 apr_size_t size;
334 svn_boolean_t found = FALSE;
335
336 apr_pool_t *subpool = svn_pool_create(scratch_pool);
337 SVN_ERR(memcache_internal_get((char **)&data,
338 &size,
339 &found,
340 cache_void,
341 key,
342 subpool));
343
344 /* If we found it, modify it and write it back to cache */
345 if (found)
346 {
347 SVN_ERR(func(&data, &size, baton, subpool));
348 err = memcache_internal_set(cache_void, key, data, size, subpool);
349 }
350
351 svn_pool_destroy(subpool);
352 return err;
353 }
354
355
356 static svn_error_t *
memcache_iter(svn_boolean_t * completed,void * cache_void,svn_iter_apr_hash_cb_t user_cb,void * user_baton,apr_pool_t * scratch_pool)357 memcache_iter(svn_boolean_t *completed,
358 void *cache_void,
359 svn_iter_apr_hash_cb_t user_cb,
360 void *user_baton,
361 apr_pool_t *scratch_pool)
362 {
363 return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
364 _("Can't iterate a memcached cache"));
365 }
366
367 static svn_boolean_t
memcache_is_cachable(void * unused,apr_size_t size)368 memcache_is_cachable(void *unused, apr_size_t size)
369 {
370 SVN_UNUSED(unused);
371
372 /* The memcached cutoff seems to be a bit (header length?) under a megabyte.
373 * We round down a little to be safe.
374 */
375 return size < 1000000;
376 }
377
378 static svn_error_t *
memcache_get_info(void * cache_void,svn_cache__info_t * info,svn_boolean_t reset,apr_pool_t * result_pool)379 memcache_get_info(void *cache_void,
380 svn_cache__info_t *info,
381 svn_boolean_t reset,
382 apr_pool_t *result_pool)
383 {
384 memcache_t *cache = cache_void;
385
386 info->id = apr_pstrdup(result_pool, cache->prefix);
387
388 /* we don't have any memory allocation info */
389
390 return SVN_NO_ERROR;
391 }
392
393 static svn_cache__vtable_t memcache_vtable = {
394 memcache_get,
395 memcache_has_key,
396 memcache_set,
397 memcache_iter,
398 memcache_is_cachable,
399 memcache_get_partial,
400 memcache_set_partial,
401 memcache_get_info
402 };
403
404 svn_error_t *
svn_cache__create_memcache(svn_cache__t ** cache_p,svn_memcache_t * memcache,svn_cache__serialize_func_t serialize_func,svn_cache__deserialize_func_t deserialize_func,apr_ssize_t klen,const char * prefix,apr_pool_t * pool)405 svn_cache__create_memcache(svn_cache__t **cache_p,
406 svn_memcache_t *memcache,
407 svn_cache__serialize_func_t serialize_func,
408 svn_cache__deserialize_func_t deserialize_func,
409 apr_ssize_t klen,
410 const char *prefix,
411 apr_pool_t *pool)
412 {
413 svn_cache__t *wrapper = apr_pcalloc(pool, sizeof(*wrapper));
414 memcache_t *cache = apr_pcalloc(pool, sizeof(*cache));
415
416 cache->serialize_func = serialize_func;
417 cache->deserialize_func = deserialize_func;
418 cache->klen = klen;
419 cache->prefix = svn_path_uri_encode(prefix, pool);
420 cache->memcache = memcache->c;
421
422 wrapper->vtable = &memcache_vtable;
423 wrapper->cache_internal = cache;
424 wrapper->error_handler = 0;
425 wrapper->error_baton = 0;
426 wrapper->pretend_empty = !!getenv("SVN_X_DOES_NOT_MARK_THE_SPOT");
427
428 *cache_p = wrapper;
429 return SVN_NO_ERROR;
430 }
431
432
433 /*** Creating apr_memcache_t from svn_config_t. ***/
434
435 /* Baton for add_memcache_server. */
436 struct ams_baton {
437 apr_memcache_t *memcache;
438 apr_pool_t *memcache_pool;
439 svn_error_t *err;
440 };
441
442 /* Implements svn_config_enumerator2_t. */
443 static svn_boolean_t
add_memcache_server(const char * name,const char * value,void * baton,apr_pool_t * pool)444 add_memcache_server(const char *name,
445 const char *value,
446 void *baton,
447 apr_pool_t *pool)
448 {
449 struct ams_baton *b = baton;
450 char *host, *scope;
451 apr_port_t port;
452 apr_status_t apr_err;
453 apr_memcache_server_t *server;
454
455 apr_err = apr_parse_addr_port(&host, &scope, &port,
456 value, pool);
457 if (apr_err != APR_SUCCESS)
458 {
459 b->err = svn_error_wrap_apr(apr_err,
460 _("Error parsing memcache server '%s'"),
461 name);
462 return FALSE;
463 }
464
465 if (scope)
466 {
467 b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
468 _("Scope not allowed in memcache server "
469 "'%s'"),
470 name);
471 return FALSE;
472 }
473 if (!host || !port)
474 {
475 b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
476 _("Must specify host and port for memcache "
477 "server '%s'"),
478 name);
479 return FALSE;
480 }
481
482 /* Note: the four numbers here are only relevant when an
483 apr_memcache_t is being shared by multiple threads. */
484 apr_err = apr_memcache_server_create(b->memcache_pool,
485 host,
486 port,
487 0, /* min connections */
488 5, /* soft max connections */
489 10, /* hard max connections */
490 /* time to live (in microseconds) */
491 apr_time_from_sec(50),
492 &server);
493 if (apr_err != APR_SUCCESS)
494 {
495 b->err = svn_error_wrap_apr(apr_err,
496 _("Unknown error creating memcache server"));
497 return FALSE;
498 }
499
500 apr_err = apr_memcache_add_server(b->memcache, server);
501 if (apr_err != APR_SUCCESS)
502 {
503 b->err = svn_error_wrap_apr(apr_err,
504 _("Unknown error adding server to memcache"));
505 return FALSE;
506 }
507
508 return TRUE;
509 }
510
511 #else /* ! SVN_HAVE_MEMCACHE */
512
513 /* Stubs for no apr memcache library. */
514
515 struct svn_memcache_t {
516 void *unused; /* Let's not have a size-zero struct. */
517 };
518
519 svn_error_t *
svn_cache__create_memcache(svn_cache__t ** cache_p,svn_memcache_t * memcache,svn_cache__serialize_func_t serialize_func,svn_cache__deserialize_func_t deserialize_func,apr_ssize_t klen,const char * prefix,apr_pool_t * pool)520 svn_cache__create_memcache(svn_cache__t **cache_p,
521 svn_memcache_t *memcache,
522 svn_cache__serialize_func_t serialize_func,
523 svn_cache__deserialize_func_t deserialize_func,
524 apr_ssize_t klen,
525 const char *prefix,
526 apr_pool_t *pool)
527 {
528 return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
529 }
530
531 #endif /* SVN_HAVE_MEMCACHE */
532
533 /* Implements svn_config_enumerator2_t. Just used for the
534 entry-counting return value of svn_config_enumerate2. */
535 static svn_boolean_t
nop_enumerator(const char * name,const char * value,void * baton,apr_pool_t * pool)536 nop_enumerator(const char *name,
537 const char *value,
538 void *baton,
539 apr_pool_t *pool)
540 {
541 return TRUE;
542 }
543
544 svn_error_t *
svn_cache__make_memcache_from_config(svn_memcache_t ** memcache_p,svn_config_t * config,apr_pool_t * result_pool,apr_pool_t * scratch_pool)545 svn_cache__make_memcache_from_config(svn_memcache_t **memcache_p,
546 svn_config_t *config,
547 apr_pool_t *result_pool,
548 apr_pool_t *scratch_pool)
549 {
550 int server_count =
551 svn_config_enumerate2(config,
552 SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
553 nop_enumerator, NULL, scratch_pool);
554
555 if (server_count == 0)
556 {
557 *memcache_p = NULL;
558 return SVN_NO_ERROR;
559 }
560
561 if (server_count > APR_INT16_MAX)
562 return svn_error_create(SVN_ERR_TOO_MANY_MEMCACHED_SERVERS, NULL, NULL);
563
564 #ifdef SVN_HAVE_MEMCACHE
565 {
566 struct ams_baton b;
567 svn_memcache_t *memcache = apr_pcalloc(result_pool, sizeof(*memcache));
568 apr_status_t apr_err = apr_memcache_create(result_pool,
569 (apr_uint16_t)server_count,
570 0, /* flags */
571 &(memcache->c));
572 if (apr_err != APR_SUCCESS)
573 return svn_error_wrap_apr(apr_err,
574 _("Unknown error creating apr_memcache_t"));
575
576 b.memcache = memcache->c;
577 b.memcache_pool = result_pool;
578 b.err = SVN_NO_ERROR;
579 svn_config_enumerate2(config,
580 SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
581 add_memcache_server, &b,
582 scratch_pool);
583
584 if (b.err)
585 return b.err;
586
587 *memcache_p = memcache;
588
589 return SVN_NO_ERROR;
590 }
591 #else /* ! SVN_HAVE_MEMCACHE */
592 {
593 return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
594 }
595 #endif /* SVN_HAVE_MEMCACHE */
596 }
597