19c7a60dfSdrh /*
29c7a60dfSdrh ** 2007 October 14
39c7a60dfSdrh **
49c7a60dfSdrh ** The author disclaims copyright to this source code. In place of
59c7a60dfSdrh ** a legal notice, here is a blessing:
69c7a60dfSdrh **
79c7a60dfSdrh ** May you do good and not evil.
89c7a60dfSdrh ** May you find forgiveness for yourself and forgive others.
99c7a60dfSdrh ** May you share freely, never taking more than you give.
109c7a60dfSdrh **
119c7a60dfSdrh *************************************************************************
129c7a60dfSdrh ** This file contains the C functions that implement a memory
139c7a60dfSdrh ** allocation subsystem for use by SQLite.
149c7a60dfSdrh **
159c7a60dfSdrh ** This version of the memory allocation subsystem omits all
1632155ef0Sdanielk1977 ** use of malloc(). The SQLite user supplies a block of memory
1732155ef0Sdanielk1977 ** before calling sqlite3_initialize() from which allocations
1832155ef0Sdanielk1977 ** are made and returned by the xMalloc() and xRealloc()
1932155ef0Sdanielk1977 ** implementations. Once sqlite3_initialize() has been called,
2032155ef0Sdanielk1977 ** the amount of memory available to SQLite is fixed and cannot
2132155ef0Sdanielk1977 ** be changed.
229c7a60dfSdrh **
2332155ef0Sdanielk1977 ** This version of the memory allocation subsystem is included
2432155ef0Sdanielk1977 ** in the build only if SQLITE_ENABLE_MEMSYS3 is defined.
259c7a60dfSdrh */
260d18020bSdrh #include "sqliteInt.h"
279c7a60dfSdrh
289c7a60dfSdrh /*
2957e5ea93Sdanielk1977 ** This version of the memory allocator is only built into the library
3032155ef0Sdanielk1977 ** SQLITE_ENABLE_MEMSYS3 is defined. Defining this symbol does not
3157e5ea93Sdanielk1977 ** mean that the library will use a memory-pool by default, just that
3257e5ea93Sdanielk1977 ** it is available. The mempool allocator is activated by calling
3357e5ea93Sdanielk1977 ** sqlite3_config().
349c7a60dfSdrh */
3532155ef0Sdanielk1977 #ifdef SQLITE_ENABLE_MEMSYS3
36ace03d1bSdrh
379c7a60dfSdrh /*
389c7a60dfSdrh ** Maximum size (in Mem3Blocks) of a "small" chunk.
399c7a60dfSdrh */
409c7a60dfSdrh #define MX_SMALL 10
419c7a60dfSdrh
429c7a60dfSdrh
439c7a60dfSdrh /*
449c7a60dfSdrh ** Number of freelist hash slots
459c7a60dfSdrh */
469c7a60dfSdrh #define N_HASH 61
479c7a60dfSdrh
489c7a60dfSdrh /*
499c7a60dfSdrh ** A memory allocation (also called a "chunk") consists of two or
509c7a60dfSdrh ** more blocks where each block is 8 bytes. The first 8 bytes are
519c7a60dfSdrh ** a header that is not returned to the user.
529c7a60dfSdrh **
539c7a60dfSdrh ** A chunk is two or more blocks that is either checked out or
5471f971b2Sdrh ** free. The first block has format u.hdr. u.hdr.size4x is 4 times the
559c7a60dfSdrh ** size of the allocation in blocks if the allocation is free.
5671f971b2Sdrh ** The u.hdr.size4x&1 bit is true if the chunk is checked out and
5771f971b2Sdrh ** false if the chunk is on the freelist. The u.hdr.size4x&2 bit
5871f971b2Sdrh ** is true if the previous chunk is checked out and false if the
5971f971b2Sdrh ** previous chunk is free. The u.hdr.prevSize field is the size of
6071f971b2Sdrh ** the previous chunk in blocks if the previous chunk is on the
6171f971b2Sdrh ** freelist. If the previous chunk is checked out, then
6271f971b2Sdrh ** u.hdr.prevSize can be part of the data for that chunk and should
6371f971b2Sdrh ** not be read or written.
649c7a60dfSdrh **
6557e5ea93Sdanielk1977 ** We often identify a chunk by its index in mem3.aPool[]. When
669c7a60dfSdrh ** this is done, the chunk index refers to the second block of
679c7a60dfSdrh ** the chunk. In this way, the first chunk has an index of 1.
689c7a60dfSdrh ** A chunk index of 0 means "no such chunk" and is the equivalent
699c7a60dfSdrh ** of a NULL pointer.
709c7a60dfSdrh **
719c7a60dfSdrh ** The second block of free chunks is of the form u.list. The
729c7a60dfSdrh ** two fields form a double-linked list of chunks of related sizes.
7357e5ea93Sdanielk1977 ** Pointers to the head of the list are stored in mem3.aiSmall[]
7457e5ea93Sdanielk1977 ** for smaller chunks and mem3.aiHash[] for larger chunks.
759c7a60dfSdrh **
769c7a60dfSdrh ** The second block of a chunk is user data if the chunk is checked
7771f971b2Sdrh ** out. If a chunk is checked out, the user data may extend into
7871f971b2Sdrh ** the u.hdr.prevSize value of the following chunk.
799c7a60dfSdrh */
809c7a60dfSdrh typedef struct Mem3Block Mem3Block;
819c7a60dfSdrh struct Mem3Block {
829c7a60dfSdrh union {
839c7a60dfSdrh struct {
8471f971b2Sdrh u32 prevSize; /* Size of previous chunk in Mem3Block elements */
8571f971b2Sdrh u32 size4x; /* 4x the size of current chunk in Mem3Block elements */
869c7a60dfSdrh } hdr;
879c7a60dfSdrh struct {
8857e5ea93Sdanielk1977 u32 next; /* Index in mem3.aPool[] of next free chunk */
8957e5ea93Sdanielk1977 u32 prev; /* Index in mem3.aPool[] of previous free chunk */
909c7a60dfSdrh } list;
919c7a60dfSdrh } u;
929c7a60dfSdrh };
939c7a60dfSdrh
949c7a60dfSdrh /*
959c7a60dfSdrh ** All of the static variables used by this module are collected
9657e5ea93Sdanielk1977 ** into a single structure named "mem3". This is to keep the
979c7a60dfSdrh ** static variables organized and to reduce namespace pollution
989c7a60dfSdrh ** when this module is combined with other in the amalgamation.
999c7a60dfSdrh */
1005c8f8587Sdanielk1977 static SQLITE_WSD struct Mem3Global {
1019c7a60dfSdrh /*
10223bf0f41Sdanielk1977 ** Memory available for allocation. nPool is the size of the array
10323bf0f41Sdanielk1977 ** (in Mem3Blocks) pointed to by aPool less 2.
10423bf0f41Sdanielk1977 */
10523bf0f41Sdanielk1977 u32 nPool;
10623bf0f41Sdanielk1977 Mem3Block *aPool;
10723bf0f41Sdanielk1977
10823bf0f41Sdanielk1977 /*
109a4e5d58fSdrh ** True if we are evaluating an out-of-memory callback.
1109c7a60dfSdrh */
1119c7a60dfSdrh int alarmBusy;
1129c7a60dfSdrh
1139c7a60dfSdrh /*
1149c7a60dfSdrh ** Mutex to control access to the memory allocation subsystem.
1159c7a60dfSdrh */
1169c7a60dfSdrh sqlite3_mutex *mutex;
1179c7a60dfSdrh
1189c7a60dfSdrh /*
119a4e5d58fSdrh ** The minimum amount of free space that we have seen.
1209c7a60dfSdrh */
121*067b92baSdrh u32 mnKeyBlk;
1229c7a60dfSdrh
1239c7a60dfSdrh /*
124*067b92baSdrh ** iKeyBlk is the index of the key chunk. Most new allocations
125*067b92baSdrh ** occur off of this chunk. szKeyBlk is the size (in Mem3Blocks)
126*067b92baSdrh ** of the current key chunk. iKeyBlk is 0 if there is no key chunk.
127*067b92baSdrh ** The key chunk is not in either the aiHash[] or aiSmall[].
1289c7a60dfSdrh */
129*067b92baSdrh u32 iKeyBlk;
130*067b92baSdrh u32 szKeyBlk;
1319c7a60dfSdrh
1329c7a60dfSdrh /*
1339c7a60dfSdrh ** Array of lists of free blocks according to the block size
1349c7a60dfSdrh ** for smaller chunks, or a hash on the block size for larger
1359c7a60dfSdrh ** chunks.
1369c7a60dfSdrh */
13771f971b2Sdrh u32 aiSmall[MX_SMALL-1]; /* For sizes 2 through MX_SMALL, inclusive */
13871f971b2Sdrh u32 aiHash[N_HASH]; /* For sizes MX_SMALL+1 and larger */
13923bf0f41Sdanielk1977 } mem3 = { 97535575 };
1405c8f8587Sdanielk1977
1415c8f8587Sdanielk1977 #define mem3 GLOBAL(struct Mem3Global, mem3)
1429c7a60dfSdrh
1439c7a60dfSdrh /*
14457e5ea93Sdanielk1977 ** Unlink the chunk at mem3.aPool[i] from list it is currently
1459c7a60dfSdrh ** on. *pRoot is the list that i is a member of.
1469c7a60dfSdrh */
memsys3UnlinkFromList(u32 i,u32 * pRoot)14771f971b2Sdrh static void memsys3UnlinkFromList(u32 i, u32 *pRoot){
14857e5ea93Sdanielk1977 u32 next = mem3.aPool[i].u.list.next;
14957e5ea93Sdanielk1977 u32 prev = mem3.aPool[i].u.list.prev;
15057e5ea93Sdanielk1977 assert( sqlite3_mutex_held(mem3.mutex) );
1519c7a60dfSdrh if( prev==0 ){
1529c7a60dfSdrh *pRoot = next;
1539c7a60dfSdrh }else{
15457e5ea93Sdanielk1977 mem3.aPool[prev].u.list.next = next;
1559c7a60dfSdrh }
1569c7a60dfSdrh if( next ){
15757e5ea93Sdanielk1977 mem3.aPool[next].u.list.prev = prev;
1589c7a60dfSdrh }
15957e5ea93Sdanielk1977 mem3.aPool[i].u.list.next = 0;
16057e5ea93Sdanielk1977 mem3.aPool[i].u.list.prev = 0;
1619c7a60dfSdrh }
1629c7a60dfSdrh
1639c7a60dfSdrh /*
1649c7a60dfSdrh ** Unlink the chunk at index i from
1659c7a60dfSdrh ** whatever list is currently a member of.
1669c7a60dfSdrh */
memsys3Unlink(u32 i)16771f971b2Sdrh static void memsys3Unlink(u32 i){
16871f971b2Sdrh u32 size, hash;
16957e5ea93Sdanielk1977 assert( sqlite3_mutex_held(mem3.mutex) );
17057e5ea93Sdanielk1977 assert( (mem3.aPool[i-1].u.hdr.size4x & 1)==0 );
17171f971b2Sdrh assert( i>=1 );
17257e5ea93Sdanielk1977 size = mem3.aPool[i-1].u.hdr.size4x/4;
17357e5ea93Sdanielk1977 assert( size==mem3.aPool[i+size-1].u.hdr.prevSize );
1749c7a60dfSdrh assert( size>=2 );
1759c7a60dfSdrh if( size <= MX_SMALL ){
17657e5ea93Sdanielk1977 memsys3UnlinkFromList(i, &mem3.aiSmall[size-2]);
1779c7a60dfSdrh }else{
1789c7a60dfSdrh hash = size % N_HASH;
17957e5ea93Sdanielk1977 memsys3UnlinkFromList(i, &mem3.aiHash[hash]);
1809c7a60dfSdrh }
1819c7a60dfSdrh }
1829c7a60dfSdrh
1839c7a60dfSdrh /*
18457e5ea93Sdanielk1977 ** Link the chunk at mem3.aPool[i] so that is on the list rooted
1859c7a60dfSdrh ** at *pRoot.
1869c7a60dfSdrh */
memsys3LinkIntoList(u32 i,u32 * pRoot)18771f971b2Sdrh static void memsys3LinkIntoList(u32 i, u32 *pRoot){
18857e5ea93Sdanielk1977 assert( sqlite3_mutex_held(mem3.mutex) );
18957e5ea93Sdanielk1977 mem3.aPool[i].u.list.next = *pRoot;
19057e5ea93Sdanielk1977 mem3.aPool[i].u.list.prev = 0;
1919c7a60dfSdrh if( *pRoot ){
19257e5ea93Sdanielk1977 mem3.aPool[*pRoot].u.list.prev = i;
1939c7a60dfSdrh }
1949c7a60dfSdrh *pRoot = i;
1959c7a60dfSdrh }
1969c7a60dfSdrh
1979c7a60dfSdrh /*
1989c7a60dfSdrh ** Link the chunk at index i into either the appropriate
1999c7a60dfSdrh ** small chunk list, or into the large chunk hash table.
2009c7a60dfSdrh */
memsys3Link(u32 i)20171f971b2Sdrh static void memsys3Link(u32 i){
20271f971b2Sdrh u32 size, hash;
20357e5ea93Sdanielk1977 assert( sqlite3_mutex_held(mem3.mutex) );
20471f971b2Sdrh assert( i>=1 );
20557e5ea93Sdanielk1977 assert( (mem3.aPool[i-1].u.hdr.size4x & 1)==0 );
20657e5ea93Sdanielk1977 size = mem3.aPool[i-1].u.hdr.size4x/4;
20757e5ea93Sdanielk1977 assert( size==mem3.aPool[i+size-1].u.hdr.prevSize );
2089c7a60dfSdrh assert( size>=2 );
2099c7a60dfSdrh if( size <= MX_SMALL ){
21057e5ea93Sdanielk1977 memsys3LinkIntoList(i, &mem3.aiSmall[size-2]);
2119c7a60dfSdrh }else{
2129c7a60dfSdrh hash = size % N_HASH;
21357e5ea93Sdanielk1977 memsys3LinkIntoList(i, &mem3.aiHash[hash]);
2149c7a60dfSdrh }
2159c7a60dfSdrh }
2169c7a60dfSdrh
2179c7a60dfSdrh /*
2186b39c2e4Sdanielk1977 ** If the STATIC_MEM mutex is not already held, obtain it now. The mutex
2196b39c2e4Sdanielk1977 ** will already be held (obtained by code in malloc.c) if
220075c23afSdanielk1977 ** sqlite3GlobalConfig.bMemStat is true.
2219c7a60dfSdrh */
memsys3Enter(void)222a4e5d58fSdrh static void memsys3Enter(void){
223075c23afSdanielk1977 if( sqlite3GlobalConfig.bMemstat==0 && mem3.mutex==0 ){
22457e5ea93Sdanielk1977 mem3.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM);
2259c7a60dfSdrh }
22657e5ea93Sdanielk1977 sqlite3_mutex_enter(mem3.mutex);
2279c7a60dfSdrh }
memsys3Leave(void)22857e5ea93Sdanielk1977 static void memsys3Leave(void){
2296b39c2e4Sdanielk1977 sqlite3_mutex_leave(mem3.mutex);
2309c7a60dfSdrh }
2319c7a60dfSdrh
2329c7a60dfSdrh /*
233a4e5d58fSdrh ** Called when we are unable to satisfy an allocation of nBytes.
2349c7a60dfSdrh */
memsys3OutOfMemory(int nByte)235a4e5d58fSdrh static void memsys3OutOfMemory(int nByte){
23657e5ea93Sdanielk1977 if( !mem3.alarmBusy ){
23757e5ea93Sdanielk1977 mem3.alarmBusy = 1;
23857e5ea93Sdanielk1977 assert( sqlite3_mutex_held(mem3.mutex) );
23957e5ea93Sdanielk1977 sqlite3_mutex_leave(mem3.mutex);
240a4e5d58fSdrh sqlite3_release_memory(nByte);
24157e5ea93Sdanielk1977 sqlite3_mutex_enter(mem3.mutex);
24257e5ea93Sdanielk1977 mem3.alarmBusy = 0;
2439c7a60dfSdrh }
244a4e5d58fSdrh }
2459c7a60dfSdrh
24640257ffdSdrh
24740257ffdSdrh /*
2489c7a60dfSdrh ** Chunk i is a free chunk that has been unlinked. Adjust its
2499c7a60dfSdrh ** size parameters for check-out and return a pointer to the
2509c7a60dfSdrh ** user portion of the chunk.
2519c7a60dfSdrh */
memsys3Checkout(u32 i,u32 nBlock)252a03396aaSdanielk1977 static void *memsys3Checkout(u32 i, u32 nBlock){
25371f971b2Sdrh u32 x;
25457e5ea93Sdanielk1977 assert( sqlite3_mutex_held(mem3.mutex) );
25571f971b2Sdrh assert( i>=1 );
25657e5ea93Sdanielk1977 assert( mem3.aPool[i-1].u.hdr.size4x/4==nBlock );
25757e5ea93Sdanielk1977 assert( mem3.aPool[i+nBlock-1].u.hdr.prevSize==nBlock );
25857e5ea93Sdanielk1977 x = mem3.aPool[i-1].u.hdr.size4x;
25957e5ea93Sdanielk1977 mem3.aPool[i-1].u.hdr.size4x = nBlock*4 | 1 | (x&2);
26057e5ea93Sdanielk1977 mem3.aPool[i+nBlock-1].u.hdr.prevSize = nBlock;
26157e5ea93Sdanielk1977 mem3.aPool[i+nBlock-1].u.hdr.size4x |= 2;
26257e5ea93Sdanielk1977 return &mem3.aPool[i];
2639c7a60dfSdrh }
2649c7a60dfSdrh
2659c7a60dfSdrh /*
266*067b92baSdrh ** Carve a piece off of the end of the mem3.iKeyBlk free chunk.
267*067b92baSdrh ** Return a pointer to the new allocation. Or, if the key chunk
2689c7a60dfSdrh ** is not large enough, return 0.
2699c7a60dfSdrh */
memsys3FromKeyBlk(u32 nBlock)270*067b92baSdrh static void *memsys3FromKeyBlk(u32 nBlock){
27157e5ea93Sdanielk1977 assert( sqlite3_mutex_held(mem3.mutex) );
272*067b92baSdrh assert( mem3.szKeyBlk>=nBlock );
273*067b92baSdrh if( nBlock>=mem3.szKeyBlk-1 ){
274*067b92baSdrh /* Use the entire key chunk */
275*067b92baSdrh void *p = memsys3Checkout(mem3.iKeyBlk, mem3.szKeyBlk);
276*067b92baSdrh mem3.iKeyBlk = 0;
277*067b92baSdrh mem3.szKeyBlk = 0;
278*067b92baSdrh mem3.mnKeyBlk = 0;
2799c7a60dfSdrh return p;
2809c7a60dfSdrh }else{
281*067b92baSdrh /* Split the key block. Return the tail. */
28271f971b2Sdrh u32 newi, x;
283*067b92baSdrh newi = mem3.iKeyBlk + mem3.szKeyBlk - nBlock;
284*067b92baSdrh assert( newi > mem3.iKeyBlk+1 );
285*067b92baSdrh mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = nBlock;
286*067b92baSdrh mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x |= 2;
28757e5ea93Sdanielk1977 mem3.aPool[newi-1].u.hdr.size4x = nBlock*4 + 1;
288*067b92baSdrh mem3.szKeyBlk -= nBlock;
289*067b92baSdrh mem3.aPool[newi-1].u.hdr.prevSize = mem3.szKeyBlk;
290*067b92baSdrh x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2;
291*067b92baSdrh mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x;
292*067b92baSdrh if( mem3.szKeyBlk < mem3.mnKeyBlk ){
293*067b92baSdrh mem3.mnKeyBlk = mem3.szKeyBlk;
294a4e5d58fSdrh }
29557e5ea93Sdanielk1977 return (void*)&mem3.aPool[newi];
2969c7a60dfSdrh }
2979c7a60dfSdrh }
2989c7a60dfSdrh
2999c7a60dfSdrh /*
3009c7a60dfSdrh ** *pRoot is the head of a list of free chunks of the same size
3019c7a60dfSdrh ** or same size hash. In other words, *pRoot is an entry in either
30257e5ea93Sdanielk1977 ** mem3.aiSmall[] or mem3.aiHash[].
3039c7a60dfSdrh **
3049c7a60dfSdrh ** This routine examines all entries on the given list and tries
3059c7a60dfSdrh ** to coalesce each entries with adjacent free chunks.
3069c7a60dfSdrh **
307*067b92baSdrh ** If it sees a chunk that is larger than mem3.iKeyBlk, it replaces
308*067b92baSdrh ** the current mem3.iKeyBlk with the new larger chunk. In order for
309*067b92baSdrh ** this mem3.iKeyBlk replacement to work, the key chunk must be
3109c7a60dfSdrh ** linked into the hash tables. That is not the normal state of
311*067b92baSdrh ** affairs, of course. The calling routine must link the key
3129c7a60dfSdrh ** chunk before invoking this routine, then must unlink the (possibly
313*067b92baSdrh ** changed) key chunk once this routine has finished.
3149c7a60dfSdrh */
memsys3Merge(u32 * pRoot)31571f971b2Sdrh static void memsys3Merge(u32 *pRoot){
31671f971b2Sdrh u32 iNext, prev, size, i, x;
3179c7a60dfSdrh
31857e5ea93Sdanielk1977 assert( sqlite3_mutex_held(mem3.mutex) );
3199c7a60dfSdrh for(i=*pRoot; i>0; i=iNext){
32057e5ea93Sdanielk1977 iNext = mem3.aPool[i].u.list.next;
32157e5ea93Sdanielk1977 size = mem3.aPool[i-1].u.hdr.size4x;
32271f971b2Sdrh assert( (size&1)==0 );
32371f971b2Sdrh if( (size&2)==0 ){
324a4e5d58fSdrh memsys3UnlinkFromList(i, pRoot);
32557e5ea93Sdanielk1977 assert( i > mem3.aPool[i-1].u.hdr.prevSize );
32657e5ea93Sdanielk1977 prev = i - mem3.aPool[i-1].u.hdr.prevSize;
3279c7a60dfSdrh if( prev==iNext ){
32857e5ea93Sdanielk1977 iNext = mem3.aPool[prev].u.list.next;
3299c7a60dfSdrh }
330a4e5d58fSdrh memsys3Unlink(prev);
33171f971b2Sdrh size = i + size/4 - prev;
33257e5ea93Sdanielk1977 x = mem3.aPool[prev-1].u.hdr.size4x & 2;
33357e5ea93Sdanielk1977 mem3.aPool[prev-1].u.hdr.size4x = size*4 | x;
33457e5ea93Sdanielk1977 mem3.aPool[prev+size-1].u.hdr.prevSize = size;
335a4e5d58fSdrh memsys3Link(prev);
3369c7a60dfSdrh i = prev;
33771f971b2Sdrh }else{
33871f971b2Sdrh size /= 4;
3399c7a60dfSdrh }
340*067b92baSdrh if( size>mem3.szKeyBlk ){
341*067b92baSdrh mem3.iKeyBlk = i;
342*067b92baSdrh mem3.szKeyBlk = size;
3439c7a60dfSdrh }
3449c7a60dfSdrh }
3459c7a60dfSdrh }
3469c7a60dfSdrh
3479c7a60dfSdrh /*
3489c7a60dfSdrh ** Return a block of memory of at least nBytes in size.
3499c7a60dfSdrh ** Return NULL if unable.
35032155ef0Sdanielk1977 **
35132155ef0Sdanielk1977 ** This function assumes that the necessary mutexes, if any, are
35232155ef0Sdanielk1977 ** already held by the caller. Hence "Unsafe".
3539c7a60dfSdrh */
memsys3MallocUnsafe(int nByte)35432155ef0Sdanielk1977 static void *memsys3MallocUnsafe(int nByte){
35571f971b2Sdrh u32 i;
356a03396aaSdanielk1977 u32 nBlock;
357a03396aaSdanielk1977 u32 toFree;
3589c7a60dfSdrh
35957e5ea93Sdanielk1977 assert( sqlite3_mutex_held(mem3.mutex) );
3609c7a60dfSdrh assert( sizeof(Mem3Block)==8 );
36171f971b2Sdrh if( nByte<=12 ){
3629c7a60dfSdrh nBlock = 2;
3639c7a60dfSdrh }else{
36471f971b2Sdrh nBlock = (nByte + 11)/8;
3659c7a60dfSdrh }
3669c7a60dfSdrh assert( nBlock>=2 );
3679c7a60dfSdrh
3689c7a60dfSdrh /* STEP 1:
3699c7a60dfSdrh ** Look for an entry of the correct size in either the small
3709c7a60dfSdrh ** chunk table or in the large chunk hash table. This is
3719c7a60dfSdrh ** successful most of the time (about 9 times out of 10).
3729c7a60dfSdrh */
3739c7a60dfSdrh if( nBlock <= MX_SMALL ){
37457e5ea93Sdanielk1977 i = mem3.aiSmall[nBlock-2];
3759c7a60dfSdrh if( i>0 ){
37657e5ea93Sdanielk1977 memsys3UnlinkFromList(i, &mem3.aiSmall[nBlock-2]);
377a4e5d58fSdrh return memsys3Checkout(i, nBlock);
3789c7a60dfSdrh }
3799c7a60dfSdrh }else{
3809c7a60dfSdrh int hash = nBlock % N_HASH;
38157e5ea93Sdanielk1977 for(i=mem3.aiHash[hash]; i>0; i=mem3.aPool[i].u.list.next){
38257e5ea93Sdanielk1977 if( mem3.aPool[i-1].u.hdr.size4x/4==nBlock ){
38357e5ea93Sdanielk1977 memsys3UnlinkFromList(i, &mem3.aiHash[hash]);
384a4e5d58fSdrh return memsys3Checkout(i, nBlock);
3859c7a60dfSdrh }
3869c7a60dfSdrh }
3879c7a60dfSdrh }
3889c7a60dfSdrh
3899c7a60dfSdrh /* STEP 2:
3909c7a60dfSdrh ** Try to satisfy the allocation by carving a piece off of the end
391*067b92baSdrh ** of the key chunk. This step usually works if step 1 fails.
3929c7a60dfSdrh */
393*067b92baSdrh if( mem3.szKeyBlk>=nBlock ){
394*067b92baSdrh return memsys3FromKeyBlk(nBlock);
3959c7a60dfSdrh }
3969c7a60dfSdrh
3979c7a60dfSdrh
3989c7a60dfSdrh /* STEP 3:
3999c7a60dfSdrh ** Loop through the entire memory pool. Coalesce adjacent free
400*067b92baSdrh ** chunks. Recompute the key chunk as the largest free chunk.
4019c7a60dfSdrh ** Then try again to satisfy the allocation by carving a piece off
402*067b92baSdrh ** of the end of the key chunk. This step happens very
4039c7a60dfSdrh ** rarely (we hope!)
4049c7a60dfSdrh */
40557e5ea93Sdanielk1977 for(toFree=nBlock*16; toFree<(mem3.nPool*16); toFree *= 2){
406979aeaa3Sdrh memsys3OutOfMemory(toFree);
407*067b92baSdrh if( mem3.iKeyBlk ){
408*067b92baSdrh memsys3Link(mem3.iKeyBlk);
409*067b92baSdrh mem3.iKeyBlk = 0;
410*067b92baSdrh mem3.szKeyBlk = 0;
4119c7a60dfSdrh }
4129c7a60dfSdrh for(i=0; i<N_HASH; i++){
41357e5ea93Sdanielk1977 memsys3Merge(&mem3.aiHash[i]);
4149c7a60dfSdrh }
4159c7a60dfSdrh for(i=0; i<MX_SMALL-1; i++){
41657e5ea93Sdanielk1977 memsys3Merge(&mem3.aiSmall[i]);
4179c7a60dfSdrh }
418*067b92baSdrh if( mem3.szKeyBlk ){
419*067b92baSdrh memsys3Unlink(mem3.iKeyBlk);
420*067b92baSdrh if( mem3.szKeyBlk>=nBlock ){
421*067b92baSdrh return memsys3FromKeyBlk(nBlock);
4229c7a60dfSdrh }
4239c7a60dfSdrh }
424979aeaa3Sdrh }
4259c7a60dfSdrh
4269c7a60dfSdrh /* If none of the above worked, then we fail. */
4279c7a60dfSdrh return 0;
4289c7a60dfSdrh }
4299c7a60dfSdrh
4309c7a60dfSdrh /*
4319c7a60dfSdrh ** Free an outstanding memory allocation.
43232155ef0Sdanielk1977 **
43332155ef0Sdanielk1977 ** This function assumes that the necessary mutexes, if any, are
43432155ef0Sdanielk1977 ** already held by the caller. Hence "Unsafe".
4359c7a60dfSdrh */
memsys3FreeUnsafe(void * pOld)436e7152dc7Sdan static void memsys3FreeUnsafe(void *pOld){
4379c7a60dfSdrh Mem3Block *p = (Mem3Block*)pOld;
4389c7a60dfSdrh int i;
43971f971b2Sdrh u32 size, x;
44057e5ea93Sdanielk1977 assert( sqlite3_mutex_held(mem3.mutex) );
44157e5ea93Sdanielk1977 assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] );
44257e5ea93Sdanielk1977 i = p - mem3.aPool;
44357e5ea93Sdanielk1977 assert( (mem3.aPool[i-1].u.hdr.size4x&1)==1 );
44457e5ea93Sdanielk1977 size = mem3.aPool[i-1].u.hdr.size4x/4;
44557e5ea93Sdanielk1977 assert( i+size<=mem3.nPool+1 );
44657e5ea93Sdanielk1977 mem3.aPool[i-1].u.hdr.size4x &= ~1;
44757e5ea93Sdanielk1977 mem3.aPool[i+size-1].u.hdr.prevSize = size;
44857e5ea93Sdanielk1977 mem3.aPool[i+size-1].u.hdr.size4x &= ~2;
449a4e5d58fSdrh memsys3Link(i);
4509c7a60dfSdrh
451*067b92baSdrh /* Try to expand the key using the newly freed chunk */
452*067b92baSdrh if( mem3.iKeyBlk ){
453*067b92baSdrh while( (mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x&2)==0 ){
454*067b92baSdrh size = mem3.aPool[mem3.iKeyBlk-1].u.hdr.prevSize;
455*067b92baSdrh mem3.iKeyBlk -= size;
456*067b92baSdrh mem3.szKeyBlk += size;
457*067b92baSdrh memsys3Unlink(mem3.iKeyBlk);
458*067b92baSdrh x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2;
459*067b92baSdrh mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x;
460*067b92baSdrh mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = mem3.szKeyBlk;
4619c7a60dfSdrh }
462*067b92baSdrh x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2;
463*067b92baSdrh while( (mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x&1)==0 ){
464*067b92baSdrh memsys3Unlink(mem3.iKeyBlk+mem3.szKeyBlk);
465*067b92baSdrh mem3.szKeyBlk += mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x/4;
466*067b92baSdrh mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x;
467*067b92baSdrh mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = mem3.szKeyBlk;
4689c7a60dfSdrh }
4699c7a60dfSdrh }
4709c7a60dfSdrh }
4719c7a60dfSdrh
4729c7a60dfSdrh /*
473c702c7ccSdrh ** Return the size of an outstanding allocation, in bytes. The
474c702c7ccSdrh ** size returned omits the 8-byte header overhead. This only
475c702c7ccSdrh ** works for chunks that are currently checked out.
476c702c7ccSdrh */
memsys3Size(void * p)477c702c7ccSdrh static int memsys3Size(void *p){
478c702c7ccSdrh Mem3Block *pBlock;
479039ca6abSdrh assert( p!=0 );
480c702c7ccSdrh pBlock = (Mem3Block*)p;
481c702c7ccSdrh assert( (pBlock[-1].u.hdr.size4x&1)!=0 );
482c702c7ccSdrh return (pBlock[-1].u.hdr.size4x&~3)*2 - 4;
483c702c7ccSdrh }
484c702c7ccSdrh
485c702c7ccSdrh /*
486c702c7ccSdrh ** Round up a request size to the next valid allocation size.
487c702c7ccSdrh */
memsys3Roundup(int n)488c702c7ccSdrh static int memsys3Roundup(int n){
489c702c7ccSdrh if( n<=12 ){
490c702c7ccSdrh return 12;
491c702c7ccSdrh }else{
492c702c7ccSdrh return ((n+11)&~7) - 4;
493c702c7ccSdrh }
494c702c7ccSdrh }
495c702c7ccSdrh
496c702c7ccSdrh /*
49757e5ea93Sdanielk1977 ** Allocate nBytes of memory.
4989c7a60dfSdrh */
memsys3Malloc(int nBytes)49932155ef0Sdanielk1977 static void *memsys3Malloc(int nBytes){
50057e5ea93Sdanielk1977 sqlite3_int64 *p;
50157e5ea93Sdanielk1977 assert( nBytes>0 ); /* malloc.c filters out 0 byte requests */
502a4e5d58fSdrh memsys3Enter();
50332155ef0Sdanielk1977 p = memsys3MallocUnsafe(nBytes);
50457e5ea93Sdanielk1977 memsys3Leave();
5059c7a60dfSdrh return (void*)p;
5069c7a60dfSdrh }
5079c7a60dfSdrh
5089c7a60dfSdrh /*
5099c7a60dfSdrh ** Free memory.
5109c7a60dfSdrh */
memsys3Free(void * pPrior)511e7152dc7Sdan static void memsys3Free(void *pPrior){
51257e5ea93Sdanielk1977 assert( pPrior );
51357e5ea93Sdanielk1977 memsys3Enter();
51432155ef0Sdanielk1977 memsys3FreeUnsafe(pPrior);
51557e5ea93Sdanielk1977 memsys3Leave();
51657e5ea93Sdanielk1977 }
51757e5ea93Sdanielk1977
51857e5ea93Sdanielk1977 /*
5199c7a60dfSdrh ** Change the size of an existing memory allocation
5209c7a60dfSdrh */
memsys3Realloc(void * pPrior,int nBytes)521e7152dc7Sdan static void *memsys3Realloc(void *pPrior, int nBytes){
5229c7a60dfSdrh int nOld;
5239c7a60dfSdrh void *p;
5249c7a60dfSdrh if( pPrior==0 ){
5259c7a60dfSdrh return sqlite3_malloc(nBytes);
5269c7a60dfSdrh }
5279c7a60dfSdrh if( nBytes<=0 ){
5289c7a60dfSdrh sqlite3_free(pPrior);
5299c7a60dfSdrh return 0;
5309c7a60dfSdrh }
53132155ef0Sdanielk1977 nOld = memsys3Size(pPrior);
532a4e5d58fSdrh if( nBytes<=nOld && nBytes>=nOld-128 ){
533a4e5d58fSdrh return pPrior;
534a4e5d58fSdrh }
53557e5ea93Sdanielk1977 memsys3Enter();
53632155ef0Sdanielk1977 p = memsys3MallocUnsafe(nBytes);
537a4e5d58fSdrh if( p ){
5389c7a60dfSdrh if( nOld<nBytes ){
5399c7a60dfSdrh memcpy(p, pPrior, nOld);
5409c7a60dfSdrh }else{
5419c7a60dfSdrh memcpy(p, pPrior, nBytes);
5429c7a60dfSdrh }
54332155ef0Sdanielk1977 memsys3FreeUnsafe(pPrior);
5449c7a60dfSdrh }
54557e5ea93Sdanielk1977 memsys3Leave();
5469c7a60dfSdrh return p;
5479c7a60dfSdrh }
5489c7a60dfSdrh
5499c7a60dfSdrh /*
55057e5ea93Sdanielk1977 ** Initialize this module.
55157e5ea93Sdanielk1977 */
memsys3Init(void * NotUsed)55232155ef0Sdanielk1977 static int memsys3Init(void *NotUsed){
553a03396aaSdanielk1977 UNUSED_PARAMETER(NotUsed);
554075c23afSdanielk1977 if( !sqlite3GlobalConfig.pHeap ){
5550d84e5b2Sdanielk1977 return SQLITE_ERROR;
5560d84e5b2Sdanielk1977 }
5570d84e5b2Sdanielk1977
5580d84e5b2Sdanielk1977 /* Store a pointer to the memory block in global structure mem3. */
5590d84e5b2Sdanielk1977 assert( sizeof(Mem3Block)==8 );
560075c23afSdanielk1977 mem3.aPool = (Mem3Block *)sqlite3GlobalConfig.pHeap;
561075c23afSdanielk1977 mem3.nPool = (sqlite3GlobalConfig.nHeap / sizeof(Mem3Block)) - 2;
5620d84e5b2Sdanielk1977
563*067b92baSdrh /* Initialize the key block. */
564*067b92baSdrh mem3.szKeyBlk = mem3.nPool;
565*067b92baSdrh mem3.mnKeyBlk = mem3.szKeyBlk;
566*067b92baSdrh mem3.iKeyBlk = 1;
567*067b92baSdrh mem3.aPool[0].u.hdr.size4x = (mem3.szKeyBlk<<2) + 2;
5680d84e5b2Sdanielk1977 mem3.aPool[mem3.nPool].u.hdr.prevSize = mem3.nPool;
5690d84e5b2Sdanielk1977 mem3.aPool[mem3.nPool].u.hdr.size4x = 1;
5700d84e5b2Sdanielk1977
57157e5ea93Sdanielk1977 return SQLITE_OK;
57257e5ea93Sdanielk1977 }
57357e5ea93Sdanielk1977
57457e5ea93Sdanielk1977 /*
57557e5ea93Sdanielk1977 ** Deinitialize this module.
57657e5ea93Sdanielk1977 */
memsys3Shutdown(void * NotUsed)57732155ef0Sdanielk1977 static void memsys3Shutdown(void *NotUsed){
578a03396aaSdanielk1977 UNUSED_PARAMETER(NotUsed);
5794591c7baSdrh mem3.mutex = 0;
58057e5ea93Sdanielk1977 return;
58157e5ea93Sdanielk1977 }
58257e5ea93Sdanielk1977
58357e5ea93Sdanielk1977
58457e5ea93Sdanielk1977
58557e5ea93Sdanielk1977 /*
5869c7a60dfSdrh ** Open the file indicated and write a log of all unfreed memory
5879c7a60dfSdrh ** allocations into that log.
5889c7a60dfSdrh */
sqlite3Memsys3Dump(const char * zFilename)58932155ef0Sdanielk1977 void sqlite3Memsys3Dump(const char *zFilename){
5905c8f8587Sdanielk1977 #ifdef SQLITE_DEBUG
5919c7a60dfSdrh FILE *out;
592a03396aaSdanielk1977 u32 i, j;
59371f971b2Sdrh u32 size;
5949c7a60dfSdrh if( zFilename==0 || zFilename[0]==0 ){
5959c7a60dfSdrh out = stdout;
5969c7a60dfSdrh }else{
5979c7a60dfSdrh out = fopen(zFilename, "w");
5989c7a60dfSdrh if( out==0 ){
5999c7a60dfSdrh fprintf(stderr, "** Unable to output memory debug output log: %s **\n",
6009c7a60dfSdrh zFilename);
6019c7a60dfSdrh return;
6029c7a60dfSdrh }
6039c7a60dfSdrh }
604a4e5d58fSdrh memsys3Enter();
6059c7a60dfSdrh fprintf(out, "CHUNKS:\n");
60632155ef0Sdanielk1977 for(i=1; i<=mem3.nPool; i+=size/4){
60757e5ea93Sdanielk1977 size = mem3.aPool[i-1].u.hdr.size4x;
60871f971b2Sdrh if( size/4<=1 ){
60957e5ea93Sdanielk1977 fprintf(out, "%p size error\n", &mem3.aPool[i]);
6109c7a60dfSdrh assert( 0 );
6119c7a60dfSdrh break;
6129c7a60dfSdrh }
61357e5ea93Sdanielk1977 if( (size&1)==0 && mem3.aPool[i+size/4-1].u.hdr.prevSize!=size/4 ){
61457e5ea93Sdanielk1977 fprintf(out, "%p tail size does not match\n", &mem3.aPool[i]);
6159c7a60dfSdrh assert( 0 );
6169c7a60dfSdrh break;
6179c7a60dfSdrh }
61857e5ea93Sdanielk1977 if( ((mem3.aPool[i+size/4-1].u.hdr.size4x&2)>>1)!=(size&1) ){
61957e5ea93Sdanielk1977 fprintf(out, "%p tail checkout bit is incorrect\n", &mem3.aPool[i]);
62071f971b2Sdrh assert( 0 );
62171f971b2Sdrh break;
62271f971b2Sdrh }
62371f971b2Sdrh if( size&1 ){
62457e5ea93Sdanielk1977 fprintf(out, "%p %6d bytes checked out\n", &mem3.aPool[i], (size/4)*8-8);
6259c7a60dfSdrh }else{
62657e5ea93Sdanielk1977 fprintf(out, "%p %6d bytes free%s\n", &mem3.aPool[i], (size/4)*8-8,
627*067b92baSdrh i==mem3.iKeyBlk ? " **key**" : "");
6289c7a60dfSdrh }
6299c7a60dfSdrh }
6309c7a60dfSdrh for(i=0; i<MX_SMALL-1; i++){
63157e5ea93Sdanielk1977 if( mem3.aiSmall[i]==0 ) continue;
6329c7a60dfSdrh fprintf(out, "small(%2d):", i);
63357e5ea93Sdanielk1977 for(j = mem3.aiSmall[i]; j>0; j=mem3.aPool[j].u.list.next){
63457e5ea93Sdanielk1977 fprintf(out, " %p(%d)", &mem3.aPool[j],
63557e5ea93Sdanielk1977 (mem3.aPool[j-1].u.hdr.size4x/4)*8-8);
6369c7a60dfSdrh }
6379c7a60dfSdrh fprintf(out, "\n");
6389c7a60dfSdrh }
6399c7a60dfSdrh for(i=0; i<N_HASH; i++){
64057e5ea93Sdanielk1977 if( mem3.aiHash[i]==0 ) continue;
6419c7a60dfSdrh fprintf(out, "hash(%2d):", i);
64257e5ea93Sdanielk1977 for(j = mem3.aiHash[i]; j>0; j=mem3.aPool[j].u.list.next){
64357e5ea93Sdanielk1977 fprintf(out, " %p(%d)", &mem3.aPool[j],
64457e5ea93Sdanielk1977 (mem3.aPool[j-1].u.hdr.size4x/4)*8-8);
6459c7a60dfSdrh }
6469c7a60dfSdrh fprintf(out, "\n");
6479c7a60dfSdrh }
648*067b92baSdrh fprintf(out, "key=%d\n", mem3.iKeyBlk);
649*067b92baSdrh fprintf(out, "nowUsed=%d\n", mem3.nPool*8 - mem3.szKeyBlk*8);
650*067b92baSdrh fprintf(out, "mxUsed=%d\n", mem3.nPool*8 - mem3.mnKeyBlk*8);
65157e5ea93Sdanielk1977 sqlite3_mutex_leave(mem3.mutex);
6529c7a60dfSdrh if( out==stdout ){
6539c7a60dfSdrh fflush(stdout);
6549c7a60dfSdrh }else{
6559c7a60dfSdrh fclose(out);
6569c7a60dfSdrh }
657f3d3c27aSdanielk1977 #else
658f3d3c27aSdanielk1977 UNUSED_PARAMETER(zFilename);
65957e5ea93Sdanielk1977 #endif
6605c8f8587Sdanielk1977 }
6619c7a60dfSdrh
66257e5ea93Sdanielk1977 /*
66357e5ea93Sdanielk1977 ** This routine is the only routine in this file with external
66457e5ea93Sdanielk1977 ** linkage.
66557e5ea93Sdanielk1977 **
66657e5ea93Sdanielk1977 ** Populate the low-level memory allocation function pointers in
667075c23afSdanielk1977 ** sqlite3GlobalConfig.m with pointers to the routines in this file. The
66857e5ea93Sdanielk1977 ** arguments specify the block of memory to manage.
66957e5ea93Sdanielk1977 **
67057e5ea93Sdanielk1977 ** This routine is only called by sqlite3_config(), and therefore
67157e5ea93Sdanielk1977 ** is not required to be threadsafe (it is not).
67257e5ea93Sdanielk1977 */
sqlite3MemGetMemsys3(void)6730d84e5b2Sdanielk1977 const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){
67457e5ea93Sdanielk1977 static const sqlite3_mem_methods mempoolMethods = {
67532155ef0Sdanielk1977 memsys3Malloc,
67632155ef0Sdanielk1977 memsys3Free,
67732155ef0Sdanielk1977 memsys3Realloc,
67832155ef0Sdanielk1977 memsys3Size,
67932155ef0Sdanielk1977 memsys3Roundup,
68032155ef0Sdanielk1977 memsys3Init,
68132155ef0Sdanielk1977 memsys3Shutdown,
68257e5ea93Sdanielk1977 0
68357e5ea93Sdanielk1977 };
6840d84e5b2Sdanielk1977 return &mempoolMethods;
68557e5ea93Sdanielk1977 }
68657e5ea93Sdanielk1977
68732155ef0Sdanielk1977 #endif /* SQLITE_ENABLE_MEMSYS3 */
688