Home
last modified time | relevance | path

Searched refs:nToken (Results 1 – 25 of 27) sorted by relevance

12

/sqlite-3.40.0/ext/fts3/
H A Dfts3_test.c66 for(ii=0; ii<p->nToken; ii++){ in nm_phrase_match()
113 if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken; in nm_near_chain()
134 for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){ in nm_match_count()
207 for(ii=0; ii<doc.nToken; ii++){ in fts3_near_match_cmd()
220 int nToken; in fts3_near_match_cmd() local
225 if( nToken>NM_MAX_TOKEN ){ in fts3_near_match_cmd()
230 for(jj=0; jj<nToken; jj++){ in fts3_near_match_cmd()
234 aPhrase[ii].nToken = nToken; in fts3_near_match_cmd()
451 sqlite3_int64 nToken; in testTokenizerNext() local
456 if( nToken>pCsr->nBuffer ){ in testTokenizerNext()
[all …]
H A Dfts3_snippet.c304 p->nToken += pPhrase->nToken; in fts3ExprLoadDoclistsCb()
477 pPhrase->nToken = pExpr->pPhrase->nToken; in fts3SnippetFindPositions()
1146 int nToken = 0; in fts3MatchinfoLcs() local
1158 nToken -= pIter->pExpr->pPhrase->nToken; in fts3MatchinfoLcs()
1159 pIter->iPosOffset = nToken; in fts3MatchinfoLcs()
1292 sqlite3_int64 nToken; in fts3MatchinfoValues() local
1469 if( nToken<-64 ) nToken = -64; in sqlite3Fts3Snippet()
1470 if( nToken>+64 ) nToken = +64; in sqlite3Fts3Snippet()
1478 if( nToken>=0 ){ in sqlite3Fts3Snippet()
1481 nFToken = -1 * nToken; in sqlite3Fts3Snippet()
[all …]
H A Dfts3.c4173 int nToken = pExpr->pPhrase->nToken; in fts3EvalAllocateReaders() local
4435 && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 in fts3EvalPhraseStart()
4702 if( p->nToken==1 ){ in fts3EvalIncrPhraseNext()
4728 assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); in fts3EvalIncrPhraseNext()
4835 int nToken = pExpr->pPhrase->nToken; in fts3EvalStartReaders() local
4836 if( nToken ){ in fts3EvalStartReaders()
5027 nToken++; in fts3EvalSelectDeferred()
5088 if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ in fts3EvalSelectDeferred()
5132 int nToken = 0; in fts3EvalStart() local
5471 int nToken = p->pPhrase->nToken; in fts3EvalNearTest() local
[all …]
H A Dfts3_expr.c200 int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; in getNextToken() local
212 pRet->pPhrase->nToken = 1; in getNextToken()
214 pRet->pPhrase->aToken[0].n = nToken; in getNextToken()
216 memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); in getNextToken()
289 int nToken = 0; in getNextString() local
327 assert( nToken==ii ); in getNextString()
337 nToken = ii+1; in getNextString()
355 p->pPhrase->nToken = nToken; in getNextString()
357 zBuf = (char *)&p->pPhrase->aToken[nToken]; in getNextString()
365 for(jj=0; jj<p->pPhrase->nToken; jj++){ in getNextString()
[all …]
H A Dfts3_tokenizer.c281 int nToken = 0; in testFunc() local
326 while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ in testFunc()
328 Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); in testFunc()
330 nToken = iEnd-iStart; in testFunc()
331 Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); in testFunc()
H A Dfts3_tokenize_vtab.c68 int nToken; /* Size of zToken in bytes */ member
290 pCsr->nToken = 0; in fts3tokResetCursor()
318 &pCsr->zToken, &pCsr->nToken, in fts3tokNextMethod()
391 sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); in fts3tokColumnMethod()
H A Dfts3_write.c756 int nToken in fts3PendingTermsAddOne() argument
761 pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); in fts3PendingTermsAddOne()
766 if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ in fts3PendingTermsAddOne()
770 assert( 0==fts3HashFind(pHash, zToken, nToken) ); in fts3PendingTermsAddOne()
802 int nToken = 0; in fts3PendingTermsAdd() local
835 if( iPos<0 || !zToken || nToken<=0 ){ in fts3PendingTermsAdd()
842 p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken in fts3PendingTermsAdd()
849 if( nToken<pIndex->nPrefix ) continue; in fts3PendingTermsAdd()
5343 zToken, nToken, iLang, 0, iDocid, iCol, iPos in fts3IntegrityCheck()
5346 if( p->aIndex[i].nPrefix<=nToken ){ in fts3IntegrityCheck()
[all …]
/sqlite-3.40.0/ext/fts5/
H A Dfts5_aux.c153 int nToken, /* Size of token in bytes */ in fts5HighlightCb() argument
161 UNUSED_PARAM2(pToken, nToken); in fts5HighlightCb()
289 int nToken, /* Size of token in bytes */ in fts5SentenceFinderCb() argument
295 UNUSED_PARAM2(pToken, nToken); in fts5SentenceFinderCb()
325 int nToken, /* Max tokens per snippet */ in fts5SnippetScore() argument
338 sqlite3_int64 iEnd = (sqlite3_int64)iPos + nToken; in fts5SnippetScore()
354 if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken; in fts5SnippetScore()
410 nToken = sqlite3_value_int(apVal[4]); in fts5SnippetFunction()
451 io, nToken, &nScore, &iAdj in fts5SnippetFunction()
468 sFinder.aFirst[jj], nToken, &nScore, 0 in fts5SnippetFunction()
[all …]
H A Dfts5_hash.c247 const char *pToken, int nToken /* Token to add or remove to or from index */ in sqlite3Fts5HashWrite() argument
258 iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); in sqlite3Fts5HashWrite()
262 && p->nKey==nToken in sqlite3Fts5HashWrite()
263 && memcmp(&zKey[1], pToken, nToken)==0 in sqlite3Fts5HashWrite()
273 sqlite3_int64 nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64; in sqlite3Fts5HashWrite()
280 iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); in sqlite3Fts5HashWrite()
290 memcpy(&zKey[1], pToken, nToken); in sqlite3Fts5HashWrite()
291 assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) ); in sqlite3Fts5HashWrite()
292 p->nKey = nToken; in sqlite3Fts5HashWrite()
293 zKey[nToken+1] = '\0'; in sqlite3Fts5HashWrite()
[all …]
H A Dfts5_test_mi.c191 sqlite3_int64 nToken; in fts5MatchinfoGlobalCb() local
192 rc = pApi->xColumnTotalSize(pFts, i, &nToken); in fts5MatchinfoGlobalCb()
194 aOut[i] = (u32)((2*nToken + nRow) / (2*nRow)); in fts5MatchinfoGlobalCb()
258 int nToken; in fts5MatchinfoLocalCb() local
259 rc = pApi->xColumnSize(pFts, i, &nToken); in fts5MatchinfoLocalCb()
260 aOut[i] = (u32)nToken; in fts5MatchinfoLocalCb()
H A Dfts5_storage.c373 int nToken, /* Size of token in bytes */ in fts5StorageInsertCallback() argument
380 if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; in fts5StorageInsertCallback()
384 return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); in fts5StorageInsertCallback()
822 int nToken, /* Size of token in bytes */ in fts5StorageIntegrityCallback() argument
835 if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; in fts5StorageIntegrityCallback()
859 rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); in fts5StorageIntegrityCallback()
862 pCtx->iRowid, iCol, iPos, 0, pToken, nToken in fts5StorageIntegrityCallback()
868 int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); in fts5StorageIntegrityCallback()
H A Dfts5_test_tok.c328 int nToken, /* Size of token in bytes */ in fts5tokCb() argument
350 pRow->zToken = sqlite3_malloc(nToken+1); in fts5tokCb()
352 memcpy(pRow->zToken, pToken, nToken); in fts5tokCb()
353 pRow->zToken[nToken] = 0; in fts5tokCb()
H A Dfts5_tcl.c152 const char *zToken, int nToken, in xTokenizeCb() argument
160 Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken)); in xTokenizeCb()
658 const char *zToken, int nToken, in xTokenizeCb2() argument
663 Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken)); in xTokenizeCb2()
668 Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken)); in xTokenizeCb2()
896 int nToken; in f5tTokenizerReturn() local
913 zToken = Tcl_GetStringFromObj(objv[objc-3], &nToken); in f5tTokenizerReturn()
927 rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd); in f5tTokenizerReturn()
H A Dfts5_tokenize.c122 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) in fts5AsciiTokenize() argument
443 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) in fts5UnicodeTokenize() argument
1179 int nToken, in fts5PorterCb() argument
1188 if( nToken>FTS5_PORTER_MAX_TOKEN || nToken<3 ) goto pass_through; in fts5PorterCb()
1190 nBuf = nToken; in fts5PorterCb()
1238 return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); in fts5PorterCb()
1249 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) in fts5PorterTokenize() argument
H A Dfts5Int.h422 const char *pToken, int nToken, /* Token (or prefix) to query for */
469 const char *pToken, int nToken /* Token to add or remove to or from index */
604 const char *pToken, int nToken /* Token to add or remove to or from index */
H A Dfts5_expr.c1664 int nToken, /* Size of token in bytes */ in fts5ParseTokenize() argument
1677 if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; in fts5ParseTokenize()
1681 sqlite3_int64 nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; in fts5ParseTokenize()
1688 memcpy(pSyn->zTerm, pToken, nToken); in fts5ParseTokenize()
1713 pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); in fts5ParseTokenize()
2915 int nToken, /* Size of token in bytes */ in fts5ExprPopulatePoslistsCb() argument
2925 if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; in fts5ExprPopulatePoslistsCb()
2932 if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix)) in fts5ExprPopulatePoslistsCb()
H A Dfts5.h512 int nToken, /* Size of token in bytes */
H A Dfts5_index.c5205 int nToken, /* Size of buffer pToken in bytes */ in fts5SetupPrefixIter() argument
5245 fts5MultiIterNew(p, pStruct, f2, pColset, pToken, nToken, -1, 0, &p1); in fts5SetupPrefixIter()
5273 assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); in fts5SetupPrefixIter()
5275 if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break; in fts5SetupPrefixIter()
5532 p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken in sqlite3Fts5IndexWrite()
5537 int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); in sqlite3Fts5IndexWrite()
5555 const char *pToken, int nToken, /* Token (or prefix) to query for */ in sqlite3Fts5IndexQuery() argument
5567 if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ in sqlite3Fts5IndexQuery()
5570 if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); in sqlite3Fts5IndexQuery()
5589 int nChar = fts5IndexCharlen(pToken, nToken); in sqlite3Fts5IndexQuery()
[all …]
/sqlite-3.40.0/src/
H A Dvdbetrace.c80 int nToken; /* Length of the parameter token */ in sqlite3VdbeExpandSql() local
102 n = findNextHostParameter(zRawSql, &nToken); in sqlite3VdbeExpandSql()
106 assert( zRawSql[0] || nToken==0 ); in sqlite3VdbeExpandSql()
107 if( nToken==0 ) break; in sqlite3VdbeExpandSql()
109 if( nToken>1 ){ in sqlite3VdbeExpandSql()
122 idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken); in sqlite3VdbeExpandSql()
125 zRawSql += nToken; in sqlite3VdbeExpandSql()
/sqlite-3.40.0/ext/fts2/
H A Dfts2_tokenizer.c155 int nToken; in testFunc() local
197 while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ in testFunc()
199 Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); in testFunc()
201 nToken = iEnd-iStart; in testFunc()
202 Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); in testFunc()
H A Dfts2.c2503 int nToken = 0; in tokenizeString() local
2513 aToken[nToken].z = z; in tokenizeString()
2514 aToken[nToken].n = n; in tokenizeString()
2515 nToken++; in tokenizeString()
2521 zCopy = (char*)&azToken[nToken]; in tokenizeString()
2522 nToken--; in tokenizeString()
2523 for(i=0; i<nToken; i++){ in tokenizeString()
2530 azToken[nToken] = 0; in tokenizeString()
2532 *pnToken = nToken; in tokenizeString()
3552 int nToken, iBegin, iEnd, iPos; in tokenizeSegment() local
[all …]
/sqlite-3.40.0/ext/fts3/tool/
H A Dfts3view.c188 int nToken = 0; in showVocabulary() local
208 nToken = sqlite3_column_int(pStmt, 0); in showVocabulary()
213 printf("Total number of distinct tokens.......... %9d\n", nToken); in showVocabulary()
214 if( nToken==0 ) goto end_vocab; in showVocabulary()
224 n, n*100.0/nToken); in showVocabulary()
234 n, n*100.0/nToken); in showVocabulary()
245 n, n*100.0/nToken); in showVocabulary()
257 n, n*100.0/nToken); in showVocabulary()
/sqlite-3.40.0/ext/fts5/test/
H A Dfts5_common.tcl267 set nToken [llength $zCol]
287 set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
288 for { } {$iLast < $nToken} {incr iLast} {
408 while {[set nToken [gobble_text text word]]} {
410 incr iOff $nToken
/sqlite-3.40.0/ext/fts1/
H A Dfts1.c1635 int nToken = 0; in tokenizeString() local
1645 aToken[nToken].z = z; in tokenizeString()
1646 aToken[nToken].n = n; in tokenizeString()
1647 nToken++; in tokenizeString()
1653 zCopy = (char*)&azToken[nToken]; in tokenizeString()
1654 nToken--; in tokenizeString()
1655 for(i=0; i<nToken; i++){ in tokenizeString()
1662 azToken[nToken] = 0; in tokenizeString()
1664 *pnToken = nToken; in tokenizeString()
2689 int nToken, iBegin, iEnd, iPos; in tokenizeSegment() local
[all …]
H A Dfulltext.c1162 int nToken, iStartOffset, iEndOffset, dummy_pos; in tokenize_segment() local
1165 &zToken, &nToken, in tokenize_segment()
1169 query_add(pQuery, !in_phrase || is_first, string_dup_n(zToken, nToken)); in tokenize_segment()

12