fts3.c.svn-base
上传用户:sunhongbo
上传日期:2022-01-25
资源大小:3010k
文件大小:200k
- if( aTerm[i].nTerm>nToken ) continue;
- if( !aTerm[i].isPrefix && aTerm[i].nTerm<nToken ) continue;
- assert( aTerm[i].nTerm<=nToken );
- if( memcmp(aTerm[i].pTerm, zToken, aTerm[i].nTerm) ) continue;
- if( aTerm[i].iPhrase>1 && (prevMatch & (1<<i))==0 ) continue;
- match |= 1<<i;
- if( i==nTerm-1 || aTerm[i+1].iPhrase==1 ){
- for(j=aTerm[i].iPhrase-1; j>=0; j--){
- int k = (iRotor-j) & FTS3_ROTOR_MASK;
- snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
- iRotorBegin[k], iRotorLen[k]);
- }
- }
- }
- prevMatch = match<<1;
- iRotor++;
- }
- pTModule->xClose(pTCursor);
- }
- /*
- ** Remove entries from the pSnippet structure to account for the NEAR
- ** operator. When this is called, pSnippet contains the list of token
- ** offsets produced by treating all NEAR operators as AND operators.
- ** This function removes any entries that should not be present after
- ** accounting for the NEAR restriction. For example, if the queried
- ** document is:
- **
- ** "A B C D E A"
- **
- ** and the query is:
- **
- ** A NEAR/0 E
- **
- ** then when this function is called the Snippet contains token offsets
- ** 0, 4 and 5. This function removes the "0" entry (because the first A
- ** is not near enough to an E).
- */
- static void trimSnippetOffsetsForNear(Query *pQuery, Snippet *pSnippet){
- int ii;
- int iDir = 1;
- while(iDir>-2) {
- assert( iDir==1 || iDir==-1 );
- for(ii=0; ii<pSnippet->nMatch; ii++){
- int jj;
- int nNear;
- struct snippetMatch *pMatch = &pSnippet->aMatch[ii];
- QueryTerm *pQueryTerm = &pQuery->pTerms[pMatch->iTerm];
- if( (pMatch->iTerm+iDir)<0
- || (pMatch->iTerm+iDir)>=pQuery->nTerms
- ){
- continue;
- }
-
- nNear = pQueryTerm->nNear;
- if( iDir<0 ){
- nNear = pQueryTerm[-1].nNear;
- }
-
- if( pMatch->iTerm>=0 && nNear ){
- int isOk = 0;
- int iNextTerm = pMatch->iTerm+iDir;
- int iPrevTerm = iNextTerm;
- int iEndToken;
- int iStartToken;
- if( iDir<0 ){
- int nPhrase = 1;
- iStartToken = pMatch->iToken;
- while( (pMatch->iTerm+nPhrase)<pQuery->nTerms
- && pQuery->pTerms[pMatch->iTerm+nPhrase].iPhrase>1
- ){
- nPhrase++;
- }
- iEndToken = iStartToken + nPhrase - 1;
- }else{
- iEndToken = pMatch->iToken;
- iStartToken = pMatch->iToken+1-pQueryTerm->iPhrase;
- }
- while( pQuery->pTerms[iNextTerm].iPhrase>1 ){
- iNextTerm--;
- }
- while( (iPrevTerm+1)<pQuery->nTerms &&
- pQuery->pTerms[iPrevTerm+1].iPhrase>1
- ){
- iPrevTerm++;
- }
-
- for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
- struct snippetMatch *p = &pSnippet->aMatch[jj];
- if( p->iCol==pMatch->iCol && ((
- p->iTerm==iNextTerm &&
- p->iToken>iEndToken &&
- p->iToken<=iEndToken+nNear
- ) || (
- p->iTerm==iPrevTerm &&
- p->iToken<iStartToken &&
- p->iToken>=iStartToken-nNear
- ))){
- isOk = 1;
- }
- }
- if( !isOk ){
- for(jj=1-pQueryTerm->iPhrase; jj<=0; jj++){
- pMatch[jj].iTerm = -1;
- }
- ii = -1;
- iDir = 1;
- }
- }
- }
- iDir -= 2;
- }
- }
- /*
- ** Compute all offsets for the current row of the query.
- ** If the offsets have already been computed, this routine is a no-op.
- */
- static void snippetAllOffsets(fulltext_cursor *p){
- int nColumn;
- int iColumn, i;
- int iFirst, iLast;
- fulltext_vtab *pFts;
- if( p->snippet.nMatch ) return;
- if( p->q.nTerms==0 ) return;
- pFts = p->q.pFts;
- nColumn = pFts->nColumn;
- iColumn = (p->iCursorType - QUERY_FULLTEXT);
- if( iColumn<0 || iColumn>=nColumn ){
- iFirst = 0;
- iLast = nColumn-1;
- }else{
- iFirst = iColumn;
- iLast = iColumn;
- }
- for(i=iFirst; i<=iLast; i++){
- const char *zDoc;
- int nDoc;
- zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
- nDoc = sqlite3_column_bytes(p->pStmt, i+1);
- snippetOffsetsOfColumn(&p->q, &p->snippet, i, zDoc, nDoc);
- }
- trimSnippetOffsetsForNear(&p->q, &p->snippet);
- }
- /*
- ** Convert the information in the aMatch[] array of the snippet
- ** into the string zOffset[0..nOffset-1].
- */
- static void snippetOffsetText(Snippet *p){
- int i;
- int cnt = 0;
- StringBuffer sb;
- char zBuf[200];
- if( p->zOffset ) return;
- initStringBuffer(&sb);
- for(i=0; i<p->nMatch; i++){
- struct snippetMatch *pMatch = &p->aMatch[i];
- if( pMatch->iTerm>=0 ){
- /* If snippetMatch.iTerm is less than 0, then the match was
- ** discarded as part of processing the NEAR operator (see the
- ** trimSnippetOffsetsForNear() function for details). Ignore
- ** it in this case
- */
- zBuf[0] = ' ';
- sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
- pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
- append(&sb, zBuf);
- cnt++;
- }
- }
- p->zOffset = stringBufferData(&sb);
- p->nOffset = stringBufferLength(&sb);
- }
- /*
- ** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
- ** of matching words some of which might be in zDoc. zDoc is column
- ** number iCol.
- **
- ** iBreak is suggested spot in zDoc where we could begin or end an
- ** excerpt. Return a value similar to iBreak but possibly adjusted
- ** to be a little left or right so that the break point is better.
- */
- static int wordBoundary(
- int iBreak, /* The suggested break point */
- const char *zDoc, /* Document text */
- int nDoc, /* Number of bytes in zDoc[] */
- struct snippetMatch *aMatch, /* Matching words */
- int nMatch, /* Number of entries in aMatch[] */
- int iCol /* The column number for zDoc[] */
- ){
- int i;
- if( iBreak<=10 ){
- return 0;
- }
- if( iBreak>=nDoc-10 ){
- return nDoc;
- }
- for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
- while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
- if( i<nMatch ){
- if( aMatch[i].iStart<iBreak+10 ){
- return aMatch[i].iStart;
- }
- if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
- return aMatch[i-1].iStart;
- }
- }
- for(i=1; i<=10; i++){
- if( safe_isspace(zDoc[iBreak-i]) ){
- return iBreak - i + 1;
- }
- if( safe_isspace(zDoc[iBreak+i]) ){
- return iBreak + i + 1;
- }
- }
- return iBreak;
- }
- /*
- ** Allowed values for Snippet.aMatch[].snStatus
- */
- #define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
- #define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
- /*
- ** Generate the text of a snippet.
- */
- static void snippetText(
- fulltext_cursor *pCursor, /* The cursor we need the snippet for */
- const char *zStartMark, /* Markup to appear before each match */
- const char *zEndMark, /* Markup to appear after each match */
- const char *zEllipsis /* Ellipsis mark */
- ){
- int i, j;
- struct snippetMatch *aMatch;
- int nMatch;
- int nDesired;
- StringBuffer sb;
- int tailCol;
- int tailOffset;
- int iCol;
- int nDoc;
- const char *zDoc;
- int iStart, iEnd;
- int tailEllipsis = 0;
- int iMatch;
-
- sqlite3_free(pCursor->snippet.zSnippet);
- pCursor->snippet.zSnippet = 0;
- aMatch = pCursor->snippet.aMatch;
- nMatch = pCursor->snippet.nMatch;
- initStringBuffer(&sb);
- for(i=0; i<nMatch; i++){
- aMatch[i].snStatus = SNIPPET_IGNORE;
- }
- nDesired = 0;
- for(i=0; i<pCursor->q.nTerms; i++){
- for(j=0; j<nMatch; j++){
- if( aMatch[j].iTerm==i ){
- aMatch[j].snStatus = SNIPPET_DESIRED;
- nDesired++;
- break;
- }
- }
- }
- iMatch = 0;
- tailCol = -1;
- tailOffset = 0;
- for(i=0; i<nMatch && nDesired>0; i++){
- if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
- nDesired--;
- iCol = aMatch[i].iCol;
- zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
- nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
- iStart = aMatch[i].iStart - 40;
- iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
- if( iStart<=10 ){
- iStart = 0;
- }
- if( iCol==tailCol && iStart<=tailOffset+20 ){
- iStart = tailOffset;
- }
- if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){
- trimWhiteSpace(&sb);
- appendWhiteSpace(&sb);
- append(&sb, zEllipsis);
- appendWhiteSpace(&sb);
- }
- iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
- iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
- if( iEnd>=nDoc-10 ){
- iEnd = nDoc;
- tailEllipsis = 0;
- }else{
- tailEllipsis = 1;
- }
- while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
- while( iStart<iEnd ){
- while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
- && aMatch[iMatch].iCol<=iCol ){
- iMatch++;
- }
- if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
- && aMatch[iMatch].iCol==iCol ){
- nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
- iStart = aMatch[iMatch].iStart;
- append(&sb, zStartMark);
- nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
- append(&sb, zEndMark);
- iStart += aMatch[iMatch].nByte;
- for(j=iMatch+1; j<nMatch; j++){
- if( aMatch[j].iTerm==aMatch[iMatch].iTerm
- && aMatch[j].snStatus==SNIPPET_DESIRED ){
- nDesired--;
- aMatch[j].snStatus = SNIPPET_IGNORE;
- }
- }
- }else{
- nappend(&sb, &zDoc[iStart], iEnd - iStart);
- iStart = iEnd;
- }
- }
- tailCol = iCol;
- tailOffset = iEnd;
- }
- trimWhiteSpace(&sb);
- if( tailEllipsis ){
- appendWhiteSpace(&sb);
- append(&sb, zEllipsis);
- }
- pCursor->snippet.zSnippet = stringBufferData(&sb);
- pCursor->snippet.nSnippet = stringBufferLength(&sb);
- }
- /*
- ** Close the cursor. For additional information see the documentation
- ** on the xClose method of the virtual table interface.
- */
- static int fulltextClose(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- FTSTRACE(("FTS3 Close %pn", c));
- sqlite3_finalize(c->pStmt);
- queryClear(&c->q);
- snippetClear(&c->snippet);
- if( c->result.nData!=0 ) dlrDestroy(&c->reader);
- dataBufferDestroy(&c->result);
- sqlite3_free(c);
- return SQLITE_OK;
- }
- static int fulltextNext(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- int rc;
- FTSTRACE(("FTS3 Next %pn", pCursor));
- snippetClear(&c->snippet);
- if( c->iCursorType < QUERY_FULLTEXT ){
- /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
- rc = sqlite3_step(c->pStmt);
- switch( rc ){
- case SQLITE_ROW:
- c->eof = 0;
- return SQLITE_OK;
- case SQLITE_DONE:
- c->eof = 1;
- return SQLITE_OK;
- default:
- c->eof = 1;
- return rc;
- }
- } else { /* full-text query */
- rc = sqlite3_reset(c->pStmt);
- if( rc!=SQLITE_OK ) return rc;
- if( c->result.nData==0 || dlrAtEnd(&c->reader) ){
- c->eof = 1;
- return SQLITE_OK;
- }
- rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
- dlrStep(&c->reader);
- if( rc!=SQLITE_OK ) return rc;
- /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
- rc = sqlite3_step(c->pStmt);
- if( rc==SQLITE_ROW ){ /* the case we expect */
- c->eof = 0;
- return SQLITE_OK;
- }
- /* an error occurred; abort */
- return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
- }
- }
- /* TODO(shess) If we pushed LeafReader to the top of the file, or to
- ** another file, term_select() could be pushed above
- ** docListOfTerm().
- */
- static int termSelect(fulltext_vtab *v, int iColumn,
- const char *pTerm, int nTerm, int isPrefix,
- DocListType iType, DataBuffer *out);
- /* Return a DocList corresponding to the query term *pTerm. If *pTerm
- ** is the first term of a phrase query, go ahead and evaluate the phrase
- ** query and return the doclist for the entire phrase query.
- **
- ** The resulting DL_DOCIDS doclist is stored in pResult, which is
- ** overwritten.
- */
- static int docListOfTerm(
- fulltext_vtab *v, /* The full text index */
- int iColumn, /* column to restrict to. No restriction if >=nColumn */
- QueryTerm *pQTerm, /* Term we are looking for, or 1st term of a phrase */
- DataBuffer *pResult /* Write the result here */
- ){
- DataBuffer left, right, new;
- int i, rc;
- /* No phrase search if no position info. */
- assert( pQTerm->nPhrase==0 || DL_DEFAULT!=DL_DOCIDS );
- /* This code should never be called with buffered updates. */
- assert( v->nPendingData<0 );
- dataBufferInit(&left, 0);
- rc = termSelect(v, iColumn, pQTerm->pTerm, pQTerm->nTerm, pQTerm->isPrefix,
- (0<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS), &left);
- if( rc ) return rc;
- for(i=1; i<=pQTerm->nPhrase && left.nData>0; i++){
- /* If this token is connected to the next by a NEAR operator, and
- ** the next token is the start of a phrase, then set nPhraseRight
- ** to the number of tokens in the phrase. Otherwise leave it at 1.
- */
- int nPhraseRight = 1;
- while( (i+nPhraseRight)<=pQTerm->nPhrase
- && pQTerm[i+nPhraseRight].nNear==0
- ){
- nPhraseRight++;
- }
- dataBufferInit(&right, 0);
- rc = termSelect(v, iColumn, pQTerm[i].pTerm, pQTerm[i].nTerm,
- pQTerm[i].isPrefix, DL_POSITIONS, &right);
- if( rc ){
- dataBufferDestroy(&left);
- return rc;
- }
- dataBufferInit(&new, 0);
- docListPhraseMerge(left.pData, left.nData, right.pData, right.nData,
- pQTerm[i-1].nNear, pQTerm[i-1].iPhrase + nPhraseRight,
- ((i<pQTerm->nPhrase) ? DL_POSITIONS : DL_DOCIDS),
- &new);
- dataBufferDestroy(&left);
- dataBufferDestroy(&right);
- left = new;
- }
- *pResult = left;
- return SQLITE_OK;
- }
- /* Add a new term pTerm[0..nTerm-1] to the query *q.
- */
- static void queryAdd(Query *q, const char *pTerm, int nTerm){
- QueryTerm *t;
- ++q->nTerms;
- q->pTerms = sqlite3_realloc(q->pTerms, q->nTerms * sizeof(q->pTerms[0]));
- if( q->pTerms==0 ){
- q->nTerms = 0;
- return;
- }
- t = &q->pTerms[q->nTerms - 1];
- CLEAR(t);
- t->pTerm = sqlite3_malloc(nTerm+1);
- memcpy(t->pTerm, pTerm, nTerm);
- t->pTerm[nTerm] = 0;
- t->nTerm = nTerm;
- t->isOr = q->nextIsOr;
- t->isPrefix = 0;
- q->nextIsOr = 0;
- t->iColumn = q->nextColumn;
- q->nextColumn = q->dfltColumn;
- }
- /*
- ** Check to see if the string zToken[0...nToken-1] matches any
- ** column name in the virtual table. If it does,
- ** return the zero-indexed column number. If not, return -1.
- */
- static int checkColumnSpecifier(
- fulltext_vtab *pVtab, /* The virtual table */
- const char *zToken, /* Text of the token */
- int nToken /* Number of characters in the token */
- ){
- int i;
- for(i=0; i<pVtab->nColumn; i++){
- if( memcmp(pVtab->azColumn[i], zToken, nToken)==0
- && pVtab->azColumn[i][nToken]==0 ){
- return i;
- }
- }
- return -1;
- }
- /*
- ** Parse the text at pSegment[0..nSegment-1]. Add additional terms
- ** to the query being assemblied in pQuery.
- **
- ** inPhrase is true if pSegment[0..nSegement-1] is contained within
- ** double-quotes. If inPhrase is true, then the first term
- ** is marked with the number of terms in the phrase less one and
- ** OR and "-" syntax is ignored. If inPhrase is false, then every
- ** term found is marked with nPhrase=0 and OR and "-" syntax is significant.
- */
- static int tokenizeSegment(
- sqlite3_tokenizer *pTokenizer, /* The tokenizer to use */
- const char *pSegment, int nSegment, /* Query expression being parsed */
- int inPhrase, /* True if within "..." */
- Query *pQuery /* Append results here */
- ){
- const sqlite3_tokenizer_module *pModule = pTokenizer->pModule;
- sqlite3_tokenizer_cursor *pCursor;
- int firstIndex = pQuery->nTerms;
- int iCol;
- int nTerm = 1;
-
- int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor);
- if( rc!=SQLITE_OK ) return rc;
- pCursor->pTokenizer = pTokenizer;
- while( 1 ){
- const char *pToken;
- int nToken, iBegin, iEnd, iPos;
- rc = pModule->xNext(pCursor,
- &pToken, &nToken,
- &iBegin, &iEnd, &iPos);
- if( rc!=SQLITE_OK ) break;
- if( !inPhrase &&
- pSegment[iEnd]==':' &&
- (iCol = checkColumnSpecifier(pQuery->pFts, pToken, nToken))>=0 ){
- pQuery->nextColumn = iCol;
- continue;
- }
- if( !inPhrase && pQuery->nTerms>0 && nToken==2
- && pSegment[iBegin+0]=='O'
- && pSegment[iBegin+1]=='R'
- ){
- pQuery->nextIsOr = 1;
- continue;
- }
- if( !inPhrase && pQuery->nTerms>0 && !pQuery->nextIsOr && nToken==4
- && pSegment[iBegin+0]=='N'
- && pSegment[iBegin+1]=='E'
- && pSegment[iBegin+2]=='A'
- && pSegment[iBegin+3]=='R'
- ){
- QueryTerm *pTerm = &pQuery->pTerms[pQuery->nTerms-1];
- if( (iBegin+6)<nSegment
- && pSegment[iBegin+4] == '/'
- && pSegment[iBegin+5]>='0' && pSegment[iBegin+5]<='9'
- ){
- pTerm->nNear = (pSegment[iBegin+5] - '0');
- nToken += 2;
- if( pSegment[iBegin+6]>='0' && pSegment[iBegin+6]<=9 ){
- pTerm->nNear = pTerm->nNear * 10 + (pSegment[iBegin+6] - '0');
- iEnd++;
- }
- pModule->xNext(pCursor, &pToken, &nToken, &iBegin, &iEnd, &iPos);
- } else {
- pTerm->nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
- }
- pTerm->nNear++;
- continue;
- }
- queryAdd(pQuery, pToken, nToken);
- if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){
- pQuery->pTerms[pQuery->nTerms-1].isNot = 1;
- }
- if( iEnd<nSegment && pSegment[iEnd]=='*' ){
- pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
- }
- pQuery->pTerms[pQuery->nTerms-1].iPhrase = nTerm;
- if( inPhrase ){
- nTerm++;
- }
- }
- if( inPhrase && pQuery->nTerms>firstIndex ){
- pQuery->pTerms[firstIndex].nPhrase = pQuery->nTerms - firstIndex - 1;
- }
- return pModule->xClose(pCursor);
- }
- /* Parse a query string, yielding a Query object pQuery.
- **
- ** The calling function will need to queryClear() to clean up
- ** the dynamically allocated memory held by pQuery.
- */
- static int parseQuery(
- fulltext_vtab *v, /* The fulltext index */
- const char *zInput, /* Input text of the query string */
- int nInput, /* Size of the input text */
- int dfltColumn, /* Default column of the index to match against */
- Query *pQuery /* Write the parse results here. */
- ){
- int iInput, inPhrase = 0;
- int ii;
- QueryTerm *aTerm;
- if( zInput==0 ) nInput = 0;
- if( nInput<0 ) nInput = strlen(zInput);
- pQuery->nTerms = 0;
- pQuery->pTerms = NULL;
- pQuery->nextIsOr = 0;
- pQuery->nextColumn = dfltColumn;
- pQuery->dfltColumn = dfltColumn;
- pQuery->pFts = v;
- for(iInput=0; iInput<nInput; ++iInput){
- int i;
- for(i=iInput; i<nInput && zInput[i]!='"'; ++i){}
- if( i>iInput ){
- tokenizeSegment(v->pTokenizer, zInput+iInput, i-iInput, inPhrase,
- pQuery);
- }
- iInput = i;
- if( i<nInput ){
- assert( zInput[i]=='"' );
- inPhrase = !inPhrase;
- }
- }
- if( inPhrase ){
- /* unmatched quote */
- queryClear(pQuery);
- return SQLITE_ERROR;
- }
- /* Modify the values of the QueryTerm.nPhrase variables to account for
- ** the NEAR operator. For the purposes of QueryTerm.nPhrase, phrases
- ** and tokens connected by the NEAR operator are handled as a single
- ** phrase. See comments above the QueryTerm structure for details.
- */
- aTerm = pQuery->pTerms;
- for(ii=0; ii<pQuery->nTerms; ii++){
- if( aTerm[ii].nNear || aTerm[ii].nPhrase ){
- while (aTerm[ii+aTerm[ii].nPhrase].nNear) {
- aTerm[ii].nPhrase += (1 + aTerm[ii+aTerm[ii].nPhrase+1].nPhrase);
- }
- }
- }
- return SQLITE_OK;
- }
- /* TODO(shess) Refactor the code to remove this forward decl. */
- static int flushPendingTerms(fulltext_vtab *v);
- /* Perform a full-text query using the search expression in
- ** zInput[0..nInput-1]. Return a list of matching documents
- ** in pResult.
- **
- ** Queries must match column iColumn. Or if iColumn>=nColumn
- ** they are allowed to match against any column.
- */
- static int fulltextQuery(
- fulltext_vtab *v, /* The full text index */
- int iColumn, /* Match against this column by default */
- const char *zInput, /* The query string */
- int nInput, /* Number of bytes in zInput[] */
- DataBuffer *pResult, /* Write the result doclist here */
- Query *pQuery /* Put parsed query string here */
- ){
- int i, iNext, rc;
- DataBuffer left, right, or, new;
- int nNot = 0;
- QueryTerm *aTerm;
- /* TODO(shess) Instead of flushing pendingTerms, we could query for
- ** the relevant term and merge the doclist into what we receive from
- ** the database. Wait and see if this is a common issue, first.
- **
- ** A good reason not to flush is to not generate update-related
- ** error codes from here.
- */
- /* Flush any buffered updates before executing the query. */
- rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ) return rc;
- /* TODO(shess) I think that the queryClear() calls below are not
- ** necessary, because fulltextClose() already clears the query.
- */
- rc = parseQuery(v, zInput, nInput, iColumn, pQuery);
- if( rc!=SQLITE_OK ) return rc;
- /* Empty or NULL queries return no results. */
- if( pQuery->nTerms==0 ){
- dataBufferInit(pResult, 0);
- return SQLITE_OK;
- }
- /* Merge AND terms. */
- /* TODO(shess) I think we can early-exit if( i>nNot && left.nData==0 ). */
- aTerm = pQuery->pTerms;
- for(i = 0; i<pQuery->nTerms; i=iNext){
- if( aTerm[i].isNot ){
- /* Handle all NOT terms in a separate pass */
- nNot++;
- iNext = i + aTerm[i].nPhrase+1;
- continue;
- }
- iNext = i + aTerm[i].nPhrase + 1;
- rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right);
- if( rc ){
- if( i!=nNot ) dataBufferDestroy(&left);
- queryClear(pQuery);
- return rc;
- }
- while( iNext<pQuery->nTerms && aTerm[iNext].isOr ){
- rc = docListOfTerm(v, aTerm[iNext].iColumn, &aTerm[iNext], &or);
- iNext += aTerm[iNext].nPhrase + 1;
- if( rc ){
- if( i!=nNot ) dataBufferDestroy(&left);
- dataBufferDestroy(&right);
- queryClear(pQuery);
- return rc;
- }
- dataBufferInit(&new, 0);
- docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new);
- dataBufferDestroy(&right);
- dataBufferDestroy(&or);
- right = new;
- }
- if( i==nNot ){ /* first term processed. */
- left = right;
- }else{
- dataBufferInit(&new, 0);
- docListAndMerge(left.pData, left.nData, right.pData, right.nData, &new);
- dataBufferDestroy(&right);
- dataBufferDestroy(&left);
- left = new;
- }
- }
- if( nNot==pQuery->nTerms ){
- /* We do not yet know how to handle a query of only NOT terms */
- return SQLITE_ERROR;
- }
- /* Do the EXCEPT terms */
- for(i=0; i<pQuery->nTerms; i += aTerm[i].nPhrase + 1){
- if( !aTerm[i].isNot ) continue;
- rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right);
- if( rc ){
- queryClear(pQuery);
- dataBufferDestroy(&left);
- return rc;
- }
- dataBufferInit(&new, 0);
- docListExceptMerge(left.pData, left.nData, right.pData, right.nData, &new);
- dataBufferDestroy(&right);
- dataBufferDestroy(&left);
- left = new;
- }
- *pResult = left;
- return rc;
- }
- /*
- ** This is the xFilter interface for the virtual table. See
- ** the virtual table xFilter method documentation for additional
- ** information.
- **
- ** If idxNum==QUERY_GENERIC then do a full table scan against
- ** the %_content table.
- **
- ** If idxNum==QUERY_DOCID then do a docid lookup for a single entry
- ** in the %_content table.
- **
- ** If idxNum>=QUERY_FULLTEXT then use the full text index. The
- ** column on the left-hand side of the MATCH operator is column
- ** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand
- ** side of the MATCH operator.
- */
- /* TODO(shess) Upgrade the cursor initialization and destruction to
- ** account for fulltextFilter() being called multiple times on the
- ** same cursor. The current solution is very fragile. Apply fix to
- ** fts3 as appropriate.
- */
- static int fulltextFilter(
- sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
- int idxNum, const char *idxStr, /* Which indexing scheme to use */
- int argc, sqlite3_value **argv /* Arguments for the indexing scheme */
- ){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- fulltext_vtab *v = cursor_vtab(c);
- int rc;
- StringBuffer sb;
- FTSTRACE(("FTS3 Filter %pn",pCursor));
- initStringBuffer(&sb);
- append(&sb, "SELECT docid, ");
- appendList(&sb, v->nColumn, v->azContentColumn);
- append(&sb, " FROM %_content");
- if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?");
- sqlite3_finalize(c->pStmt);
- rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, stringBufferData(&sb));
- stringBufferDestroy(&sb);
- if( rc!=SQLITE_OK ) return rc;
- c->iCursorType = idxNum;
- switch( idxNum ){
- case QUERY_GENERIC:
- break;
- case QUERY_DOCID:
- rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
- if( rc!=SQLITE_OK ) return rc;
- break;
- default: /* full-text search */
- {
- const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
- assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
- assert( argc==1 );
- queryClear(&c->q);
- if( c->result.nData!=0 ){
- /* This case happens if the same cursor is used repeatedly. */
- dlrDestroy(&c->reader);
- dataBufferReset(&c->result);
- }else{
- dataBufferInit(&c->result, 0);
- }
- rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &c->result, &c->q);
- if( rc!=SQLITE_OK ) return rc;
- if( c->result.nData!=0 ){
- dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
- }
- break;
- }
- }
- return fulltextNext(pCursor);
- }
- /* This is the xEof method of the virtual table. The SQLite core
- ** calls this routine to find out if it has reached the end of
- ** a query's results set.
- */
- static int fulltextEof(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- return c->eof;
- }
- /* This is the xColumn method of the virtual table. The SQLite
- ** core calls this method during a query when it needs the value
- ** of a column from the virtual table. This method needs to use
- ** one of the sqlite3_result_*() routines to store the requested
- ** value back in the pContext.
- */
- static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
- sqlite3_context *pContext, int idxCol){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- fulltext_vtab *v = cursor_vtab(c);
- if( idxCol<v->nColumn ){
- sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
- sqlite3_result_value(pContext, pVal);
- }else if( idxCol==v->nColumn ){
- /* The extra column whose name is the same as the table.
- ** Return a blob which is a pointer to the cursor
- */
- sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT);
- }else if( idxCol==v->nColumn+1 ){
- /* The docid column, which is an alias for rowid. */
- sqlite3_value *pVal = sqlite3_column_value(c->pStmt, 0);
- sqlite3_result_value(pContext, pVal);
- }
- return SQLITE_OK;
- }
- /* This is the xRowid method. The SQLite core calls this routine to
- ** retrieve the rowid for the current row of the result set. fts3
- ** exposes %_content.docid as the rowid for the virtual table. The
- ** rowid should be written to *pRowid.
- */
- static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- *pRowid = sqlite3_column_int64(c->pStmt, 0);
- return SQLITE_OK;
- }
- /* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0,
- ** we also store positions and offsets in the hash table using that
- ** column number.
- */
- static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
- const char *zText, int iColumn){
- sqlite3_tokenizer *pTokenizer = v->pTokenizer;
- sqlite3_tokenizer_cursor *pCursor;
- const char *pToken;
- int nTokenBytes;
- int iStartOffset, iEndOffset, iPosition;
- int rc;
- rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
- if( rc!=SQLITE_OK ) return rc;
- pCursor->pTokenizer = pTokenizer;
- while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
- &pToken, &nTokenBytes,
- &iStartOffset, &iEndOffset,
- &iPosition)) ){
- DLCollector *p;
- int nData; /* Size of doclist before our update. */
- /* Positions can't be negative; we use -1 as a terminator
- * internally. Token can't be NULL or empty. */
- if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){
- rc = SQLITE_ERROR;
- break;
- }
- p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes);
- if( p==NULL ){
- nData = 0;
- p = dlcNew(iDocid, DL_DEFAULT);
- fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
- /* Overhead for our hash table entry, the key, and the value. */
- v->nPendingData += sizeof(struct fts3HashElem)+sizeof(*p)+nTokenBytes;
- }else{
- nData = p->b.nData;
- if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid);
- }
- if( iColumn>=0 ){
- dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
- }
- /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
- v->nPendingData += p->b.nData-nData;
- }
- /* TODO(shess) Check return? Should this be able to cause errors at
- ** this point? Actually, same question about sqlite3_finalize(),
- ** though one could argue that failure there means that the data is
- ** not durable. *ponder*
- */
- pTokenizer->pModule->xClose(pCursor);
- if( SQLITE_DONE == rc ) return SQLITE_OK;
- return rc;
- }
- /* Add doclists for all terms in [pValues] to pendingTerms table. */
- static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid,
- sqlite3_value **pValues){
- int i;
- for(i = 0; i < v->nColumn ; ++i){
- char *zText = (char*)sqlite3_value_text(pValues[i]);
- int rc = buildTerms(v, iDocid, zText, i);
- if( rc!=SQLITE_OK ) return rc;
- }
- return SQLITE_OK;
- }
- /* Add empty doclists for all terms in the given row's content to
- ** pendingTerms.
- */
- static int deleteTerms(fulltext_vtab *v, sqlite_int64 iDocid){
- const char **pValues;
- int i, rc;
- /* TODO(shess) Should we allow such tables at all? */
- if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;
- rc = content_select(v, iDocid, &pValues);
- if( rc!=SQLITE_OK ) return rc;
- for(i = 0 ; i < v->nColumn; ++i) {
- rc = buildTerms(v, iDocid, pValues[i], -1);
- if( rc!=SQLITE_OK ) break;
- }
- freeStringArray(v->nColumn, pValues);
- return SQLITE_OK;
- }
- /* TODO(shess) Refactor the code to remove this forward decl. */
- static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);
- /* Insert a row into the %_content table; set *piDocid to be the ID of the
- ** new row. Add doclists for terms to pendingTerms.
- */
- static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestDocid,
- sqlite3_value **pValues, sqlite_int64 *piDocid){
- int rc;
- rc = content_insert(v, pRequestDocid, pValues); /* execute an SQL INSERT */
- if( rc!=SQLITE_OK ) return rc;
- /* docid column is an alias for rowid. */
- *piDocid = sqlite3_last_insert_rowid(v->db);
- rc = initPendingTerms(v, *piDocid);
- if( rc!=SQLITE_OK ) return rc;
- return insertTerms(v, *piDocid, pValues);
- }
- /* Delete a row from the %_content table; add empty doclists for terms
- ** to pendingTerms.
- */
- static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
- int rc = initPendingTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
- rc = deleteTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
- return content_delete(v, iRow); /* execute an SQL DELETE */
- }
- /* Update a row in the %_content table; add delete doclists to
- ** pendingTerms for old terms not in the new data, add insert doclists
- ** to pendingTerms for terms in the new data.
- */
- static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
- sqlite3_value **pValues){
- int rc = initPendingTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
- /* Generate an empty doclist for each term that previously appeared in this
- * row. */
- rc = deleteTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
- rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */
- if( rc!=SQLITE_OK ) return rc;
- /* Now add positions for terms which appear in the updated row. */
- return insertTerms(v, iRow, pValues);
- }
- /*******************************************************************/
- /* InteriorWriter is used to collect terms and block references into
- ** interior nodes in %_segments. See commentary at top of file for
- ** format.
- */
- /* How large interior nodes can grow. */
- #define INTERIOR_MAX 2048
- /* Minimum number of terms per interior node (except the root). This
- ** prevents large terms from making the tree too skinny - must be >0
- ** so that the tree always makes progress. Note that the min tree
- ** fanout will be INTERIOR_MIN_TERMS+1.
- */
- #define INTERIOR_MIN_TERMS 7
- #if INTERIOR_MIN_TERMS<1
- # error INTERIOR_MIN_TERMS must be greater than 0.
- #endif
- /* ROOT_MAX controls how much data is stored inline in the segment
- ** directory.
- */
- /* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's
- ** only here so that interiorWriterRootInfo() and leafWriterRootInfo()
- ** can both see it, but if the caller passed it in, we wouldn't even
- ** need a define.
- */
- #define ROOT_MAX 1024
- #if ROOT_MAX<VARINT_MAX*2
- # error ROOT_MAX must have enough space for a header.
- #endif
- /* InteriorBlock stores a linked-list of interior blocks while a lower
- ** layer is being constructed.
- */
- typedef struct InteriorBlock {
- DataBuffer term; /* Leftmost term in block's subtree. */
- DataBuffer data; /* Accumulated data for the block. */
- struct InteriorBlock *next;
- } InteriorBlock;
- static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock,
- const char *pTerm, int nTerm){
- InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock));
- char c[VARINT_MAX+VARINT_MAX];
- int n;
- if( block ){
- memset(block, 0, sizeof(*block));
- dataBufferInit(&block->term, 0);
- dataBufferReplace(&block->term, pTerm, nTerm);
- n = fts3PutVarint(c, iHeight);
- n += fts3PutVarint(c+n, iChildBlock);
- dataBufferInit(&block->data, INTERIOR_MAX);
- dataBufferReplace(&block->data, c, n);
- }
- return block;
- }
- #ifndef NDEBUG
- /* Verify that the data is readable as an interior node. */
- static void interiorBlockValidate(InteriorBlock *pBlock){
- const char *pData = pBlock->data.pData;
- int nData = pBlock->data.nData;
- int n, iDummy;
- sqlite_int64 iBlockid;
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
- /* Must lead with height of node as a varint(n), n>0 */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n<nData );
- pData += n;
- nData -= n;
- /* Must contain iBlockid. */
- n = fts3GetVarint(pData, &iBlockid);
- assert( n>0 );
- assert( n<=nData );
- pData += n;
- nData -= n;
- /* Zero or more terms of positive length */
- if( nData!=0 ){
- /* First term is not delta-encoded. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0);
- assert( n+iDummy<=nData );
- pData += n+iDummy;
- nData -= n+iDummy;
- /* Following terms delta-encoded. */
- while( nData!=0 ){
- /* Length of shared prefix. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>=0 );
- assert( n<nData );
- pData += n;
- nData -= n;
- /* Length and data of distinct suffix. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0);
- assert( n+iDummy<=nData );
- pData += n+iDummy;
- nData -= n+iDummy;
- }
- }
- }
- #define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x)
- #else
- #define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 )
- #endif
- typedef struct InteriorWriter {
- int iHeight; /* from 0 at leaves. */
- InteriorBlock *first, *last;
- struct InteriorWriter *parentWriter;
- DataBuffer term; /* Last term written to block "last". */
- sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */
- #ifndef NDEBUG
- sqlite_int64 iLastChildBlock; /* for consistency checks. */
- #endif
- } InteriorWriter;
- /* Initialize an interior node where pTerm[nTerm] marks the leftmost
- ** term in the tree. iChildBlock is the leftmost child block at the
- ** next level down the tree.
- */
- static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm,
- sqlite_int64 iChildBlock,
- InteriorWriter *pWriter){
- InteriorBlock *block;
- assert( iHeight>0 );
- CLEAR(pWriter);
- pWriter->iHeight = iHeight;
- pWriter->iOpeningChildBlock = iChildBlock;
- #ifndef NDEBUG
- pWriter->iLastChildBlock = iChildBlock;
- #endif
- block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm);
- pWriter->last = pWriter->first = block;
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
- dataBufferInit(&pWriter->term, 0);
- }
- /* Append the child node rooted at iChildBlock to the interior node,
- ** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree.
- */
- static void interiorWriterAppend(InteriorWriter *pWriter,
- const char *pTerm, int nTerm,
- sqlite_int64 iChildBlock){
- char c[VARINT_MAX+VARINT_MAX];
- int n, nPrefix = 0;
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
- /* The first term written into an interior node is actually
- ** associated with the second child added (the first child was added
- ** in interiorWriterInit, or in the if clause at the bottom of this
- ** function). That term gets encoded straight up, with nPrefix left
- ** at 0.
- */
- if( pWriter->term.nData==0 ){
- n = fts3PutVarint(c, nTerm);
- }else{
- while( nPrefix<pWriter->term.nData &&
- pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
- nPrefix++;
- }
- n = fts3PutVarint(c, nPrefix);
- n += fts3PutVarint(c+n, nTerm-nPrefix);
- }
- #ifndef NDEBUG
- pWriter->iLastChildBlock++;
- #endif
- assert( pWriter->iLastChildBlock==iChildBlock );
- /* Overflow to a new block if the new term makes the current block
- ** too big, and the current block already has enough terms.
- */
- if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX &&
- iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){
- pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock,
- pTerm, nTerm);
- pWriter->last = pWriter->last->next;
- pWriter->iOpeningChildBlock = iChildBlock;
- dataBufferReset(&pWriter->term);
- }else{
- dataBufferAppend2(&pWriter->last->data, c, n,
- pTerm+nPrefix, nTerm-nPrefix);
- dataBufferReplace(&pWriter->term, pTerm, nTerm);
- }
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
- }
- /* Free the space used by pWriter, including the linked-list of
- ** InteriorBlocks, and parentWriter, if present.
- */
- static int interiorWriterDestroy(InteriorWriter *pWriter){
- InteriorBlock *block = pWriter->first;
- while( block!=NULL ){
- InteriorBlock *b = block;
- block = block->next;
- dataBufferDestroy(&b->term);
- dataBufferDestroy(&b->data);
- sqlite3_free(b);
- }
- if( pWriter->parentWriter!=NULL ){
- interiorWriterDestroy(pWriter->parentWriter);
- sqlite3_free(pWriter->parentWriter);
- }
- dataBufferDestroy(&pWriter->term);
- SCRAMBLE(pWriter);
- return SQLITE_OK;
- }
- /* If pWriter can fit entirely in ROOT_MAX, return it as the root info
- ** directly, leaving *piEndBlockid unchanged. Otherwise, flush
- ** pWriter to %_segments, building a new layer of interior nodes, and
- ** recursively ask for their root into.
- */
- static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter,
- char **ppRootInfo, int *pnRootInfo,
- sqlite_int64 *piEndBlockid){
- InteriorBlock *block = pWriter->first;
- sqlite_int64 iBlockid = 0;
- int rc;
- /* If we can fit the segment inline */
- if( block==pWriter->last && block->data.nData<ROOT_MAX ){
- *ppRootInfo = block->data.pData;
- *pnRootInfo = block->data.nData;
- return SQLITE_OK;
- }
- /* Flush the first block to %_segments, and create a new level of
- ** interior node.
- */
- ASSERT_VALID_INTERIOR_BLOCK(block);
- rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- *piEndBlockid = iBlockid;
- pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter));
- interiorWriterInit(pWriter->iHeight+1,
- block->term.pData, block->term.nData,
- iBlockid, pWriter->parentWriter);
- /* Flush additional blocks and append to the higher interior
- ** node.
- */
- for(block=block->next; block!=NULL; block=block->next){
- ASSERT_VALID_INTERIOR_BLOCK(block);
- rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- *piEndBlockid = iBlockid;
- interiorWriterAppend(pWriter->parentWriter,
- block->term.pData, block->term.nData, iBlockid);
- }
- /* Parent node gets the chance to be the root. */
- return interiorWriterRootInfo(v, pWriter->parentWriter,
- ppRootInfo, pnRootInfo, piEndBlockid);
- }
- /****************************************************************/
- /* InteriorReader is used to read off the data from an interior node
- ** (see comment at top of file for the format).
- */
- typedef struct InteriorReader {
- const char *pData;
- int nData;
- DataBuffer term; /* previous term, for decoding term delta. */
- sqlite_int64 iBlockid;
- } InteriorReader;
- static void interiorReaderDestroy(InteriorReader *pReader){
- dataBufferDestroy(&pReader->term);
- SCRAMBLE(pReader);
- }
- /* TODO(shess) The assertions are great, but what if we're in NDEBUG
- ** and the blob is empty or otherwise contains suspect data?
- */
- static void interiorReaderInit(const char *pData, int nData,
- InteriorReader *pReader){
- int n, nTerm;
- /* Require at least the leading flag byte */
- assert( nData>0 );
- assert( pData[0]!=' ' );
- CLEAR(pReader);
- /* Decode the base blockid, and set the cursor to the first term. */
- n = fts3GetVarint(pData+1, &pReader->iBlockid);
- assert( 1+n<=nData );
- pReader->pData = pData+1+n;
- pReader->nData = nData-(1+n);
- /* A single-child interior node (such as when a leaf node was too
- ** large for the segment directory) won't have any terms.
- ** Otherwise, decode the first term.
- */
- if( pReader->nData==0 ){
- dataBufferInit(&pReader->term, 0);
- }else{
- n = fts3GetVarint32(pReader->pData, &nTerm);
- dataBufferInit(&pReader->term, nTerm);
- dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
- assert( n+nTerm<=pReader->nData );
- pReader->pData += n+nTerm;
- pReader->nData -= n+nTerm;
- }
- }
- static int interiorReaderAtEnd(InteriorReader *pReader){
- return pReader->term.nData==0;
- }
- static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
- return pReader->iBlockid;
- }
- static int interiorReaderTermBytes(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
- return pReader->term.nData;
- }
- static const char *interiorReaderTerm(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
- return pReader->term.pData;
- }
- /* Step forward to the next term in the node. */
- static void interiorReaderStep(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
- /* If the last term has been read, signal eof, else construct the
- ** next term.
- */
- if( pReader->nData==0 ){
- dataBufferReset(&pReader->term);
- }else{
- int n, nPrefix, nSuffix;
- n = fts3GetVarint32(pReader->pData, &nPrefix);
- n += fts3GetVarint32(pReader->pData+n, &nSuffix);
- /* Truncate the current term and append suffix data. */
- pReader->term.nData = nPrefix;
- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
- assert( n+nSuffix<=pReader->nData );
- pReader->pData += n+nSuffix;
- pReader->nData -= n+nSuffix;
- }
- pReader->iBlockid++;
- }
- /* Compare the current term to pTerm[nTerm], returning strcmp-style
- ** results. If isPrefix, equality means equal through nTerm bytes.
- */
- static int interiorReaderTermCmp(InteriorReader *pReader,
- const char *pTerm, int nTerm, int isPrefix){
- const char *pReaderTerm = interiorReaderTerm(pReader);
- int nReaderTerm = interiorReaderTermBytes(pReader);
- int c, n = nReaderTerm<nTerm ? nReaderTerm : nTerm;
- if( n==0 ){
- if( nReaderTerm>0 ) return -1;
- if( nTerm>0 ) return 1;
- return 0;
- }
- c = memcmp(pReaderTerm, pTerm, n);
- if( c!=0 ) return c;
- if( isPrefix && n==nTerm ) return 0;
- return nReaderTerm - nTerm;
- }
- /****************************************************************/
- /* LeafWriter is used to collect terms and associated doclist data
- ** into leaf blocks in %_segments (see top of file for format info).
- ** Expected usage is:
- **
- ** LeafWriter writer;
- ** leafWriterInit(0, 0, &writer);
- ** while( sorted_terms_left_to_process ){
- ** // data is doclist data for that term.
- ** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData);
- ** if( rc!=SQLITE_OK ) goto err;
- ** }
- ** rc = leafWriterFinalize(v, &writer);
- **err:
- ** leafWriterDestroy(&writer);
- ** return rc;
- **
- ** leafWriterStep() may write a collected leaf out to %_segments.
- ** leafWriterFinalize() finishes writing any buffered data and stores
- ** a root node in %_segdir. leafWriterDestroy() frees all buffers and
- ** InteriorWriters allocated as part of writing this segment.
- **
- ** TODO(shess) Document leafWriterStepMerge().
- */
- /* Put terms with data this big in their own block. */
- #define STANDALONE_MIN 1024
- /* Keep leaf blocks below this size. */
- #define LEAF_MAX 2048
- typedef struct LeafWriter {
- int iLevel;
- int idx;
- sqlite_int64 iStartBlockid; /* needed to create the root info */
- sqlite_int64 iEndBlockid; /* when we're done writing. */
- DataBuffer term; /* previous encoded term */
- DataBuffer data; /* encoding buffer */
- /* bytes of first term in the current node which distinguishes that
- ** term from the last term of the previous node.
- */
- int nTermDistinct;
- InteriorWriter parentWriter; /* if we overflow */
- int has_parent;
- } LeafWriter;
- static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){
- CLEAR(pWriter);
- pWriter->iLevel = iLevel;
- pWriter->idx = idx;
- dataBufferInit(&pWriter->term, 32);
- /* Start out with a reasonably sized block, though it can grow. */
- dataBufferInit(&pWriter->data, LEAF_MAX);
- }
- #ifndef NDEBUG
- /* Verify that the data is readable as a leaf node. */
- static void leafNodeValidate(const char *pData, int nData){
- int n, iDummy;
- if( nData==0 ) return;
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
- /* Must lead with a varint(0) */
- n = fts3GetVarint32(pData, &iDummy);
- assert( iDummy==0 );
- assert( n>0 );
- assert( n<nData );
- pData += n;
- nData -= n;
- /* Leading term length and data must fit in buffer. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<nData );
- pData += n+iDummy;
- nData -= n+iDummy;
- /* Leading term's doclist length and data must fit. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<=nData );
- ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
- pData += n+iDummy;
- nData -= n+iDummy;
- /* Verify that trailing terms and doclists also are readable. */
- while( nData!=0 ){
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>=0 );
- assert( n<nData );
- pData += n;
- nData -= n;
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<nData );
- pData += n+iDummy;
- nData -= n+iDummy;
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<=nData );
- ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
- pData += n+iDummy;
- nData -= n+iDummy;
- }
- }
- #define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n)
- #else
- #define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 )
- #endif
- /* Flush the current leaf node to %_segments, and adding the resulting
- ** blockid and the starting term to the interior node which will
- ** contain it.
- */
- static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter,
- int iData, int nData){
- sqlite_int64 iBlockid = 0;
- const char *pStartingTerm;
- int nStartingTerm, rc, n;
- /* Must have the leading varint(0) flag, plus at least some
- ** valid-looking data.
- */
- assert( nData>2 );
- assert( iData>=0 );
- assert( iData+nData<=pWriter->data.nData );
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData);
- rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- assert( iBlockid!=0 );
- /* Reconstruct the first term in the leaf for purposes of building
- ** the interior node.
- */
- n = fts3GetVarint32(pWriter->data.pData+iData+1, &nStartingTerm);
- pStartingTerm = pWriter->data.pData+iData+1+n;
- assert( pWriter->data.nData>iData+1+n+nStartingTerm );
- assert( pWriter->nTermDistinct>0 );
- assert( pWriter->nTermDistinct<=nStartingTerm );
- nStartingTerm = pWriter->nTermDistinct;
- if( pWriter->has_parent ){
- interiorWriterAppend(&pWriter->parentWriter,
- pStartingTerm, nStartingTerm, iBlockid);
- }else{
- interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid,
- &pWriter->parentWriter);
- pWriter->has_parent = 1;
- }
- /* Track the span of this segment's leaf nodes. */
- if( pWriter->iEndBlockid==0 ){
- pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid;
- }else{
- pWriter->iEndBlockid++;
- assert( iBlockid==pWriter->iEndBlockid );
- }
- return SQLITE_OK;
- }
- static int leafWriterFlush(fulltext_vtab *v, LeafWriter *pWriter){
- int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData);
- if( rc!=SQLITE_OK ) return rc;
- /* Re-initialize the output buffer. */
- dataBufferReset(&pWriter->data);
- return SQLITE_OK;
- }
- /* Fetch the root info for the segment. If the entire leaf fits
- ** within ROOT_MAX, then it will be returned directly, otherwise it
- ** will be flushed and the root info will be returned from the
- ** interior node. *piEndBlockid is set to the blockid of the last
- ** interior or leaf node written to disk (0 if none are written at
- ** all).
- */
- static int leafWriterRootInfo(fulltext_vtab *v, LeafWriter *pWriter,
- char **ppRootInfo, int *pnRootInfo,
- sqlite_int64 *piEndBlockid){
- /* we can fit the segment entirely inline */
- if( !pWriter->has_parent && pWriter->data.nData<ROOT_MAX ){
- *ppRootInfo = pWriter->data.pData;
- *pnRootInfo = pWriter->data.nData;
- *piEndBlockid = 0;
- return SQLITE_OK;
- }
- /* Flush remaining leaf data. */
- if( pWriter->data.nData>0 ){
- int rc = leafWriterFlush(v, pWriter);
- if( rc!=SQLITE_OK ) return rc;
- }
- /* We must have flushed a leaf at some point. */
- assert( pWriter->has_parent );
- /* Tenatively set the end leaf blockid as the end blockid. If the
- ** interior node can be returned inline, this will be the final
- ** blockid, otherwise it will be overwritten by
- ** interiorWriterRootInfo().
- */
- *piEndBlockid = pWriter->iEndBlockid;
- return interiorWriterRootInfo(v, &pWriter->parentWriter,
- ppRootInfo, pnRootInfo, piEndBlockid);
- }
- /* Collect the rootInfo data and store it into the segment directory.
- ** This has the effect of flushing the segment's leaf data to
- ** %_segments, and also flushing any interior nodes to %_segments.
- */
- static int leafWriterFinalize(fulltext_vtab *v, LeafWriter *pWriter){
- sqlite_int64 iEndBlockid;
- char *pRootInfo;
- int rc, nRootInfo;
- rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
- /* Don't bother storing an entirely empty segment. */
- if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK;
- return segdir_set(v, pWriter->iLevel, pWriter->idx,
- pWriter->iStartBlockid, pWriter->iEndBlockid,
- iEndBlockid, pRootInfo, nRootInfo);
- }
- static void leafWriterDestroy(LeafWriter *pWriter){
- if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter);
- dataBufferDestroy(&pWriter->term);
- dataBufferDestroy(&pWriter->data);
- }
- /* Encode a term into the leafWriter, delta-encoding as appropriate.
- ** Returns the length of the new term which distinguishes it from the
- ** previous term, which can be used to set nTermDistinct when a node
- ** boundary is crossed.
- */
- static int leafWriterEncodeTerm(LeafWriter *pWriter,
- const char *pTerm, int nTerm){
- char c[VARINT_MAX+VARINT_MAX];
- int n, nPrefix = 0;
- assert( nTerm>0 );
- while( nPrefix<pWriter->term.nData &&
- pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
- nPrefix++;
- /* Failing this implies that the terms weren't in order. */
- assert( nPrefix<nTerm );
- }
- if( pWriter->data.nData==0 ){
- /* Encode the node header and leading term as:
- ** varint(0)
- ** varint(nTerm)
- ** char pTerm[nTerm]
- */
- n = fts3PutVarint(c, ' ');
- n += fts3PutVarint(c+n, nTerm);
- dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm);
- }else{
- /* Delta-encode the term as:
- ** varint(nPrefix)
- ** varint(nSuffix)
- ** char pTermSuffix[nSuffix]
- */
- n = fts3PutVarint(c, nPrefix);
- n += fts3PutVarint(c+n, nTerm-nPrefix);
- dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix);
- }
- dataBufferReplace(&pWriter->term, pTerm, nTerm);
- return nPrefix+1;
- }
- /* Used to avoid a memmove when a large amount of doclist data is in
- ** the buffer. This constructs a node and term header before
- ** iDoclistData and flushes the resulting complete node using
- ** leafWriterInternalFlush().
- */
- static int leafWriterInlineFlush(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- int iDoclistData){
- char c[VARINT_MAX+VARINT_MAX];
- int iData, n = fts3PutVarint(c, 0);
- n += fts3PutVarint(c+n, nTerm);
- /* There should always be room for the header. Even if pTerm shared
- ** a substantial prefix with the previous term, the entire prefix
- ** could be constructed from earlier data in the doclist, so there
- ** should be room.
- */
- assert( iDoclistData>=n+nTerm );
- iData = iDoclistData-(n+nTerm);
- memcpy(pWriter->data.pData+iData, c, n);
- memcpy(pWriter->data.pData+iData+n, pTerm, nTerm);
- return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData);
- }
- /* Push pTerm[nTerm] along with the doclist data to the leaf layer of
- ** %_segments.
- */
- static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- DLReader *pReaders, int nReaders){
- char c[VARINT_MAX+VARINT_MAX];
- int iTermData = pWriter->data.nData, iDoclistData;
- int i, nData, n, nActualData, nActual, rc, nTermDistinct;
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
- nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm);
- /* Remember nTermDistinct if opening a new node. */
- if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct;
- iDoclistData = pWriter->data.nData;
- /* Estimate the length of the merged doclist so we can leave space
- ** to encode it.
- */
- for(i=0, nData=0; i<nReaders; i++){
- nData += dlrAllDataBytes(&pReaders[i]);
- }
- n = fts3PutVarint(c, nData);
- dataBufferAppend(&pWriter->data, c, n);
- docListMerge(&pWriter->data, pReaders, nReaders);
- ASSERT_VALID_DOCLIST(DL_DEFAULT,
- pWriter->data.pData+iDoclistData+n,
- pWriter->data.nData-iDoclistData-n, NULL);
- /* The actual amount of doclist data at this point could be smaller
- ** than the length we encoded. Additionally, the space required to
- ** encode this length could be smaller. For small doclists, this is
- ** not a big deal, we can just use memmove() to adjust things.
- */
- nActualData = pWriter->data.nData-(iDoclistData+n);
- nActual = fts3PutVarint(c, nActualData);
- assert( nActualData<=nData );
- assert( nActual<=n );
- /* If the new doclist is big enough for force a standalone leaf
- ** node, we can immediately flush it inline without doing the
- ** memmove().
- */
- /* TODO(shess) This test matches leafWriterStep(), which does this
- ** test before it knows the cost to varint-encode the term and
- ** doclist lengths. At some point, change to
- ** pWriter->data.nData-iTermData>STANDALONE_MIN.
- */
- if( nTerm+nActualData>STANDALONE_MIN ){
- /* Push leaf node from before this term. */
- if( iTermData>0 ){
- rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
- if( rc!=SQLITE_OK ) return rc;
- pWriter->nTermDistinct = nTermDistinct;
- }
- /* Fix the encoded doclist length. */
- iDoclistData += n - nActual;
- memcpy(pWriter->data.pData+iDoclistData, c, nActual);
- /* Push the standalone leaf node. */
- rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData);
- if( rc!=SQLITE_OK ) return rc;
- /* Leave the node empty. */
- dataBufferReset(&pWriter->data);
- return rc;
- }
- /* At this point, we know that the doclist was small, so do the
- ** memmove if indicated.
- */
- if( nActual<n ){
- memmove(pWriter->data.pData+iDoclistData+nActual,
- pWriter->data.pData+iDoclistData+n,
- pWriter->data.nData-(iDoclistData+n));
- pWriter->data.nData -= n-nActual;
- }
- /* Replace written length with actual length. */
- memcpy(pWriter->data.pData+iDoclistData, c, nActual);
- /* If the node is too large, break things up. */
- /* TODO(shess) This test matches leafWriterStep(), which does this
- ** test before it knows the cost to varint-encode the term and
- ** doclist lengths. At some point, change to
- ** pWriter->data.nData>LEAF_MAX.
- */
- if( iTermData+nTerm+nActualData>LEAF_MAX ){
- /* Flush out the leading data as a node */
- rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
- if( rc!=SQLITE_OK ) return rc;
- pWriter->nTermDistinct = nTermDistinct;
- /* Rebuild header using the current term */
- n = fts3PutVarint(pWriter->data.pData, 0);
- n += fts3PutVarint(pWriter->data.pData+n, nTerm);
- memcpy(pWriter->data.pData+n, pTerm, nTerm);
- n += nTerm;
- /* There should always be room, because the previous encoding
- ** included all data necessary to construct the term.
- */
- assert( n<iDoclistData );
- /* So long as STANDALONE_MIN is half or less of LEAF_MAX, the
- ** following memcpy() is safe (as opposed to needing a memmove).
- */
- assert( 2*STANDALONE_MIN<=LEAF_MAX );
- assert( n+pWriter->data.nData-iDoclistData<iDoclistData );
- memcpy(pWriter->data.pData+n,
- pWriter->data.pData+iDoclistData,
- pWriter->data.nData-iDoclistData);
- pWriter->data.nData -= iDoclistData-n;
- }
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
- return SQLITE_OK;
- }
- /* Push pTerm[nTerm] along with the doclist data to the leaf layer of
- ** %_segments.
- */
- /* TODO(shess) Revise writeZeroSegment() so that doclists are
- ** constructed directly in pWriter->data.
- */
- static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- const char *pData, int nData){
- int rc;
- DLReader reader;
- dlrInit(&reader, DL_DEFAULT, pData, nData);
- rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
- dlrDestroy(&reader);
- return rc;
- }
- /****************************************************************/
- /* LeafReader is used to iterate over an individual leaf node. */
- typedef struct LeafReader {
- DataBuffer term; /* copy of current term. */
- const char *pData; /* data for current term. */
- int nData;
- } LeafReader;
- static void leafReaderDestroy(LeafReader *pReader){
- dataBufferDestroy(&pReader->term);
- SCRAMBLE(pReader);
- }
- static int leafReaderAtEnd(LeafReader *pReader){
- return pReader->nData<=0;
- }
- /* Access the current term. */
- static int leafReaderTermBytes(LeafReader *pReader){
- return pReader->term.nData;
- }
- static const char *leafReaderTerm(LeafReader *pReader){
- assert( pReader->term.nData>0 );
- return pReader->term.pData;
- }
- /* Access the doclist data for the current term. */
- static int leafReaderDataBytes(LeafReader *pReader){
- int nData;
- assert( pReader->term.nData>0 );
- fts3GetVarint32(pReader->pData, &nData);
- return nData;
- }
- static const char *leafReaderData(LeafReader *pReader){
- int n, nData;
- assert( pReader->term.nData>0 );
- n = fts3GetVarint32(pReader->pData, &nData);
- return pReader->pData+n;
- }
- static void leafReaderInit(const char *pData, int nData,
- LeafReader *pReader){
- int nTerm, n;
- assert( nData>0 );
- assert( pData[0]==' ' );
- CLEAR(pReader);
- /* Read the first term, skipping the header byte. */
- n = fts3GetVarint32(pData+1, &nTerm);
- dataBufferInit(&pReader->term, nTerm);
- dataBufferReplace(&pReader->term, pData+1+n, nTerm);
- /* Position after the first term. */
- assert( 1+n+nTerm<nData );
- pReader->pData = pData+1+n+nTerm;
- pReader->nData = nData-1-n-nTerm;
- }
- /* Step the reader forward to the next term. */
- static void leafReaderStep(LeafReader *pReader){
- int n, nData, nPrefix, nSuffix;
- assert( !leafReaderAtEnd(pReader) );
- /* Skip previous entry's data block. */
- n = fts3GetVarint32(pReader->pData, &nData);
- assert( n+nData<=pReader->nData );
- pReader->pData += n+nData;
- pReader->nData -= n+nData;
- if( !leafReaderAtEnd(pReader) ){
- /* Construct the new term using a prefix from the old term plus a
- ** suffix from the leaf data.
- */
- n = fts3GetVarint32(pReader->pData, &nPrefix);
- n += fts3GetVarint32(pReader->pData+n, &nSuffix);
- assert( n+nSuffix<pReader->nData );
- pReader->term.nData = nPrefix;
- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
- pReader->pData += n+nSuffix;
- pReader->nData -= n+nSuffix;
- }
- }
- /* strcmp-style comparison of pReader's current term against pTerm.
- ** If isPrefix, equality means equal through nTerm bytes.
- */
- static int leafReaderTermCmp(LeafReader *pReader,
- const char *pTerm, int nTerm, int isPrefix){
- int c, n = pReader->term.nData<nTerm ? pReader->term.nData : nTerm;
- if( n==0 ){
- if( pReader->term.nData>0 ) return -1;
- if(nTerm>0 ) return 1;
- return 0;
- }
- c = memcmp(pReader->term.pData, pTerm, n);
- if( c!=0 ) return c;
- if( isPrefix && n==nTerm ) return 0;
- return pReader->term.nData - nTerm;
- }
- /****************************************************************/
- /* LeavesReader wraps LeafReader to allow iterating over the entire
- ** leaf layer of the tree.
- */
- typedef struct LeavesReader {
- int idx; /* Index within the segment. */
- sqlite3_stmt *pStmt; /* Statement we're streaming leaves from. */
- int eof; /* we've seen SQLITE_DONE from pStmt. */
- LeafReader leafReader; /* reader for the current leaf. */
- DataBuffer rootData; /* root data for inline. */
- } LeavesReader;
- /* Access the current term. */
- static int leavesReaderTermBytes(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderTermBytes(&pReader->leafReader);
- }
- static const char *leavesReaderTerm(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderTerm(&pReader->leafReader);
- }
- /* Access the doclist data for the current term. */
- static int leavesReaderDataBytes(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderDataBytes(&pReader->leafReader);
- }
- static const char *leavesReaderData(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderData(&pReader->leafReader);
- }
- static int leavesReaderAtEnd(LeavesReader *pReader){
- return pReader->eof;
- }
- /* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus
- ** leaving the statement handle open, which locks the table.
- */
- /* TODO(shess) This "solution" is not satisfactory. Really, there
- ** should be check-in function for all statement handles which
- ** arranges to call sqlite3_reset(). This most likely will require
- ** modification to control flow all over the place, though, so for now
- ** just punt.
- **
- ** Note the the current system assumes that segment merges will run to
- ** completion, which is why this particular probably hasn't arisen in
- ** this case. Probably a brittle assumption.
- */
- static int leavesReaderReset(LeavesReader *pReader){
- return sqlite3_reset(pReader->pStmt);
- }
- static void leavesReaderDestroy(LeavesReader *pReader){
- leafReaderDestroy(&pReader->leafReader);
- dataBufferDestroy(&pReader->rootData);
- SCRAMBLE(pReader);
- }
- /* Initialize pReader with the given root data (if iStartBlockid==0
- ** the leaf data was entirely contained in the root), or from the
- ** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
- */
- static int leavesReaderInit(fulltext_vtab *v,
- int idx,
- sqlite_int64 iStartBlockid,
- sqlite_int64 iEndBlockid,
- const char *pRootData, int nRootData,
- LeavesReader *pReader){
- CLEAR(pReader);
- pReader->idx = idx;
- dataBufferInit(&pReader->rootData, 0);
- if( iStartBlockid==0 ){
- /* Entire leaf level fit in root data. */
- dataBufferReplace(&pReader->rootData, pRootData, nRootData);
- leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
- &pReader->leafReader);
- }else{
- sqlite3_stmt *s;
- int rc = sql_get_leaf_statement(v, idx, &s);
- if( rc!=SQLITE_OK ) return rc;
- rc = sqlite3_bind_int64(s, 1, iStartBlockid);
- if( rc!=SQLITE_OK ) return rc;
- rc = sqlite3_bind_int64(s, 2, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ){
- pReader->eof = 1;
- return SQLITE_OK;
- }
- if( rc!=SQLITE_ROW ) return rc;
- pReader->pStmt = s;
- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
- sqlite3_column_bytes(pReader->pStmt, 0),
- &pReader->leafReader);
- }
- return SQLITE_OK;
- }
- /* Step the current leaf forward to the next term. If we reach the
- ** end of the current leaf, step forward to the next leaf block.
- */
- static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
- assert( !leavesReaderAtEnd(pReader) );
- leafReaderStep(&pReader->leafReader);
- if( leafReaderAtEnd(&pReader->leafReader) ){
- int rc;
- if( pReader->rootData.pData ){
- pReader->eof = 1;
- return SQLITE_OK;
- }
- rc = sqlite3_step(pReader->pStmt);
- if( rc!=SQLITE_ROW ){
- pReader->eof = 1;
- return rc==SQLITE_DONE ? SQLITE_OK : rc;
- }
- leafReaderDestroy(&pReader->leafReader);
- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
- sqlite3_column_bytes(pReader->pStmt, 0),
- &pReader->leafReader);
- }
- return SQLITE_OK;
- }
- /* Order LeavesReaders by their term, ignoring idx. Readers at eof
- ** always sort to the end.
- */
- static int leavesReaderTermCmp(LeavesReader *lr1, LeavesReader *lr2){
- if( leavesReaderAtEnd(lr1) ){
- if( leavesReaderAtEnd(lr2) ) return 0;
- return 1;
- }
- if( leavesReaderAtEnd(lr2) ) return -1;
- return leafReaderTermCmp(&lr1->leafReader,
- leavesReaderTerm(lr2), leavesReaderTermBytes(lr2),
- 0);
- }
- /* Similar to leavesReaderTermCmp(), with additional ordering by idx
- ** so that older segments sort before newer segments.
- */
- static int leavesReaderCmp(LeavesReader *lr1, LeavesReader *lr2){
- int c = leavesReaderTermCmp(lr1, lr2);
- if( c!=0 ) return c;
- return lr1->idx-lr2->idx;
- }
- /* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its
- ** sorted position.
- */
- static void leavesReaderReorder(LeavesReader *pLr, int nLr){
- while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){
- LeavesReader tmp = pLr[0];
- pLr[0] = pLr[1];
- pLr[1] = tmp;
- nLr--;
- pLr++;
- }
- }
- /* Initializes pReaders with the segments from level iLevel, returning
- ** the number of segments in *piReaders. Leaves pReaders in sorted
- ** order.
- */
- static int leavesReadersInit(fulltext_vtab *v, int iLevel,
- LeavesReader *pReaders, int *piReaders){
- sqlite3_stmt *s;
- int i, rc = sql_get_statement(v, SEGDIR_SELECT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
- i = 0;
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- sqlite_int64 iStart = sqlite3_column_int64(s, 0);
- sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
- const char *pRootData = sqlite3_column_blob(s, 2);
- int nRootData = sqlite3_column_bytes(s, 2);
- assert( i<MERGE_COUNT );
- rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
- &pReaders[i]);
- if( rc!=SQLITE_OK ) break;
- i++;
- }
- if( rc!=SQLITE_DONE ){
- while( i-->0 ){
- leavesReaderDestroy(&pReaders[i]);
- }
- return rc;
- }
- *piReaders = i;
- /* Leave our results sorted by term, then age. */
- while( i-- ){
- leavesReaderReorder(pReaders+i, *piReaders-i);
- }
- return SQLITE_OK;
- }
- /* Merge doclists from pReaders[nReaders] into a single doclist, which
- ** is written to pWriter. Assumes pReaders is ordered oldest to
- ** newest.
- */
- /* TODO(shess) Consider putting this inline in segmentMerge(). */
- static int leavesReadersMerge(fulltext_vtab *v,
- LeavesReader *pReaders, int nReaders,
- LeafWriter *pWriter){
- DLReader dlReaders[MERGE_COUNT];
- const char *pTerm = leavesReaderTerm(pReaders);
- int i, nTerm = leavesReaderTermBytes(pReaders);
- assert( nReaders<=MERGE_COUNT );
- for(i=0; i<nReaders; i++){
- dlrInit(&dlReaders[i], DL_DEFAULT,
- leavesReaderData(pReaders+i),
- leavesReaderDataBytes(pReaders+i));
- }
- return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
- }
- /* Forward ref due to mutual recursion with segdirNextIndex(). */
- static int segmentMerge(fulltext_vtab *v, int iLevel);
- /* Put the next available index at iLevel into *pidx. If iLevel
- ** already has MERGE_COUNT segments, they are merged to a higher
- ** level to make room.
- */
- static int segdirNextIndex(fulltext_vtab *v, int iLevel, int *pidx){
- int rc = segdir_max_index(v, iLevel, pidx);
- if( rc==SQLITE_DONE ){ /* No segments at iLevel. */
- *pidx = 0;
- }else if( rc==SQLITE_ROW ){
- if( *pidx==(MERGE_COUNT-1) ){
- rc = segmentMerge(v, iLevel);
- if( rc!=SQLITE_OK ) return rc;
- *pidx = 0;
- }else{
- (*pidx)++;
- }
- }else{
- return rc;
- }
- return SQLITE_OK;
- }
- /* Merge MERGE_COUNT segments at iLevel into a new segment at
- ** iLevel+1. If iLevel+1 is already full of segments, those will be
- ** merged to make room.
- */
- static int segmentMerge(fulltext_vtab *v, int iLevel){
- LeafWriter writer;
- LeavesReader lrs[MERGE_COUNT];
- int i, rc, idx = 0;
- /* Determine the next available segment index at the next level,
- ** merging as necessary.
- */
- rc = segdirNextIndex(v, iLevel+1, &idx);
- if( rc!=SQLITE_OK ) return rc;
- /* TODO(shess) This assumes that we'll always see exactly
- ** MERGE_COUNT segments to merge at a given level. That will be
- ** broken if we allow the developer to request preemptive or
- ** deferred merging.
- */
- memset(&lrs, ' ', sizeof(lrs));
- rc = leavesReadersInit(v, iLevel, lrs, &i);
- if( rc!=SQLITE_OK ) return rc;
- assert( i==MERGE_COUNT );
- leafWriterInit(iLevel+1, idx, &writer);
- /* Since leavesReaderReorder() pushes readers at eof to the end,
- ** when the first reader is empty, all will be empty.
- */
- while( !leavesReaderAtEnd(lrs) ){
- /* Figure out how many readers share their next term. */
- for(i=1; i<MERGE_COUNT && !leavesReaderAtEnd(lrs+i); i++){
- if( 0!=leavesReaderTermCmp(lrs, lrs+i) ) break;
- }
- rc = leavesReadersMerge(v, lrs, i, &writer);
- if( rc!=SQLITE_OK ) goto err;
- /* Step forward those that were merged. */
- while( i-->0 ){
- rc = leavesReaderStep(v, lrs+i);
- if( rc!=SQLITE_OK ) goto err;
- /* Reorder by term, then by age. */
- leavesReaderReorder(lrs+i, MERGE_COUNT-i);
- }
- }
- for(i=0; i<MERGE_COUNT; i++){
- leavesReaderDestroy(&lrs[i]);
- }
- rc = leafWriterFinalize(v, &writer);
- leafWriterDestroy(&writer);
- if( rc!=SQLITE_OK ) return rc;
- /* Delete the merged segment data. */
- return segdir_delete(v, iLevel);
- err:
- for(i=0; i<MERGE_COUNT; i++){
- leavesReaderDestroy(&lrs[i]);
- }
- leafWriterDestroy(&writer);
- return rc;
- }
- /* Accumulate the union of *acc and *pData into *acc. */
- static void docListAccumulateUnion(DataBuffer *acc,
- const char *pData, int nData) {
- DataBuffer tmp = *acc;
- dataBufferInit(acc, tmp.nData+nData);
- docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
- dataBufferDestroy(&tmp);
- }
- /* TODO(shess) It might be interesting to explore different merge
- ** strategies, here. For instance, since this is a sorted merge, we
- ** could easily merge many doclists in parallel. With some
- ** comprehension of the storage format, we could merge all of the
- ** doclists within a leaf node directly from the leaf node's storage.
- ** It may be worthwhile to merge smaller doclists before larger
- ** doclists, since they can be traversed more quickly - but the
- ** results may have less overlap, making them more expensive in a
- ** different way.
- */
- /* Scan pReader for pTerm/nTerm, and merge the term's doclist over
- ** *out (any doclists with duplicate docids overwrite those in *out).
- ** Internal function for loadSegmentLeaf().
- */
- static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- /* doclist data is accumulated into pBuffers similar to how one does
- ** increment in binary arithmetic. If index 0 is empty, the data is
- ** stored there. If there is data there, it is merged and the
- ** results carried into position 1, with further merge-and-carry
- ** until an empty position is found.
- */
- DataBuffer *pBuffers = NULL;
- int nBuffers = 0, nMaxBuffers = 0, rc;
- assert( nTerm>0 );
- for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader);
- rc=leavesReaderStep(v, pReader)){
- /* TODO(shess) Really want leavesReaderTermCmp(), but that name is
- ** already taken to compare the terms of two LeavesReaders. Think
- ** on a better name. [Meanwhile, break encapsulation rather than
- ** use a confusing name.]
- */
- int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
- if( c>0 ) break; /* Past any possible matches. */
- if( c==0 ){
- const char *pData = leavesReaderData(pReader);
- int iBuffer, nData = leavesReaderDataBytes(pReader);
- /* Find the first empty buffer. */
- for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
- if( 0==pBuffers[iBuffer].nData ) break;
- }
- /* Out of buffers, add an empty one. */
- if( iBuffer==nBuffers ){
- if( nBuffers==nMaxBuffers ){
- DataBuffer *p;
- nMaxBuffers += 20;
- /* Manual realloc so we can handle NULL appropriately. */
- p = sqlite3_malloc(nMaxBuffers*sizeof(*pBuffers));
- if( p==NULL ){
- rc = SQLITE_NOMEM;
- break;
- }
- if( nBuffers>0 ){
- assert(pBuffers!=NULL);
- memcpy(p, pBuffers, nBuffers*sizeof(*pBuffers));
- sqlite3_free(pBuffers);
- }
- pBuffers = p;
- }
- dataBufferInit(&(pBuffers[nBuffers]), 0);
- nBuffers++;
- }
- /* At this point, must have an empty at iBuffer. */
- assert(iBuffer<nBuffers && pBuffers[iBuffer].nData==0);
- /* If empty was first buffer, no need for merge logic. */
- if( iBuffer==0 ){
- dataBufferReplace(&(pBuffers[0]), pData, nData);
- }else{
- /* pAcc is the empty buffer the merged data will end up in. */
- DataBuffer *pAcc = &(pBuffers[iBuffer]);
- DataBuffer *p = &(pBuffers[0]);
- /* Handle position 0 specially to avoid need to prime pAcc
- ** with pData/nData.
- */
- dataBufferSwap(p, pAcc);
- docListAccumulateUnion(pAcc, pData, nData);
- /* Accumulate remaining doclists into pAcc. */
- for(++p; p<pAcc; ++p){
- docListAccumulateUnion(pAcc, p->pData, p->nData);
- /* dataBufferReset() could allow a large doclist to blow up
- ** our memory requirements.
- */
- if( p->nCapacity<1024 ){
- dataBufferReset(p);
- }else{
- dataBufferDestroy(p);
- dataBufferInit(p, 0);
- }
- }
- }
- }
- }
- /* Union all the doclists together into *out. */
- /* TODO(shess) What if *out is big? Sigh. */
- if( rc==SQLITE_OK && nBuffers>0 ){
- int iBuffer;
- for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
- if( pBuffers[iBuffer].nData>0 ){
- if( out->nData==0 ){
- dataBufferSwap(out, &(pBuffers[iBuffer]));
- }else{
- docListAccumulateUnion(out, pBuffers[iBuffer].pData,
- pBuffers[iBuffer].nData);
- }
- }
- }
- }
- while( nBuffers-- ){
- dataBufferDestroy(&(pBuffers[nBuffers]));
- }
- if( pBuffers!=NULL ) sqlite3_free(pBuffers);
- return rc;
- }
- /* Call loadSegmentLeavesInt() with pData/nData as input. */
- static int loadSegmentLeaf(fulltext_vtab *v, const char *pData, int nData,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- LeavesReader reader;
- int rc;
- assert( nData>1 );
- assert( *pData==' ' );
- rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader);
- if( rc!=SQLITE_OK ) return rc;
- rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
- leavesReaderReset(&reader);
- leavesReaderDestroy(&reader);
- return rc;
- }
- /* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to
- ** iEndLeaf (inclusive) as input, and merge the resulting doclist into
- ** out.
- */
- static int loadSegmentLeaves(fulltext_vtab *v,
- sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- int rc;
- LeavesReader reader;
- assert( iStartLeaf<=iEndLeaf );
- rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader);
- if( rc!=SQLITE_OK ) return rc;
- rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
- leavesReaderReset(&reader);
- leavesReaderDestroy(&reader);
- return rc;
- }
- /* Taking pData/nData as an interior node, find the sequence of child
- ** nodes which could include pTerm/nTerm/isPrefix. Note that the
- ** interior node terms logically come between the blocks, so there is
- ** one more blockid than there are terms (that block contains terms >=
- ** the last interior-node term).
- */
- /* TODO(shess) The calling code may already know that the end child is
- ** not worth calculating, because the end may be in a later sibling
- ** node. Consider whether breaking symmetry is worthwhile. I suspect
- ** it is not worthwhile.
- */
- static void getChildrenContaining(const char *pData, int nData,
- const char *pTerm, int nTerm, int isPrefix,
- sqlite_int64 *piStartChild,
- sqlite_int64 *piEndChild){
- InteriorReader reader;
- assert( nData>1 );
- assert( *pData!=' ' );
- interiorReaderInit(pData, nData, &reader);
- /* Scan for the first child which could contain pTerm/nTerm. */
- while( !interiorReaderAtEnd(&reader) ){
- if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
- interiorReaderStep(&reader);
- }
- *piStartChild = interiorReaderCurrentBlockid(&reader);
- /* Keep scanning to find a term greater than our term, using prefix
- ** comparison if indicated. If isPrefix is false, this will be the
- ** same blockid as the starting block.
- */
- while( !interiorReaderAtEnd(&reader) ){
- if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
- interiorReaderStep(&reader);
- }
- *piEndChild = interiorReaderCurrentBlockid(&reader);
- interiorReaderDestroy(&reader);
- /* Children must ascend, and if !prefix, both must be the same. */
- assert( *piEndChild>=*piStartChild );
- assert( isPrefix || *piStartChild==*piEndChild );
- }
- /* Read block at iBlockid and pass it with other params to
- ** getChildrenContaining().
- */
- static int loadAndGetChildrenContaining(
- fulltext_vtab *v,
- sqlite_int64 iBlockid,
- const char *pTerm, int nTerm, int isPrefix,
- sqlite_int64 *piStartChild, sqlite_int64 *piEndChild
- ){
- sqlite3_stmt *s = NULL;
- int rc;
- assert( iBlockid!=0 );
- assert( pTerm!=NULL );
- assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */
- assert( piStartChild!=NULL );
- assert( piEndChild!=NULL );
- rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
- rc = sqlite3_bind_int64(s, 1, iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_ERROR;
- if( rc!=SQLITE_ROW ) return rc;
- getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
- pTerm, nTerm, isPrefix, piStartChild, piEndChild);
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain
- * locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
- return SQLITE_OK;
- }
- /* Traverse the tree represented by pData[nData] looking for
- ** pTerm[nTerm], placing its doclist into *out. This is internal to
- ** loadSegment() to make error-handling cleaner.
- */
- static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 iLeavesEnd,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- /* Special case where root is a leaf. */
- if( *pData==' ' ){
- return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out);
- }else{
- int rc;
- sqlite_int64 iStartChild, iEndChild;
- /* Process pData as an interior node, then loop down the tree
- ** until we find the set of leaf nodes to scan for the term.
- */
- getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
- &iStartChild, &iEndChild);
- while( iStartChild>iLeavesEnd ){
- sqlite_int64 iNextStart, iNextEnd;
- rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
- &iNextStart, &iNextEnd);
- if( rc!=SQLITE_OK ) return rc;
- /* If we've branched, follow the end branch, too. */
- if( iStartChild!=iEndChild ){
- sqlite_int64 iDummy;
- rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix,
- &iDummy, &iNextEnd);
- if( rc!=SQLITE_OK ) return rc;
- }
- assert( iNextStart<=iNextEnd );
- iStartChild = iNextStart;
- iEndChild = iNextEnd;
- }
- assert( iStartChild<=iLeavesEnd );
- assert( iEndChild<=iLeavesEnd );
- /* Scan through the leaf segments for doclists. */
- return loadSegmentLeaves(v, iStartChild, iEndChild,
- pTerm, nTerm, isPrefix, out);
- }
- }
- /* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then
- ** merge its doclist over *out (any duplicate doclists read from the
- ** segment rooted at pData will overwrite those in *out).
- */
- /* TODO(shess) Consider changing this to determine the depth of the
- ** leaves using either the first characters of interior nodes (when
- ** ==1, we're one level above the leaves), or the first character of
- ** the root (which will describe the height of the tree directly).
- ** Either feels somewhat tricky to me.
- */
- /* TODO(shess) The current merge is likely to be slow for large
- ** doclists (though it should process from newest/smallest to
- ** oldest/largest, so it may not be that bad). It might be useful to
- ** modify things to allow for N-way merging. This could either be
- ** within a segment, with pairwise merges across segments, or across
- ** all segments at once.
- */
- static int loadSegment(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 iLeavesEnd,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- DataBuffer result;
- int rc;
- assert( nData>1 );
- /* This code should never be called with buffered updates. */
- assert( v->nPendingData<0 );
- dataBufferInit(&result, 0);
- rc = loadSegmentInt(v, pData, nData, iLeavesEnd,
- pTerm, nTerm, isPrefix, &result);
- if( rc==SQLITE_OK && result.nData>0 ){
- if( out->nData==0 ){
- DataBuffer tmp = *out;
- *out = result;
- result = tmp;
- }else{
- DataBuffer merged;
- DLReader readers[2];
- dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
- dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
- dataBufferInit(&merged, out->nData+result.nData);
- docListMerge(&merged, readers, 2);
- dataBufferDestroy(out);
- *out = merged;
- dlrDestroy(&readers[0]);
- dlrDestroy(&readers[1]);
- }
- }
- dataBufferDestroy(&result);
- return rc;
- }
- /* Scan the database and merge together the posting lists for the term
- ** into *out.
- */
- static int termSelect(fulltext_vtab *v, int iColumn,
- const char *pTerm, int nTerm, int isPrefix,
- DocListType iType, DataBuffer *out){
- DataBuffer doclist;
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
- /* This code should never be called with buffered updates. */
- assert( v->nPendingData<0 );
- dataBufferInit(&doclist, 0);
- /* Traverse the segments from oldest to newest so that newer doclist
- ** elements for given docids overwrite older elements.
- */
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- const char *pData = sqlite3_column_blob(s, 0);
- const int nData = sqlite3_column_bytes(s, 0);
- const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
- rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
- &doclist);
- if( rc!=SQLITE_OK ) goto err;
- }
- if( rc==SQLITE_DONE ){
- if( doclist.nData!=0 ){
- /* TODO(shess) The old term_select_all() code applied the column
- ** restrict as we merged segments, leading to smaller buffers.
- ** This is probably worthwhile to bring back, once the new storage
- ** system is checked in.
- */
- if( iColumn==v->nColumn) iColumn = -1;
- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
- iColumn, iType, out);
- }
- rc = SQLITE_OK;
- }
- err:
- dataBufferDestroy(&doclist);
- return rc;
- }
- /****************************************************************/
- /* Used to hold hashtable data for sorting. */
- typedef struct TermData {
- const char *pTerm;
- int nTerm;
- DLCollector *pCollector;
- } TermData;
- /* Orders TermData elements in strcmp fashion ( <0 for less-than, 0
- ** for equal, >0 for greater-than).
- */
- static int termDataCmp(const void *av, const void *bv){
- const TermData *a = (const TermData *)av;
- const TermData *b = (const TermData *)bv;
- int n = a->nTerm<b->nTerm ? a->nTerm : b->nTerm;
- int c = memcmp(a->pTerm, b->pTerm, n);
- if( c!=0 ) return c;
- return a->nTerm-b->nTerm;
- }
- /* Order pTerms data by term, then write a new level 0 segment using
- ** LeafWriter.
- */
- static int writeZeroSegment(fulltext_vtab *v, fts3Hash *pTerms){
- fts3HashElem *e;
- int idx, rc, i, n;
- TermData *pData;
- LeafWriter writer;
- DataBuffer dl;
- /* Determine the next index at level 0, merging as necessary. */
- rc = segdirNextIndex(v, 0, &idx);
- if( rc!=SQLITE_OK ) return rc;
- n = fts3HashCount(pTerms);
- pData = sqlite3_malloc(n*sizeof(TermData));
- for(i = 0, e = fts3HashFirst(pTerms); e; i++, e = fts3HashNext(e)){
- assert( i<n );
- pData[i].pTerm = fts3HashKey(e);
- pData[i].nTerm = fts3HashKeysize(e);
- pData[i].pCollector = fts3HashData(e);
- }
- assert( i==n );
- /* TODO(shess) Should we allow user-defined collation sequences,
- ** here? I think we only need that once we support prefix searches.
- */
- if( n>1 ) qsort(pData, n, sizeof(*pData), termDataCmp);
- /* TODO(shess) Refactor so that we can write directly to the segment
- ** DataBuffer, as happens for segment merges.
- */
- leafWriterInit(0, idx, &writer);
- dataBufferInit(&dl, 0);
- for(i=0; i<n; i++){
- dataBufferReset(&dl);
- dlcAddDoclist(pData[i].pCollector, &dl);
- rc = leafWriterStep(v, &writer,
- pData[i].pTerm, pData[i].nTerm, dl.pData, dl.nData);
- if( rc!=SQLITE_OK ) goto err;
- }
- rc = leafWriterFinalize(v, &writer);
- err:
- dataBufferDestroy(&dl);
- sqlite3_free(pData);
- leafWriterDestroy(&writer);
- return rc;
- }
- /* If pendingTerms has data, free it. */
- static int clearPendingTerms(fulltext_vtab *v){
- if( v->nPendingData>=0 ){
- fts3HashElem *e;
- for(e=fts3HashFirst(&v->pendingTerms); e; e=fts3HashNext(e)){
- dlcDelete(fts3HashData(e));
- }
- fts3HashClear(&v->pendingTerms);
- v->nPendingData = -1;
- }
- return SQLITE_OK;
- }
- /* If pendingTerms has data, flush it to a level-zero segment, and
- ** free it.
- */
- static int flushPendingTerms(fulltext_vtab *v){
- if( v->nPendingData>=0 ){
- int rc = writeZeroSegment(v, &v->pendingTerms);
- if( rc==SQLITE_OK ) clearPendingTerms(v);
- return rc;
- }
- return SQLITE_OK;
- }
- /* If pendingTerms is "too big", or docid is out of order, flush it.
- ** Regardless, be certain that pendingTerms is initialized for use.
- */
- static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
- /* TODO(shess) Explore whether partially flushing the buffer on
- ** forced-flush would provide better performance. I suspect that if
- ** we ordered the doclists by size and flushed the largest until the
- ** buffer was half empty, that would let the less frequent terms
- ** generate longer doclists.
- */
- if( iDocid<=v->iPrevDocid || v->nPendingData>kPendingThreshold ){
- int rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ) return rc;
- }
- if( v->nPendingData<0 ){
- fts3HashInit(&v->pendingTerms, FTS3_HASH_STRING, 1);
- v->nPendingData = 0;
- }
- v->iPrevDocid = iDocid;
- return SQLITE_OK;
- }
- /* This function implements the xUpdate callback; it is the top-level entry
- * point for inserting, deleting or updating a row in a full-text table. */
- static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
- sqlite_int64 *pRowid){
- fulltext_vtab *v = (fulltext_vtab *) pVtab;
- int rc;
- FTSTRACE(("FTS3 Update %pn", pVtab));
- if( nArg<2 ){
- rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
- } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
- /* An update:
- * ppArg[0] = old rowid
- * ppArg[1] = new rowid
- * ppArg[2..2+v->nColumn-1] = values
- * ppArg[2+v->nColumn] = value for magic column (we ignore this)
- * ppArg[2+v->nColumn+1] = value for docid
- */
- sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
- if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER ||
- sqlite3_value_int64(ppArg[1]) != rowid ){
- rc = SQLITE_ERROR; /* we don't allow changing the rowid */
- }else if( sqlite3_value_type(ppArg[2+v->nColumn+1]) != SQLITE_INTEGER ||
- sqlite3_value_int64(ppArg[2+v->nColumn+1]) != rowid ){
- rc = SQLITE_ERROR; /* we don't allow changing the docid */
- }else{
- assert( nArg==2+v->nColumn+2);
- rc = index_update(v, rowid, &ppArg[2]);
- }
- } else {
- /* An insert:
- * ppArg[1] = requested rowid
- * ppArg[2..2+v->nColumn-1] = values
- * ppArg[2+v->nColumn] = value for magic column (we ignore this)
- * ppArg[2+v->nColumn+1] = value for docid
- */
- sqlite3_value *pRequestDocid = ppArg[2+v->nColumn+1];
- assert( nArg==2+v->nColumn+2);
- if( SQLITE_NULL != sqlite3_value_type(pRequestDocid) &&
- SQLITE_NULL != sqlite3_value_type(ppArg[1]) ){
- /* TODO(shess) Consider allowing this to work if the values are
- ** identical. I'm inclined to discourage that usage, though,
- ** given that both rowid and docid are special columns. Better
- ** would be to define one or the other as the default winner,
- ** but should it be fts3-centric (docid) or SQLite-centric
- ** (rowid)?
- */
- rc = SQLITE_ERROR;
- }else{
- if( SQLITE_NULL == sqlite3_value_type(pRequestDocid) ){
- pRequestDocid = ppArg[1];
- }
- rc = index_insert(v, pRequestDocid, &ppArg[2], pRowid);
- }
- }
- return rc;
- }
- static int fulltextSync(sqlite3_vtab *pVtab){
- FTSTRACE(("FTS3 xSync()n"));
- return flushPendingTerms((fulltext_vtab *)pVtab);
- }
- static int fulltextBegin(sqlite3_vtab *pVtab){
- fulltext_vtab *v = (fulltext_vtab *) pVtab;
- FTSTRACE(("FTS3 xBegin()n"));
- /* Any buffered updates should have been cleared by the previous
- ** transaction.
- */
- assert( v->nPendingData<0 );
- return clearPendingTerms(v);
- }
- static int fulltextCommit(sqlite3_vtab *pVtab){
- fulltext_vtab *v = (fulltext_vtab *) pVtab;
- FTSTRACE(("FTS3 xCommit()n"));
- /* Buffered updates should have been cleared by fulltextSync(). */
- assert( v->nPendingData<0 );
- return clearPendingTerms(v);
- }
- static int fulltextRollback(sqlite3_vtab *pVtab){
- FTSTRACE(("FTS3 xRollback()n"));
- return clearPendingTerms((fulltext_vtab *)pVtab);
- }
- /*
- ** Implementation of the snippet() function for FTS3
- */
- static void snippetFunc(
- sqlite3_context *pContext,
- int argc,
- sqlite3_value **argv
- ){
- fulltext_cursor *pCursor;
- if( argc<1 ) return;
- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
- }else{
- const char *zStart = "<b>";
- const char *zEnd = "</b>";
- const char *zEllipsis = "<b>...</b>";
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- if( argc>=2 ){
- zStart = (const char*)sqlite3_value_text(argv[1]);
- if( argc>=3 ){
- zEnd = (const char*)sqlite3_value_text(argv[2]);
- if( argc>=4 ){
- zEllipsis = (const char*)sqlite3_value_text(argv[3]);
- }
- }
- }
- snippetAllOffsets(pCursor);
- snippetText(pCursor, zStart, zEnd, zEllipsis);
- sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
- pCursor->snippet.nSnippet, SQLITE_STATIC);
- }
- }
- /*
- ** Implementation of the offsets() function for FTS3
- */
- static void snippetOffsetsFunc(
- sqlite3_context *pContext,
- int argc,
- sqlite3_value **argv
- ){
- fulltext_cursor *pCursor;
- if( argc<1 ) return;
- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
- }else{
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- snippetAllOffsets(pCursor);
- snippetOffsetText(&pCursor->snippet);
- sqlite3_result_text(pContext,
- pCursor->snippet.zOffset, pCursor->snippet.nOffset,
- SQLITE_STATIC);
- }
- }
- /*
- ** This routine implements the xFindFunction method for the FTS3
- ** virtual table.
- */
- static int fulltextFindFunction(
- sqlite3_vtab *pVtab,
- int nArg,
- const char *zName,
- void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
- void **ppArg
- ){
- if( strcmp(zName,"snippet")==0 ){
- *pxFunc = snippetFunc;
- return 1;
- }else if( strcmp(zName,"offsets")==0 ){
- *pxFunc = snippetOffsetsFunc;
- return 1;
- }
- return 0;
- }
- /*
- ** Rename an fts3 table.
- */
- static int fulltextRename(
- sqlite3_vtab *pVtab,
- const char *zName
- ){
- fulltext_vtab *p = (fulltext_vtab *)pVtab;
- int rc = SQLITE_NOMEM;
- char *zSql = sqlite3_mprintf(
- "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
- "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
- "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
- , p->zDb, p->zName, zName
- , p->zDb, p->zName, zName
- , p->zDb, p->zName, zName
- );
- if( zSql ){
- rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
- sqlite3_free(zSql);
- }
- return rc;
- }
- static const sqlite3_module fts3Module = {
- /* iVersion */ 0,
- /* xCreate */ fulltextCreate,
- /* xConnect */ fulltextConnect,
- /* xBestIndex */ fulltextBestIndex,
- /* xDisconnect */ fulltextDisconnect,
- /* xDestroy */ fulltextDestroy,
- /* xOpen */ fulltextOpen,
- /* xClose */ fulltextClose,
- /* xFilter */ fulltextFilter,
- /* xNext */ fulltextNext,
- /* xEof */ fulltextEof,
- /* xColumn */ fulltextColumn,
- /* xRowid */ fulltextRowid,
- /* xUpdate */ fulltextUpdate,
- /* xBegin */ fulltextBegin,
- /* xSync */ fulltextSync,
- /* xCommit */ fulltextCommit,
- /* xRollback */ fulltextRollback,
- /* xFindFunction */ fulltextFindFunction,
- /* xRename */ fulltextRename,
- };
- static void hashDestroy(void *p){
- fts3Hash *pHash = (fts3Hash *)p;
- sqlite3Fts3HashClear(pHash);
- sqlite3_free(pHash);
- }
- /*
- ** The fts3 built-in tokenizers - "simple" and "porter" - are implemented
- ** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following
- ** two forward declarations are for functions declared in these files
- ** used to retrieve the respective implementations.
- **
- ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
- ** to by the argument to point a the "simple" tokenizer implementation.
- ** Function ...PorterTokenizerModule() sets *pModule to point to the
- ** porter tokenizer/stemmer implementation.
- */
- void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
- void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
- void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
- int sqlite3Fts3InitHashTable(sqlite3 *, fts3Hash *, const char *);
- /*
- ** Initialise the fts3 extension. If this extension is built as part
- ** of the sqlite library, then this function is called directly by
- ** SQLite. If fts3 is built as a dynamically loadable extension, this
- ** function is called by the sqlite3_extension_init() entry point.
- */
- int sqlite3Fts3Init(sqlite3 *db){
- int rc = SQLITE_OK;
- fts3Hash *pHash = 0;
- const sqlite3_tokenizer_module *pSimple = 0;
- const sqlite3_tokenizer_module *pPorter = 0;
- const sqlite3_tokenizer_module *pIcu = 0;
- sqlite3Fts3SimpleTokenizerModule(&pSimple);
- sqlite3Fts3PorterTokenizerModule(&pPorter);
- #ifdef SQLITE_ENABLE_ICU
- sqlite3Fts3IcuTokenizerModule(&pIcu);
- #endif
- /* Allocate and initialise the hash-table used to store tokenizers. */
- pHash = sqlite3_malloc(sizeof(fts3Hash));
- if( !pHash ){
- rc = SQLITE_NOMEM;
- }else{
- sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
- }
- /* Load the built-in tokenizers into the hash table */
- if( rc==SQLITE_OK ){
- if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
- || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
- || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
- ){
- rc = SQLITE_NOMEM;
- }
- }
- /* Create the virtual table wrapper around the hash-table and overload
- ** the two scalar functions. If this is successful, register the
- ** module with sqlite.
- */
- if( SQLITE_OK==rc
- && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
- ){
- return sqlite3_create_module_v2(
- db, "fts3", &fts3Module, (void *)pHash, hashDestroy
- );
- }
- /* An error has occured. Delete the hash table and return the error code. */
- assert( rc!=SQLITE_OK );
- if( pHash ){
- sqlite3Fts3HashClear(pHash);
- sqlite3_free(pHash);
- }
- return rc;
- }
- #if !SQLITE_CORE
- int sqlite3_extension_init(
- sqlite3 *db,
- char **pzErrMsg,
- const sqlite3_api_routines *pApi
- ){
- SQLITE_EXTENSION_INIT2(pApi)
- return sqlite3Fts3Init(db);
- }
- #endif
- #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */