fts3_tokenizer1.c.svn-base
上传用户:sunhongbo
上传日期:2022-01-25
资源大小:3010k
文件大小:6k
- /*
- ** 2006 Oct 10
- **
- ** The author disclaims copyright to this source code. In place of
- ** a legal notice, here is a blessing:
- **
- ** May you do good and not evil.
- ** May you find forgiveness for yourself and forgive others.
- ** May you share freely, never taking more than you give.
- **
- ******************************************************************************
- **
- ** Implementation of the "simple" full-text-search tokenizer.
- */
- /*
- ** The code in this file is only compiled if:
- **
- ** * The FTS3 module is being built as an extension
- ** (in which case SQLITE_CORE is not defined), or
- **
- ** * The FTS3 module is being built into the core of
- ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
- */
- #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
- #include <assert.h>
- #include <stdlib.h>
- #include <stdio.h>
- #include <string.h>
- #include <ctype.h>
- #include "fts3_tokenizer.h"
- typedef struct simple_tokenizer {
- sqlite3_tokenizer base;
- char delim[128]; /* flag ASCII delimiters */
- } simple_tokenizer;
- typedef struct simple_tokenizer_cursor {
- sqlite3_tokenizer_cursor base;
- const char *pInput; /* input we are tokenizing */
- int nBytes; /* size of the input */
- int iOffset; /* current position in pInput */
- int iToken; /* index of next token to be returned */
- char *pToken; /* storage for current token */
- int nTokenAllocated; /* space allocated to zToken buffer */
- } simple_tokenizer_cursor;
- /* Forward declaration */
- static const sqlite3_tokenizer_module simpleTokenizerModule;
- static int simpleDelim(simple_tokenizer *t, unsigned char c){
- return c<0x80 && t->delim[c];
- }
- /*
- ** Create a new tokenizer instance.
- */
- static int simpleCreate(
- int argc, const char * const *argv,
- sqlite3_tokenizer **ppTokenizer
- ){
- simple_tokenizer *t;
- t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
- if( t==NULL ) return SQLITE_NOMEM;
- memset(t, 0, sizeof(*t));
- /* TODO(shess) Delimiters need to remain the same from run to run,
- ** else we need to reindex. One solution would be a meta-table to
- ** track such information in the database, then we'd only want this
- ** information on the initial create.
- */
- if( argc>1 ){
- int i, n = strlen(argv[1]);
- for(i=0; i<n; i++){
- unsigned char ch = argv[1][i];
- /* We explicitly don't support UTF-8 delimiters for now. */
- if( ch>=0x80 ){
- sqlite3_free(t);
- return SQLITE_ERROR;
- }
- t->delim[ch] = 1;
- }
- } else {
- /* Mark non-alphanumeric ASCII characters as delimiters */
- int i;
- for(i=1; i<0x80; i++){
- t->delim[i] = !isalnum(i);
- }
- }
- *ppTokenizer = &t->base;
- return SQLITE_OK;
- }
- /*
- ** Destroy a tokenizer
- */
- static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
- sqlite3_free(pTokenizer);
- return SQLITE_OK;
- }
- /*
- ** Prepare to begin tokenizing a particular string. The input
- ** string to be tokenized is pInput[0..nBytes-1]. A cursor
- ** used to incrementally tokenize this string is returned in
- ** *ppCursor.
- */
- static int simpleOpen(
- sqlite3_tokenizer *pTokenizer, /* The tokenizer */
- const char *pInput, int nBytes, /* String to be tokenized */
- sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
- ){
- simple_tokenizer_cursor *c;
- c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
- if( c==NULL ) return SQLITE_NOMEM;
- c->pInput = pInput;
- if( pInput==0 ){
- c->nBytes = 0;
- }else if( nBytes<0 ){
- c->nBytes = (int)strlen(pInput);
- }else{
- c->nBytes = nBytes;
- }
- c->iOffset = 0; /* start tokenizing at the beginning */
- c->iToken = 0;
- c->pToken = NULL; /* no space allocated, yet. */
- c->nTokenAllocated = 0;
- *ppCursor = &c->base;
- return SQLITE_OK;
- }
- /*
- ** Close a tokenization cursor previously opened by a call to
- ** simpleOpen() above.
- */
- static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
- simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
- sqlite3_free(c->pToken);
- sqlite3_free(c);
- return SQLITE_OK;
- }
- /*
- ** Extract the next token from a tokenization cursor. The cursor must
- ** have been opened by a prior call to simpleOpen().
- */
- static int simpleNext(
- sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
- const char **ppToken, /* OUT: *ppToken is the token text */
- int *pnBytes, /* OUT: Number of bytes in token */
- int *piStartOffset, /* OUT: Starting offset of token */
- int *piEndOffset, /* OUT: Ending offset of token */
- int *piPosition /* OUT: Position integer of token */
- ){
- simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
- simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
- unsigned char *p = (unsigned char *)c->pInput;
- while( c->iOffset<c->nBytes ){
- int iStartOffset;
- /* Scan past delimiter characters */
- while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
- c->iOffset++;
- }
- /* Count non-delimiter characters. */
- iStartOffset = c->iOffset;
- while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
- c->iOffset++;
- }
- if( c->iOffset>iStartOffset ){
- int i, n = c->iOffset-iStartOffset;
- if( n>c->nTokenAllocated ){
- c->nTokenAllocated = n+20;
- c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
- if( c->pToken==NULL ) return SQLITE_NOMEM;
- }
- for(i=0; i<n; i++){
- /* TODO(shess) This needs expansion to handle UTF-8
- ** case-insensitivity.
- */
- unsigned char ch = p[iStartOffset+i];
- c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
- }
- *ppToken = c->pToken;
- *pnBytes = n;
- *piStartOffset = iStartOffset;
- *piEndOffset = c->iOffset;
- *piPosition = c->iToken++;
- return SQLITE_OK;
- }
- }
- return SQLITE_DONE;
- }
- /*
- ** The set of routines that implement the simple tokenizer
- */
- static const sqlite3_tokenizer_module simpleTokenizerModule = {
- 0,
- simpleCreate,
- simpleDestroy,
- simpleOpen,
- simpleClose,
- simpleNext,
- };
- /*
- ** Allocate a new simple tokenizer. Return a pointer to the new
- ** tokenizer in *ppModule
- */
- void sqlite3Fts3SimpleTokenizerModule(
- sqlite3_tokenizer_module const**ppModule
- ){
- *ppModule = &simpleTokenizerModule;
- }
- #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */