fts3_icu.c.svn-base
上传用户:sunhongbo
上传日期:2022-01-25
资源大小:3010k
文件大小:7k
- /*
- ** 2007 June 22
- **
- ** The author disclaims copyright to this source code. In place of
- ** a legal notice, here is a blessing:
- **
- ** May you do good and not evil.
- ** May you find forgiveness for yourself and forgive others.
- ** May you share freely, never taking more than you give.
- **
- *************************************************************************
- ** This file implements a tokenizer for fts3 based on the ICU library.
- **
- ** $Id: fts3_icu.c,v 1.2 2007/10/24 21:52:37 shess Exp $
- */
- #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
- #ifdef SQLITE_ENABLE_ICU
- #include <assert.h>
- #include <string.h>
- #include "fts3_tokenizer.h"
- #include <unicode/ubrk.h>
- #include <unicode/ucol.h>
- #include <unicode/ustring.h>
- #include <unicode/utf16.h>
- typedef struct IcuTokenizer IcuTokenizer;
- typedef struct IcuCursor IcuCursor;
- struct IcuTokenizer {
- sqlite3_tokenizer base;
- char *zLocale;
- };
- struct IcuCursor {
- sqlite3_tokenizer_cursor base;
- UBreakIterator *pIter; /* ICU break-iterator object */
- int nChar; /* Number of UChar elements in pInput */
- UChar *aChar; /* Copy of input using utf-16 encoding */
- int *aOffset; /* Offsets of each character in utf-8 input */
- int nBuffer;
- char *zBuffer;
- int iToken;
- };
- /*
- ** Create a new tokenizer instance.
- */
- static int icuCreate(
- int argc, /* Number of entries in argv[] */
- const char * const *argv, /* Tokenizer creation arguments */
- sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
- ){
- IcuTokenizer *p;
- int n = 0;
- if( argc>0 ){
- n = strlen(argv[0])+1;
- }
- p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
- if( !p ){
- return SQLITE_NOMEM;
- }
- memset(p, 0, sizeof(IcuTokenizer));
- if( n ){
- p->zLocale = (char *)&p[1];
- memcpy(p->zLocale, argv[0], n);
- }
- *ppTokenizer = (sqlite3_tokenizer *)p;
- return SQLITE_OK;
- }
- /*
- ** Destroy a tokenizer
- */
- static int icuDestroy(sqlite3_tokenizer *pTokenizer){
- IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
- sqlite3_free(p);
- return SQLITE_OK;
- }
- /*
- ** Prepare to begin tokenizing a particular string. The input
- ** string to be tokenized is pInput[0..nBytes-1]. A cursor
- ** used to incrementally tokenize this string is returned in
- ** *ppCursor.
- */
- static int icuOpen(
- sqlite3_tokenizer *pTokenizer, /* The tokenizer */
- const char *zInput, /* Input string */
- int nInput, /* Length of zInput in bytes */
- sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
- ){
- IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
- IcuCursor *pCsr;
- const int32_t opt = U_FOLD_CASE_DEFAULT;
- UErrorCode status = U_ZERO_ERROR;
- int nChar;
- UChar32 c;
- int iInput = 0;
- int iOut = 0;
- *ppCursor = 0;
- if( -1 == nInput ) nInput = strlen(nInput);
- nChar = nInput+1;
- pCsr = (IcuCursor *)sqlite3_malloc(
- sizeof(IcuCursor) + /* IcuCursor */
- nChar * sizeof(UChar) + /* IcuCursor.aChar[] */
- (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */
- );
- if( !pCsr ){
- return SQLITE_NOMEM;
- }
- memset(pCsr, 0, sizeof(IcuCursor));
- pCsr->aChar = (UChar *)&pCsr[1];
- pCsr->aOffset = (int *)&pCsr->aChar[nChar];
- pCsr->aOffset[iOut] = iInput;
- U8_NEXT(zInput, iInput, nInput, c);
- while( c>0 ){
- int isError = 0;
- c = u_foldCase(c, opt);
- U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
- if( isError ){
- sqlite3_free(pCsr);
- return SQLITE_ERROR;
- }
- pCsr->aOffset[iOut] = iInput;
- if( iInput<nInput ){
- U8_NEXT(zInput, iInput, nInput, c);
- }else{
- c = 0;
- }
- }
- pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
- if( !U_SUCCESS(status) ){
- sqlite3_free(pCsr);
- return SQLITE_ERROR;
- }
- pCsr->nChar = iOut;
- ubrk_first(pCsr->pIter);
- *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
- return SQLITE_OK;
- }
- /*
- ** Close a tokenization cursor previously opened by a call to icuOpen().
- */
- static int icuClose(sqlite3_tokenizer_cursor *pCursor){
- IcuCursor *pCsr = (IcuCursor *)pCursor;
- ubrk_close(pCsr->pIter);
- sqlite3_free(pCsr->zBuffer);
- sqlite3_free(pCsr);
- return SQLITE_OK;
- }
- /*
- ** Extract the next token from a tokenization cursor.
- */
- static int icuNext(
- sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
- const char **ppToken, /* OUT: *ppToken is the token text */
- int *pnBytes, /* OUT: Number of bytes in token */
- int *piStartOffset, /* OUT: Starting offset of token */
- int *piEndOffset, /* OUT: Ending offset of token */
- int *piPosition /* OUT: Position integer of token */
- ){
- IcuCursor *pCsr = (IcuCursor *)pCursor;
- int iStart = 0;
- int iEnd = 0;
- int nByte = 0;
- while( iStart==iEnd ){
- UChar32 c;
- iStart = ubrk_current(pCsr->pIter);
- iEnd = ubrk_next(pCsr->pIter);
- if( iEnd==UBRK_DONE ){
- return SQLITE_DONE;
- }
- while( iStart<iEnd ){
- int iWhite = iStart;
- U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
- if( u_isspace(c) ){
- iStart = iWhite;
- }else{
- break;
- }
- }
- assert(iStart<=iEnd);
- }
- do {
- UErrorCode status = U_ZERO_ERROR;
- if( nByte ){
- char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
- if( !zNew ){
- return SQLITE_NOMEM;
- }
- pCsr->zBuffer = zNew;
- pCsr->nBuffer = nByte;
- }
- u_strToUTF8(
- pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */
- &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */
- &status /* Output success/failure */
- );
- } while( nByte>pCsr->nBuffer );
- *ppToken = pCsr->zBuffer;
- *pnBytes = nByte;
- *piStartOffset = pCsr->aOffset[iStart];
- *piEndOffset = pCsr->aOffset[iEnd];
- *piPosition = pCsr->iToken++;
- return SQLITE_OK;
- }
- /*
- ** The set of routines that implement the simple tokenizer
- */
- static const sqlite3_tokenizer_module icuTokenizerModule = {
- 0, /* iVersion */
- icuCreate, /* xCreate */
- icuDestroy, /* xCreate */
- icuOpen, /* xOpen */
- icuClose, /* xClose */
- icuNext, /* xNext */
- };
- /*
- ** Set *ppModule to point at the implementation of the ICU tokenizer.
- */
- void sqlite3Fts3IcuTokenizerModule(
- sqlite3_tokenizer_module const**ppModule
- ){
- *ppModule = &icuTokenizerModule;
- }
- #endif /* defined(SQLITE_ENABLE_ICU) */
- #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */