fts3_tokenizer.c.svn-base
上传用户:sunhongbo
上传日期:2022-01-25
资源大小:3010k
文件大小:10k
- /*
- ** 2007 June 22
- **
- ** The author disclaims copyright to this source code. In place of
- ** a legal notice, here is a blessing:
- **
- ** May you do good and not evil.
- ** May you find forgiveness for yourself and forgive others.
- ** May you share freely, never taking more than you give.
- **
- ******************************************************************************
- **
- ** This is part of an SQLite module implementing full-text search.
- ** This particular file implements the generic tokenizer interface.
- */
- /*
- ** The code in this file is only compiled if:
- **
- ** * The FTS3 module is being built as an extension
- ** (in which case SQLITE_CORE is not defined), or
- **
- ** * The FTS3 module is being built into the core of
- ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
- */
- #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
- #include "sqlite3ext.h"
- #ifndef SQLITE_CORE
- SQLITE_EXTENSION_INIT1
- #endif
- #include "fts3_hash.h"
- #include "fts3_tokenizer.h"
- #include <assert.h>
- /*
- ** Implementation of the SQL scalar function for accessing the underlying
- ** hash table. This function may be called as follows:
- **
- ** SELECT <function-name>(<key-name>);
- ** SELECT <function-name>(<key-name>, <pointer>);
- **
- ** where <function-name> is the name passed as the second argument
- ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
- **
- ** If the <pointer> argument is specified, it must be a blob value
- ** containing a pointer to be stored as the hash data corresponding
- ** to the string <key-name>. If <pointer> is not specified, then
- ** the string <key-name> must already exist in the has table. Otherwise,
- ** an error is returned.
- **
- ** Whether or not the <pointer> argument is specified, the value returned
- ** is a blob containing the pointer stored as the hash data corresponding
- ** to string <key-name> (after the hash-table is updated, if applicable).
- */
- static void scalarFunc(
- sqlite3_context *context,
- int argc,
- sqlite3_value **argv
- ){
- fts3Hash *pHash;
- void *pPtr = 0;
- const unsigned char *zName;
- int nName;
- assert( argc==1 || argc==2 );
- pHash = (fts3Hash *)sqlite3_user_data(context);
- zName = sqlite3_value_text(argv[0]);
- nName = sqlite3_value_bytes(argv[0])+1;
- if( argc==2 ){
- void *pOld;
- int n = sqlite3_value_bytes(argv[1]);
- if( n!=sizeof(pPtr) ){
- sqlite3_result_error(context, "argument type mismatch", -1);
- return;
- }
- pPtr = *(void **)sqlite3_value_blob(argv[1]);
- pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
- if( pOld==pPtr ){
- sqlite3_result_error(context, "out of memory", -1);
- return;
- }
- }else{
- pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
- if( !pPtr ){
- char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
- sqlite3_result_error(context, zErr, -1);
- sqlite3_free(zErr);
- return;
- }
- }
- sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
- }
- #ifdef SQLITE_TEST
- #include <tcl.h>
- #include <string.h>
- /*
- ** Implementation of a special SQL scalar function for testing tokenizers
- ** designed to be used in concert with the Tcl testing framework. This
- ** function must be called with two arguments:
- **
- ** SELECT <function-name>(<key-name>, <input-string>);
- ** SELECT <function-name>(<key-name>, <pointer>);
- **
- ** where <function-name> is the name passed as the second argument
- ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
- ** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
- **
- ** The return value is a string that may be interpreted as a Tcl
- ** list. For each token in the <input-string>, three elements are
- ** added to the returned list. The first is the token position, the
- ** second is the token text (folded, stemmed, etc.) and the third is the
- ** substring of <input-string> associated with the token. For example,
- ** using the built-in "simple" tokenizer:
- **
- ** SELECT fts_tokenizer_test('simple', 'I don't see how');
- **
- ** will return the string:
- **
- ** "{0 i I 1 dont don't 2 see see 3 how how}"
- **
- */
- static void testFunc(
- sqlite3_context *context,
- int argc,
- sqlite3_value **argv
- ){
- fts3Hash *pHash;
- sqlite3_tokenizer_module *p;
- sqlite3_tokenizer *pTokenizer = 0;
- sqlite3_tokenizer_cursor *pCsr = 0;
- const char *zErr = 0;
- const char *zName;
- int nName;
- const char *zInput;
- int nInput;
- const char *zArg = 0;
- const char *zToken;
- int nToken;
- int iStart;
- int iEnd;
- int iPos;
- Tcl_Obj *pRet;
- assert( argc==2 || argc==3 );
- nName = sqlite3_value_bytes(argv[0]);
- zName = (const char *)sqlite3_value_text(argv[0]);
- nInput = sqlite3_value_bytes(argv[argc-1]);
- zInput = (const char *)sqlite3_value_text(argv[argc-1]);
- if( argc==3 ){
- zArg = (const char *)sqlite3_value_text(argv[1]);
- }
- pHash = (fts3Hash *)sqlite3_user_data(context);
- p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
- if( !p ){
- char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
- sqlite3_result_error(context, zErr, -1);
- sqlite3_free(zErr);
- return;
- }
- pRet = Tcl_NewObj();
- Tcl_IncrRefCount(pRet);
- if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
- zErr = "error in xCreate()";
- goto finish;
- }
- pTokenizer->pModule = p;
- if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
- zErr = "error in xOpen()";
- goto finish;
- }
- pCsr->pTokenizer = pTokenizer;
- while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
- Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
- Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
- zToken = &zInput[iStart];
- nToken = iEnd-iStart;
- Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
- }
- if( SQLITE_OK!=p->xClose(pCsr) ){
- zErr = "error in xClose()";
- goto finish;
- }
- if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
- zErr = "error in xDestroy()";
- goto finish;
- }
- finish:
- if( zErr ){
- sqlite3_result_error(context, zErr, -1);
- }else{
- sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
- }
- Tcl_DecrRefCount(pRet);
- }
- static
- int registerTokenizer(
- sqlite3 *db,
- char *zName,
- const sqlite3_tokenizer_module *p
- ){
- int rc;
- sqlite3_stmt *pStmt;
- const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
- rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
- if( rc!=SQLITE_OK ){
- return rc;
- }
- sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
- sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
- sqlite3_step(pStmt);
- return sqlite3_finalize(pStmt);
- }
- static
- int queryTokenizer(
- sqlite3 *db,
- char *zName,
- const sqlite3_tokenizer_module **pp
- ){
- int rc;
- sqlite3_stmt *pStmt;
- const char zSql[] = "SELECT fts3_tokenizer(?)";
- *pp = 0;
- rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
- if( rc!=SQLITE_OK ){
- return rc;
- }
- sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
- if( SQLITE_ROW==sqlite3_step(pStmt) ){
- if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
- memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
- }
- }
- return sqlite3_finalize(pStmt);
- }
- void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
- /*
- ** Implementation of the scalar function fts3_tokenizer_internal_test().
- ** This function is used for testing only, it is not included in the
- ** build unless SQLITE_TEST is defined.
- **
- ** The purpose of this is to test that the fts3_tokenizer() function
- ** can be used as designed by the C-code in the queryTokenizer and
- ** registerTokenizer() functions above. These two functions are repeated
- ** in the README.tokenizer file as an example, so it is important to
- ** test them.
- **
- ** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
- ** function with no arguments. An assert() will fail if a problem is
- ** detected. i.e.:
- **
- ** SELECT fts3_tokenizer_internal_test();
- **
- */
- static void intTestFunc(
- sqlite3_context *context,
- int argc,
- sqlite3_value **argv
- ){
- int rc;
- const sqlite3_tokenizer_module *p1;
- const sqlite3_tokenizer_module *p2;
- sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
- /* Test the query function */
- sqlite3Fts3SimpleTokenizerModule(&p1);
- rc = queryTokenizer(db, "simple", &p2);
- assert( rc==SQLITE_OK );
- assert( p1==p2 );
- rc = queryTokenizer(db, "nosuchtokenizer", &p2);
- assert( rc==SQLITE_ERROR );
- assert( p2==0 );
- assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
- /* Test the storage function */
- rc = registerTokenizer(db, "nosuchtokenizer", p1);
- assert( rc==SQLITE_OK );
- rc = queryTokenizer(db, "nosuchtokenizer", &p2);
- assert( rc==SQLITE_OK );
- assert( p2==p1 );
- sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
- }
- #endif
- /*
- ** Set up SQL objects in database db used to access the contents of
- ** the hash table pointed to by argument pHash. The hash table must
- ** been initialised to use string keys, and to take a private copy
- ** of the key when a value is inserted. i.e. by a call similar to:
- **
- ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
- **
- ** This function adds a scalar function (see header comment above
- ** scalarFunc() in this file for details) and, if ENABLE_TABLE is
- ** defined at compilation time, a temporary virtual table (see header
- ** comment above struct HashTableVtab) to the database schema. Both
- ** provide read/write access to the contents of *pHash.
- **
- ** The third argument to this function, zName, is used as the name
- ** of both the scalar and, if created, the virtual table.
- */
- int sqlite3Fts3InitHashTable(
- sqlite3 *db,
- fts3Hash *pHash,
- const char *zName
- ){
- int rc = SQLITE_OK;
- void *p = (void *)pHash;
- const int any = SQLITE_ANY;
- char *zTest = 0;
- char *zTest2 = 0;
- #ifdef SQLITE_TEST
- void *pdb = (void *)db;
- zTest = sqlite3_mprintf("%s_test", zName);
- zTest2 = sqlite3_mprintf("%s_internal_test", zName);
- if( !zTest || !zTest2 ){
- rc = SQLITE_NOMEM;
- }
- #endif
- if( rc!=SQLITE_OK
- || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
- || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
- #ifdef SQLITE_TEST
- || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
- || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
- || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
- #endif
- );
- sqlite3_free(zTest);
- sqlite3_free(zTest2);
- return rc;
- }
- #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */