开通博客赚积分
发布资源赚积分
分类

源码开发语言/平台
当前位置：首页 > 源码/资料 > 多国语言处理 > 查看源码
Span.cpp
资源名称：ictclas_Source_Code.zip　[点击查看]
上传用户：chen_dj
上传日期：2013-04-22
资源大小：111k
文件大小：27k
源码类别：

多国语言处理

开发平台：

C/C++

Span.cpp：源码内容
  1. /****************************************************************************
  2.  *
  3.  * Copyright (c) 2000, 2001 
  4.  *     Machine Group
  5.  *     Software Research Lab.
  6.  *     Institute of Computing Tech.
  7.  *     Chinese Academy of Sciences
  8.  *     All rights reserved.
  9.  *
  10.  * This file is the confidential and proprietary property of 
  11.  * Institute of Computing Tech. and the posession or use of this file requires 
  12.  * a written license from the author.
  13.  * Filename: Span.cpp
  14.  * Abstract:
  15.  *           implementation of the CSpan class.
  16.  * Author:   Kevin Zhang 
  17.  *          (zhanghp@software.ict.ac.cn)
  18.  * Date:     2002-4-23
  19.  *
  20.  * Notes:    Tagging with Hidden Markov Model
  21.  *                
  22.  ****************************************************************************/
  24. #include "stdafx.h"
  25. #include "Span.h"
  26. #include "..\Segment\Segment.h"
  27. #include "..\Utility\Utility.h"
  28. #include <math.h>
  29. #include <string.h>
  30. #include <stdio.h>
  31. #include <time.h>
  32. //////////////////////////////////////////////////////////////////////
  33. // Construction/Destruction
  34. //////////////////////////////////////////////////////////////////////
  35. CSpan::CSpan()
  36. {
  37. if(m_tagType!=TT_NORMAL)
  38.       m_nTags[0][0]=100;//Begin tag
  39. else
  40.       m_nTags[0][0]=0;//Begin tag
  42. m_nTags[0][1]=-1;
  43. m_dFrequency[0][0]=0;
  44. m_nCurLength=1;
  45. m_nUnknownIndex=0;
  46. m_nStartPos=0;
  47. m_nWordPosition[1]=0;
  48. m_sWords[0][0]=0;
  50. m_tagType=TT_NORMAL;//Default tagging type
  51. }
  53. CSpan::~CSpan()
  54. {
  55. }
  57. bool CSpan::Disamb()
  58. {
  59. int i,j,k,nMinCandidate;
  60. double dMinFee,dTmp;
  61. for(i=1;i<m_nCurLength;i++)//For every word
  62. {
  63. for(j=0;m_nTags[i][j]>=0;j++)//For every word
  64. {
  65. nMinCandidate=MAX_POS_PER_WORD+1;
  66. for(k=0;m_nTags[i-1][k]>=0;k++)
  67. {
  68. //ConvertPOS(m_nTags[i-1][k],&nKey,&nPrevPOS);
  69. //ConvertPOS(m_nTags[i][j],&nKey,&nCurPOS);
  70. //dTmp=m_context.GetContextPossibility(nKey,nPrevPOS,nCurPOS);
  71. dTmp=-log(m_context.GetContextPossibility(0,m_nTags[i-1][k],m_nTags[i][j]));
  72. dTmp+=m_dFrequency[i-1][k];//Add the fees
  73. if(nMinCandidate>10||dTmp<dMinFee)//Get the minimum fee
  74. {
  75. nMinCandidate=k;
  76. dMinFee=dTmp;
  77. }
  78. }
  79. m_nBestPrev[i][j]=nMinCandidate;//The best previous for j
  80. m_dFrequency[i][j]=m_dFrequency[i][j]+dMinFee;
  81. }
  82. }
  84. return true;
  85. }
  87. bool CSpan::Reset(bool bContinue)
  88. {
  89. if(!bContinue)
  90. {//||CC_Find("。！”〕〉》」〗】",m_sWords[m_nCurLength-1])
  91. if(m_tagType!=TT_NORMAL)//Get the last POS in the last sentence
  92.       m_nTags[0][0]=100;//Begin tag
  93. else
  94.       m_nTags[0][0]=0;//Begin tag
  95. m_nUnknownIndex=0;
  96. m_dFrequency[0][0]=0;
  97. m_nStartPos=0;
  98. }
  99. else
  100. {
  101. m_nTags[0][0]=m_nTags[m_nCurLength-1][0];//Get the last POS in the last sentence
  102. m_dFrequency[0][0]=m_dFrequency[m_nCurLength-1][0];
  103. }
  104.     m_nTags[0][1]=-1;//Get the last POS in the last sentence,set the -1 as end flag
  105. m_nCurLength=1;
  106. m_nWordPosition[1]=m_nStartPos;
  107. m_sWords[0][0]=0;
  108. return true;
  109. }
  112. bool CSpan::LoadContext(char *sFilename)
  113. {
  114. return m_context.Load(sFilename);
  115. }
  116. bool CSpan::UnknownMatch()
  117. {//Find the template
  118.   char sPOS[MAX_WORDS_PER_SENTENCE]="z";
  119.   int nStart,nEnd;
  120.   for(int i=1;m_nBestTag[i]>-1;i++)
  121. sPOS[i]=m_nBestTag[i]+'a';
  122.   sPOS[i]=0;
  123.   char *pFind=strchr(sPOS+1,'b');
  124.   char *pFindEnd;
  126.   while(pFind!=NULL&&*pFind!=NULL)
  127.   {
  128.     pFindEnd=pFind+1;
  129.     while(pFindEnd!=NULL&&*pFindEnd!=NULL&&(*pFindEnd=='c'||*pFindEnd=='d'))
  130.        pFindEnd=pFindEnd+1;
  131.     nStart=pFind-sPOS;
  132.     nEnd=pFindEnd-sPOS;
  133. //Save the unknown word position
  134.     m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[nStart];
  135. m_nUnknownWords[m_nUnknownIndex++][1]=m_nWordPosition[nEnd];
  136. pFind=strchr(pFindEnd,'b');
  137.   }
  138.   return true;
  139. }
  141. bool CSpan::GetBestPOS()
  142. {
  143.   Disamb();
  144.   for(int i=m_nCurLength-1,j=0;i>0;i--)
  145.   {
  146.  if(m_sWords[i][0])
  147.  {//Not virtual ending
  148.  m_nBestTag[i]=m_nTags[i][j];//Record the best POS and its possibility
  149.  }
  150.  j=m_nBestPrev[i][j];
  151.   }
  152.   int nEnd=m_nCurLength;//Set the end of POS tagging
  153.   if(m_sWords[m_nCurLength-1][0]==0)
  154.   nEnd=m_nCurLength-1;
  155.   m_nBestTag[nEnd]=-1;
  156.   return true;
  157. }
  159. bool CSpan::SplitPersonPOS(CDictionary &unlistDict)
  160. {//Split the word with POS 21 and 22
  161.     int i=m_nCurLength-1,j;
  162. unsigned int nLenWord,nLenPart;
  163. char sFirstPart[50],sLastPart[50];
  164. int nFirstPOS,nLastPOS;
  165. for(;i>0;i--)
  166. {
  167. if(m_nBestTag[i]==21||m_nBestTag[i]==22)
  168. {//Find the POS which need to split
  169. for(j=m_nCurLength-1;j>i;j--)
  170. {//Move the POS and words
  171. strcpy(m_sWords[j+1],m_sWords[j]);
  172. m_nBestTag[j+1]=m_nBestTag[j];
  173. m_nWordPosition[j+1]=m_nWordPosition[j];
  174. }
  175. m_nCurLength+=1;//The length increment 
  176.         /*
  177. CSegment segment;
  178. segment.Segment(m_sWords[i],unlistDict,1);
  179. */
  180. //Generate new segment words and POS
  181. if(m_nBestTag[i]==21)
  182. {//Combination by Previous and first component
  183. nLenWord=strlen(m_sWords[i]);
  184. if(nLenWord>4)//Get first component
  185. {
  186. strcpy(sLastPart,m_sWords[i]+nLenWord-4);
  187. if(!unlistDict.IsExist(sLastPart,-1))
  188. strcpy(sLastPart,m_sWords[i]+nLenWord-2);
  189. }
  190. else
  191. {
  192. strcpy(sLastPart,m_sWords[i]+nLenWord-2);
  193. }
  194. nLenPart=strlen(sLastPart);
  195. if(nLenPart<nLenWord)
  196. {//Get first part
  197. strncpy(sFirstPart,m_sWords[i],nLenWord-nLenPart);
  198. sFirstPart[nLenWord-nLenPart]=0;
  199. }
  200. else
  201. {
  202. strncpy(sFirstPart,m_sWords[i],nLenWord-2);
  203. sFirstPart[nLenWord-2]=0;
  204. strncpy(sLastPart,m_sWords[i]+nLenWord-2,2);
  205. sLastPart[2]=0;
  206. }
  207. nFirstPOS=11;
  208. nLastPOS=1;
  209. }
  210. else
  211. {//Combination by Next word and last component
  212. nLenWord=strlen(m_sWords[i]);
  213. if(nLenWord>4)//Get last component
  214. {
  215. strncpy(sFirstPart,m_sWords[i],4);
  216. sFirstPart[4]=0;
  217. if(!unlistDict.IsExist(sFirstPart,-1))
  218. sFirstPart[2]=0;
  219. }
  220. else
  221. {
  222. strncpy(sFirstPart,m_sWords[i],2);
  223. sFirstPart[2]=0;
  224. }
  225. nLenPart=strlen(sFirstPart);
  226. if(nLenPart<nLenWord)
  227. {//Get first part
  228. strncpy(sLastPart,m_sWords[i]+nLenPart,nLenWord-nLenPart);
  229. sLastPart[nLenWord-nLenPart]=0;
  230. }
  231. else
  232. {
  233. strncpy(sFirstPart,m_sWords[i],2);
  234. sFirstPart[2]=0;
  235. strncpy(sLastPart,m_sWords[i]+2,nLenWord-2);
  236. sLastPart[nLenWord-2]=0;
  237. }
  238. if(unlistDict.IsExist(sFirstPart,1)&&m_nBestTag[i-1]==5)
  239. //小陈说：
  240. nFirstPOS=1;
  241. else if(unlistDict.IsExist(m_sWords[i-1],1)&&!unlistDict.IsExist(m_sWords[i-2],1))
  242. nFirstPOS=4;
  243. else
  244. nFirstPOS=3;
  245. nLastPOS=12;
  246. }
  247.             strcpy(m_sWords[i],sFirstPart);
  248. m_nBestTag[i]=nFirstPOS;
  249.             strcpy(m_sWords[i+1],sLastPart);
  250. m_nBestTag[i+1]=nLastPOS;
  251. m_nWordPosition[i+1]=m_nWordPosition[i]+strlen(sFirstPart);
  252. }
  253. }
  254. return true;
  255. }
  257. bool CSpan::PersonRecognize(CDictionary &personDict)
  258. {
  259.   char sPOS[MAX_WORDS_PER_SENTENCE]="z",sPersonName[100];
  260.                           //0     1    2    3    4   5   
  261.   char sPatterns[][5]={ "BBCD","BBC","BBE","BBZ","BCD","BEE","BE",
  262.  "BG",  "BXD","BZ", "CDCD","CD","EE", 
  263.  "FB", "Y","XD",""};
  264.   double dFactor[]={0.0011,0.0011,0.0011,0.0011,0.7614,0.0011,0.2055,
  265.  0.0160,0.0011,0.0011,0,0.0160,0.0011,
  266.  0.0160,0.0011,0.0011,0
  267.   };
  268.   //About parameter:
  269. /*
  270. Given Name: 486     0.0160
  271. Surname+postfix:484 0.0160
  272. m_lPerson2Num:6265   0.2055
  273. m_lPerson3Num: 23184 0.7614
  274. m_lPerson4Num:32     0.0011
  275.   */
  276.   //The person recognition patterns set
  277.   //BBCD:姓+姓+名1+名2;
  278.   //BBE: 姓+姓+单名;
  279.   //BBZ: 姓+姓+双名成词;
  280.   //BCD: 姓+名1+名2;
  281.   //BE:  姓+单名;
  282.   //BEE: 姓+单名+单名;韩磊磊
  283.   //BG:  姓+后缀
  284.   //BXD: 姓+姓双名首字成词+双名末字
  285.   //BZ:  姓+双名成词;
  286.   //B:  姓
  287.   //CD:  名1+名2;
  288.   //EE:  单名+单名;
  289.   //FB:  前缀+姓
  290.   //XD:  姓双名首字成词+双名末字
  291.   //Y:   姓单名成词
  292.   int nPatternLen[]={4,3,3,3,3,3,2,2,3,2,4,2,2,2,1,2,0};
  294.   for(int i=1;m_nBestTag[i]>-1;i++)//Convert to string from POS
  295. sPOS[i]=m_nBestTag[i]+'A';
  296.   sPOS[i]=0;
  297.   int j=1,k,nPos;//Find the proper pattern from the first POS
  298.   int nLittleFreqCount;//Counter for the person name role with little frequecy
  299.   bool bMatched=false;   
  300.   while(j<i)
  301.   {
  302. bMatched=false;   
  303. for(k=0;!bMatched&&nPatternLen[k]>0;k++)
  304. {
  305. if(strncmp(sPatterns[k],sPOS+j,nPatternLen[k])==0&&strcmp(m_sWords[j-1],"·")!=0&&strcmp(m_sWords[j+nPatternLen[k]],"·")!=0)
  306. {//Find the proper pattern k
  307. if(strcmp(sPatterns[k],"FB")==0&&(sPOS[j+2]=='E'||sPOS[j+2]=='C'||sPOS[j+2]=='G'))
  308. {//Rule 1 for exclusion:前缀+姓+名1(名2): 规则(前缀+姓)失效；
  309. continue;
  310. }
  311. /* if((strcmp(sPatterns[k],"BEE")==0||strcmp(sPatterns[k],"EE")==0)&&strcmp(m_sWords[j+nPatternLen[k]-1],m_sWords[j+nPatternLen[k]-2])!=0)
  312. {//Rule 2 for exclusion:姓+单名+单名:单名+单名 若EE对应的字不同，规则失效.如：韩磊磊
  313. continue;
  314. }
  316. if(strcmp(sPatterns[k],"B")==0&&m_nBestTag[j+1]!=12)
  317. {//Rule 3 for exclusion: 若姓后不是后缀，规则失效.如：江主席、刘大娘
  318. continue;
  319. }
  320. */ //Get the possible name
  321. nPos=j;//Record the person position in the tag sequence
  322. sPersonName[0]=0;
  323. nLittleFreqCount=0;//Record the number of role with little frequency
  324. while(nPos<j+nPatternLen[k])
  325. {//Get the possible person name
  326.  //
  327. if(m_nBestTag[nPos]<4&&personDict.GetFrequency(m_sWords[nPos],m_nBestTag[nPos])<LITTLE_FREQUENCY)
  328. nLittleFreqCount++;//The counter increase
  329. strcat(sPersonName,m_sWords[nPos]);
  330. nPos+=1;
  331. }
  332. if(IsAllForeign(sPersonName)&&personDict.GetFrequency(m_sWords[j],1)<LITTLE_FREQUENCY)
  333. {//Exclusion foreign name
  334.  //Rule 2 for exclusion:若均为外国人名用字 规则(名1+名2)失效
  335. j+=nPatternLen[k]-1;
  336. continue;
  337. }
  338. if(strcmp(sPatterns[k],"CDCD")==0)
  339. {//Rule for exclusion
  340.  //规则(名1+名2+名1+名2)本身是排除规则:女高音歌唱家迪里拜尔演唱
  341.    //Rule 3 for exclusion:含外国人名用字 规则适用
  342.  //否则，排除规则失效:黑妞白妞姐俩拔了头筹。
  343. if(GetForeignCharCount(sPersonName)>0)
  344. j+=nPatternLen[k]-1;
  345. continue;
  346. }
  347. if(strcmp(sPatterns[k],"CD")==0&&IsAllForeign(sPersonName))
  348. {//
  349. j+=nPatternLen[k]-1;
  350. continue;
  351. }
  352. if(nLittleFreqCount==nPatternLen[k]||nLittleFreqCount==3)
  353. //马哈蒂尔;小扎耶德与他的中国阿姨胡彩玲受华黎明大使之邀，
  354. //The all roles appear with two lower frequecy,we will ignore them
  355. continue;
  356. m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[j];
  357. m_nUnknownWords[m_nUnknownIndex][1]=m_nWordPosition[j+nPatternLen[k]];
  358. m_dWordsPossibility[m_nUnknownIndex]=log(dFactor[k])+ComputePossibility(j,nPatternLen[k],personDict);
  359. //Mutiply the factor 
  360. m_nUnknownIndex+=1;
  361. j+=nPatternLen[k];
  362. bMatched=true;
  363. }
  364. }
  365.     if(!bMatched)//Not matched, add j by 1
  366. j+=1;
  367.   }
  368.   return true;
  369. }
  371. int CSpan::GetFrom(PWORD_RESULT pWordItems,int nIndex,CDictionary &dictCore, CDictionary &dictUnknown)
  372. {
  373. int nCount,aPOS[MAX_POS_PER_WORD],aFreq[MAX_POS_PER_WORD];
  374. int nFreq=0,j,nRetPos=0,nWordsIndex=0;
  375. bool bSplit=false;//Need to split in Transliteration recognition 
  376.     int i=1;
  377. nWordsIndex=i+nIndex-1;
  378. for(;i<MAX_WORDS_PER_SENTENCE&&pWordItems[nWordsIndex].sWord[0]!=0;i++)
  379. {
  380. if(m_tagType==TT_NORMAL||!dictUnknown.IsExist(pWordItems[nWordsIndex].sWord,44))
  381.         {
  382. strcpy(m_sWords[i],pWordItems[nWordsIndex].sWord);//store current word
  383.         m_nWordPosition[i+1]=m_nWordPosition[i]+strlen(m_sWords[i]);
  384. }
  385. else
  386. {
  387. if(!bSplit)
  388. {
  389. strncpy(m_sWords[i],pWordItems[nWordsIndex].sWord,2);//store current word
  390. m_sWords[i][2]=0;
  391. bSplit=true;
  392. }
  393. else
  394. {
  395. unsigned int nLen=strlen(pWordItems[nWordsIndex].sWord+2);
  396. strncpy(m_sWords[i],pWordItems[nWordsIndex].sWord+2,nLen);//store current word
  397. m_sWords[i][nLen]=0;
  398. bSplit=false;
  399. }
  400.         m_nWordPosition[i+1]=m_nWordPosition[i]+strlen(m_sWords[i]);
  401. }
  402. //Record the position of current word
  403. m_nStartPos=m_nWordPosition[i+1];
  404. //Move the Start POS to the ending
  405. if(m_tagType!=TT_NORMAL)
  406. {
  407. //Get the POSs from the unknown recognition dictionary
  408. dictUnknown.GetHandle(m_sWords[i],&nCount,aPOS,aFreq);
  409. for(j=0;j<nCount;j++) 
  410. {//Get the POS set of sCurWord in the unknown dictionary
  411. m_nTags[i][j]=aPOS[j];
  412.     m_dFrequency[i][j]=-log((double)(1+aFreq[j]))+log((double)(m_context.GetFrequency(0,aPOS[j])+1));
  413. }
  414. //Get the POS set of sCurWord in the core dictionary
  415. //We ignore the POS in the core dictionary and recognize them as other (0).
  416. //We add their frequency to get the possibility as POS 0
  417. dictCore.GetHandle(m_sWords[i],&nCount,aPOS,aFreq);
  418. nFreq=0;
  419. for(int k=0;k<nCount;k++) 
  420. {
  421. nFreq+=aFreq[k];
  422. }
  423. if(nCount>0)
  424. {
  425. m_nTags[i][j]=0;
  426. //m_dFrequency[i][j]=(double)(1+nFreq)/(double)(m_context.GetFrequency(0,0)+1);
  427. m_dFrequency[i][j]=-log((double)(1+nFreq))+log((double)(m_context.GetFrequency(0,0)+1));
  428. j++;
  429. }
  430. }
  431. else//For normal POS tagging
  432. {
  433. j=0;
  434. //Get the POSs from the unknown recognition dictionary
  435. if(pWordItems[nWordsIndex].nHandle>0)
  436. {//The word has  is only one POS value
  437.  //We have record its POS and nFrequncy in the items.
  438. m_nTags[i][j]=pWordItems[nWordsIndex].nHandle;
  439. m_dFrequency[i][j]=pWordItems[nWordsIndex].dValue-log(MAX_FREQUENCE)+log((double)(m_context.GetFrequency(0,m_nTags[i][j])+1));
  440. if(m_dFrequency[i][j]<0)//Not permit the value less than 0
  441. m_dFrequency[i][j]=0;
  442. j++;
  443. }
  444. else
  445. {//The word has multiple POSs, we should retrieve the information from Core Dictionary 
  447. if(pWordItems[nWordsIndex].nHandle<0)
  448. {//The word has  is only one POS value
  449.  //We have record its POS and nFrequncy in the items.
  450. if(pWordItems[nWordsIndex].nHandle==-'t'*256-'t')//tt
  451. {
  452. char sWordOrg[100],sPostfix[10];
  453. double dRatio=0.6925;//The ratio which transliteration as a person name 
  454. PostfixSplit(pWordItems[nWordsIndex].sWord,sWordOrg,sPostfix);
  455. if(sPostfix[0]!=0)
  456. dRatio=0.01;
  457. m_nTags[i][j]='n'*256+'r';
  458. m_dFrequency[i][j]=-log(dRatio)+pWordItems[nWordsIndex].dValue;
  459. //m_dFrequency[i][j]=log(dRatio)+pWordItems[nWordsIndex].dValue-log(m_context.GetFrequency(0,m_nTags[i][j]))+log(MAX_FREQUENCE);
  460. //P(W|R)=P(WRT)/P(RT)=P(R)*P(W|T)/P(R|T)
  461. j++;
  462. m_nTags[i][j]='n'*256+'s';
  463. m_dFrequency[i][j]=-log(1-dRatio)+pWordItems[nWordsIndex].dValue;
  464. //m_dFrequency[i][j]=log(1-dRatio)+pWordItems[nWordsIndex].dValue-log(m_context.GetFrequency(0,m_nTags[i][j]))+log(MAX_FREQUENCE);
  465. j++;
  466. }
  467. else//Unknown words such as Chinese person name or place name
  468. {
  469. m_nTags[i][j]=-pWordItems[nWordsIndex].nHandle;
  470.     // m_dFrequency[i][j++]=(double)(1+pWordItems[nWordsIndex].nFrequency)/(double)(m_context.GetFrequency(0,aPOS[j])+1);
  471. m_dFrequency[i][j++]=pWordItems[nWordsIndex].dValue;
  472. }
  473. }
  474. dictCore.GetHandle(m_sWords[i],&nCount,aPOS,aFreq);
  475. for(;j<nCount;j++) 
  476. {//Get the POS set of sCurWord in the unknown dictionary
  477. m_nTags[i][j]=aPOS[j];
  478.     m_dFrequency[i][j]=-log(1+aFreq[j])+log(m_context.GetFrequency(0,m_nTags[i][j])+1);
  479. }
  480. }
  481. }
  482. if(j==0)
  483. {//We donot know the POS, so we have to guess them according lexical knowledge
  484. GuessPOS(i,&j);//Guess the POS of current word
  485. }
  486. m_nTags[i][j]=-1;//Set the ending POS 
  487. if(j==1)//No ambuguity
  488. {//No ambuguity, so we can break from the loop
  489. i++;
  490. m_sWords[i][0]=0;
  491. break;
  492. }
  493. if(!bSplit)
  494. nWordsIndex++;
  495. }
  496. if(pWordItems[nWordsIndex].sWord[0]==0)
  497. nRetPos=-1;//Reaching ending
  499. if(m_nTags[i-1][1]!=-1)//||m_sWords[i][0]==0
  500. {//Set end for words like "张/华/平"
  501. if(m_tagType!=TT_NORMAL)
  502.        m_nTags[i][0]=101;
  503. else
  504.        m_nTags[i][0]=1;
  506. m_dFrequency[i][0]=0;
  507.     m_sWords[i][0]=0;//Set virtual ending
  508. m_nTags[i++][1]=-1;
  509. }
  510. m_nCurLength=i;//The current word count
  511. if(nRetPos!=-1)
  512. return nWordsIndex+1;//Next start position
  513. return -1;//Reaching ending
  514. }
  517. //Set the tag type
  518. void CSpan::SetTagType(enum TAG_TYPE  nType)
  519. {
  520. m_tagType=nType;
  521. }
  522. //POS tagging with Hidden Markov Model
  523. bool CSpan::POSTagging(PWORD_RESULT pWordItems,CDictionary &dictCore,CDictionary &dictUnknown)
  524. {
  525. //pWordItems: Items; nItemCount: the count of items;core dictionary and unknown recognition dictionary
  526.     int i=0,j,nStartPos;
  527. Reset(false);
  528.     while(i>-1&&pWordItems[i].sWord[0]!=0)
  529. {
  530. nStartPos=i;//Start Position
  531. i=GetFrom(pWordItems,nStartPos,dictCore,dictUnknown);
  532. GetBestPOS();
  533. switch(m_tagType)
  534. {
  535. case TT_NORMAL://normal POS tagging
  536. j=1;
  537. while(m_nBestTag[j]!=-1&&j<m_nCurLength)
  538. {//Store the best POS tagging
  539. pWordItems[j+nStartPos-1].nHandle=m_nBestTag[j];
  540. //Let 。be 0
  541. if(pWordItems[j+nStartPos-1].dValue>0&&dictCore.IsExist(pWordItems[j+nStartPos-1].sWord,-1))//Exist and update its frequncy as a POS value
  542. pWordItems[j+nStartPos-1].dValue=log(MAX_FREQUENCE)-log(dictCore.GetFrequency(pWordItems[j+nStartPos-1].sWord,m_nBestTag[j])+1);
  543. j+=1;
  544. }
  545. break;
  546. case TT_PERSON://Person recognition
  547. /*clock_t lStart,lEnd;
  548.     lStart=clock();
  549. */
  550. SplitPersonPOS(dictUnknown);
  551. //lEnd=clock();
  552. //printf("SplitPersonPOS=%fn",(double)(lEnd-lStart)*1000/CLOCKS_PER_SEC);
  553. //Spit Persons POS
  554. //lStart=clock();
  555. PersonRecognize(dictUnknown);
  556. //lEnd=clock();
  557. //printf("PersonRecognize=%fn",(double)(lEnd-lStart)/CLOCKS_PER_SEC);
  558. //Person Recognition with the person recognition dictionary
  559. break;
  560. case TT_PLACE://Place name recognition
  561. PlaceRecognize(dictCore,dictUnknown);
  562. break;
  563. case TT_TRANS://Transliteration
  564. TransRecognize(dictCore,dictUnknown);
  565. break;
  566. default:
  567. break;
  568. }
  569. Reset();
  570. }
  571. return true;
  572. }
  573. //Guess the POS of No. nIndex word item
  574. bool CSpan::GuessPOS(int nIndex,int *pSubIndex)
  575. {
  576. int j=0,i=nIndex,nCharType;
  577. unsigned int nLen;
  578. switch(m_tagType)
  579. {
  580. case TT_NORMAL:
  581. break;
  582. case TT_PERSON:
  583. j=0;
  584. if(CC_Find("××",m_sWords[nIndex]))
  585. {
  586. m_nTags[i][j]=6;
  587. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,6)+1);
  588. }
  589. else
  590. {
  591. m_nTags[i][j]=0;
  592. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  593. nLen=strlen(m_sWords[nIndex]);
  594. if(nLen>=4)
  595. {
  596. m_nTags[i][j]=0;
  597. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  598. m_nTags[i][j]=11;
  599. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  600. m_nTags[i][j]=12;
  601. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  602. m_nTags[i][j]=13;
  603. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  604. }
  605. else if(nLen==2)
  606. {
  607. m_nTags[i][j]=0;
  608. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  609. nCharType=charType((unsigned char *)m_sWords[nIndex]);
  610. if(nCharType==CT_OTHER||nCharType==CT_CHINESE)
  611. {
  612. m_nTags[i][j]=1;
  613. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)+1);
  614. m_nTags[i][j]=2;
  615. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)+1);
  616. m_nTags[i][j]=3;
  617. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)+1);
  618. m_nTags[i][j]=4;
  619. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,4)+1);
  620. }
  621. m_nTags[i][j]=11;
  622. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  623. m_nTags[i][j]=12;
  624. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  625. m_nTags[i][j]=13;
  626. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  627. }
  628. }
  629. break;
  630. case TT_PLACE:
  631. j=0;
  632. m_nTags[i][j]=0;
  633. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  634. nLen=strlen(m_sWords[nIndex]);
  635. if(nLen>=4)
  636. {
  637. m_nTags[i][j]=11;
  638. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  639. m_nTags[i][j]=12;
  640. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  641. m_nTags[i][j]=13;
  642. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  643. }
  644. else if(nLen==2)
  645. {
  646. m_nTags[i][j]=0;
  647. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  648. nCharType=charType((unsigned char *)m_sWords[nIndex]);
  649. if(nCharType==CT_OTHER||nCharType==CT_CHINESE)
  650. {
  651. m_nTags[i][j]=1;
  652. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)+1);
  653. m_nTags[i][j]=2;
  654. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)+1);
  655. m_nTags[i][j]=3;
  656. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)+1);
  657. m_nTags[i][j]=4;
  658. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,4)+1);
  659. }
  660. m_nTags[i][j]=11;
  661. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  662. m_nTags[i][j]=12;
  663. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  664. m_nTags[i][j]=13;
  665. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  666. }
  667. break;
  668. case TT_TRANS:
  669. j=0;
  670. nLen=strlen(m_sWords[nIndex]);
  672. m_nTags[i][j]=0;
  673. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  675. if(!IsAllChinese((unsigned char *)m_sWords[nIndex]))
  676. {
  677. if(IsAllLetter((unsigned char *)m_sWords[nIndex]))
  678. {
  679. m_nTags[i][j]=1;
  680. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)+1);
  681. m_nTags[i][j]=11;
  682. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)+1);
  683. /* }
  684. if(IsAllNum((unsigned char *)m_sWords[nIndex])||IsAllLetter((unsigned char *)m_sWords[nIndex]))
  685. {
  686. */ m_nTags[i][j]=2;
  687. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)*2+1);
  688. m_nTags[i][j]=3;
  689. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)*2+1);
  690. m_nTags[i][j]=12;
  691. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*2+1);
  692. m_nTags[i][j]=13;
  693. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*2+1);
  694. }
  695. m_nTags[i][j]=41;
  696. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,41)*8);
  697. m_nTags[i][j]=42;
  698. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,42)*8);
  699. m_nTags[i][j]=43;
  700. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,43)*8);
  701. }
  702. else if(nLen>=4)
  703. {
  704. m_nTags[i][j]=41;
  705. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,41)*8);
  706. m_nTags[i][j]=42;
  707. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,42)*8);
  708. m_nTags[i][j]=43;
  709. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,43)*8);
  710. }
  711. else if(nLen==2)
  712. {
  713. nCharType=charType((unsigned char *)m_sWords[nIndex]);
  714. if(nCharType==CT_OTHER||nCharType==CT_CHINESE)
  715. {
  716. m_nTags[i][j]=1;
  717. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)*2+1);
  718. m_nTags[i][j]=2;
  719. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)*2+1);
  720. m_nTags[i][j]=3;
  721. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)*2+1);
  722. m_nTags[i][j]=30;
  723. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,30)*8+1);
  724. m_nTags[i][j]=11;
  725. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*4+1);
  726. m_nTags[i][j]=12;
  727. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*4+1);
  728. m_nTags[i][j]=13;
  729. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*4+1);
  730. m_nTags[i][j]=21;
  731. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,21)*2+1);
  732. m_nTags[i][j]=22;
  733. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,22)*2+1);
  734. m_nTags[i][j]=23;
  735. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,23)*2+1);
  736. }
  737. m_nTags[i][j]=41;
  738. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,41)*8);
  739. m_nTags[i][j]=42;
  740. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,42)*8);
  741. m_nTags[i][j]=43;
  742. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,43)*8);
  743. }
  744. break;
  745. default:
  746. break;
  747. }
  748. *pSubIndex=j;
  749. return true;
  750. }
  752. ELEMENT_TYPE  CSpan::ComputePossibility(int nStartPos,int nLength,CDictionary &dict)
  753. {
  754. ELEMENT_TYPE dRetValue=0,dPOSPoss;
  755. //dPOSPoss: the possibility of a POS appears
  756. //dContextPoss: The possibility of context POS appears
  757. int nFreq;
  758. for(int i=nStartPos;i<nStartPos+nLength;i++)
  759. {
  760. nFreq=dict.GetFrequency(m_sWords[i],m_nBestTag[i]);
  761. //nFreq is word being the POS
  762. dPOSPoss=log((double)(m_context.GetFrequency(0,m_nBestTag[i])+1))-log((double)(nFreq+1));
  763. dRetValue+=dPOSPoss;
  764. /* if(i<nStartPos+nLength-1)
  765. {
  766. dContextPoss=log((double)(m_context.GetContextPossibility(0,m_nBestTag[i],m_nBestTag[i+1])+1));
  767. dRetValue+=dPOSPoss-dContextPoss;
  768. }
  769. */ }
  770. return dRetValue;
  771. }
  772. bool CSpan::TransRecognize(CDictionary &dictCore,CDictionary &transDict)
  773. {
  774.   char sPOS[MAX_WORDS_PER_SENTENCE]="Z";
  775.   int nStart=1,nEnd=1,i=1;
  776.   while(m_nBestTag[i]>-1)
  777.   {
  778.   if(m_nBestTag[i]==1||m_nBestTag[i]==11||m_nBestTag[i]==21)//1,11,21 Trigger the recognition
  779.   {
  780. nStart=i;
  781. nEnd=nStart+1;
  782. while(m_nBestTag[nEnd]==m_nBestTag[nStart])//1,11,21
  783. nEnd++;
  784. while(m_nBestTag[nEnd]==m_nBestTag[nStart]+1)//2,12,22
  785. nEnd++;
  786. while(m_nBestTag[nEnd]==m_nBestTag[nStart]+2)//3,13,23
  787. nEnd++;
  788. while(m_nBestTag[nEnd]==30)//3,13,23
  789. nEnd++;
  790.   }
  791.   else if(m_nBestTag[i]==2||m_nBestTag[i]==12||m_nBestTag[i]==22)//1,11,21 Trigger the recognition
  792.   {
  793. nStart=i;
  794. nEnd=nStart+1;
  795. while(m_nBestTag[nEnd]==m_nBestTag[nStart])//2,12,22
  796. nEnd++;
  797. while(m_nBestTag[nEnd]==m_nBestTag[nStart]+1)//2,12,22
  798. nEnd++;
  799. while(m_nBestTag[nEnd]==30)//3,13,23
  800. nEnd++;
  801.   }
  802.   if(nEnd>nStart&&!IsAllNum((unsigned char *)m_sWords[nStart])&&(nEnd>nStart+2||(nEnd==nStart+2&&(m_nBestTag[nEnd-1]!=30||strlen(m_sWords[nStart])>2))||(nEnd==nStart+1&&strlen(m_sWords[nStart])>2&&!dictCore.IsExist(m_sWords[nStart],-1))))
  803.   {
  804. m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[nStart];
  805. m_nUnknownWords[m_nUnknownIndex][1]=m_nWordPosition[nEnd];
  806. m_dWordsPossibility[m_nUnknownIndex++]=ComputePossibility(nStart,nEnd-nStart+1,transDict);
  807. nStart=nEnd;
  808.   }
  810.   if(i<nEnd)
  811.   i=nEnd;
  812.   else
  813.   i=i+1;
  814.   }
  815.   return true;
  816. }
  817. bool CSpan::PlaceRecognize(CDictionary &dictCore,CDictionary &placeDict)
  818. {
  819.   int nStart=1,nEnd=1,i=1;
  820.   while(m_nBestTag[i]>-1)
  821.   {
  822.   if(m_nBestTag[i]==1)//1 Trigger the recognition procession
  823.   {
  824. nStart=i;
  825. nEnd=nStart+1;
  826. while(m_nBestTag[nEnd]==1)//
  827. nEnd++;
  828. while(m_nBestTag[nEnd]==2)//2,12,22
  829. nEnd++;
  830. while(m_nBestTag[nEnd]==3)
  831. nEnd++;
  832. while(m_nBestTag[nEnd]==4)
  833. nEnd++;   
  834.   }
  835.   else if(m_nBestTag[i]==2)//1,11,21 Trigger the recognition
  836.   {
  837. nStart=i;
  838. nEnd=nStart+1;
  839. while(m_nBestTag[nEnd]==2)//2
  840. nEnd++;
  841. while(m_nBestTag[nEnd]==3)//2
  842. nEnd++;
  843. while(m_nBestTag[nEnd]==4)//2
  844. nEnd++;
  845.   }
  846.   if(nEnd>nStart)
  847.   {
  848. m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[nStart];
  849. m_nUnknownWords[m_nUnknownIndex][1]=m_nWordPosition[nEnd];
  850. m_dWordsPossibility[m_nUnknownIndex++]=ComputePossibility(nStart,nEnd-nStart+1,placeDict);
  851. nStart=nEnd;
  852.   }
  854.   if(i<nEnd)
  855.   i=nEnd;
  856.   else
  857.   i=i+1;
  858.   }
  859.   return true;
  860. }