开通博客赚积分
发布资源赚积分
分类

源码开发语言/平台
当前位置：首页 > 源码/资料 > 多国语言处理 > 查看源码
Span.cpp
资源名称：ICTCLASV1.2.rar　[点击查看]
上传用户：sunyong76
上传日期：2021-10-03
资源大小：2236k
文件大小：29k
源码类别：

多国语言处理

开发平台：

Java

Span.cpp：源码内容
  1. //////////////////////////////////////////////////////////////////////
  2. //ICTCLAS简介：计算所汉语词法分析系统ICTCLAS(Institute of Computing Technology, Chinese Lexical Analysis System)，
  3. //             功能有：中文分词；词性标注；未登录词识别。
  4. //             分词正确率高达97.58%(973专家评测结果)，
  5. //             未登录词识别召回率均高于90%，其中中国人名的识别召回率接近98%;
  6. //             处理速度为31.5Kbytes/s。
  7. //著作权：  Copyright?2002-2005中科院计算所 职务著作权人：张华平 刘群
  8. //遵循协议：自然语言处理开放资源许可证1.0
  9. //Email: zhanghp@software.ict.ac.cn
  10. //Homepage:www.nlp.org.cn;mtgroup.ict.ac.cn
  11. /****************************************************************************
  12.  *
  13.  * Copyright (c) 2000, 2001 
  14.  *     Machine Group
  15.  *     Software Research Lab.
  16.  *     Institute of Computing Tech.
  17.  *     Chinese Academy of Sciences
  18.  *     All rights reserved.
  19.  *
  20.  * This file is the confidential and proprietary property of 
  21.  * Institute of Computing Tech. and the posession or use of this file requires 
  22.  * a written license from the author.
  23.  * Filename: Span.cpp
  24.  * Abstract:
  25.  *           implementation of the CSpan class.
  26.  * Author:   Kevin Zhang 
  27.  *          (zhanghp@software.ict.ac.cn)
  28.  * Date:     2002-4-23
  29.  *
  30.  * Notes:    Tagging with Hidden Markov Model
  31.  *                
  32.  ****************************************************************************/
  34. #include "stdafx.h"
  35. #include "Span.h"
  36. #include "..\Segment\Segment.h"
  37. #include "..\Utility\Utility.h"
  38. #include <math.h>
  39. #include <string.h>
  40. #include <stdio.h>
  41. #include <time.h>
  42. //////////////////////////////////////////////////////////////////////
  43. // Construction/Destruction
  44. //////////////////////////////////////////////////////////////////////
  45. CSpan::CSpan()
  46. {
  47. if(m_tagType!=TT_NORMAL)
  48.       m_nTags[0][0]=100;//Begin tag
  49. else
  50.       m_nTags[0][0]=0;//Begin tag
  52. m_nTags[0][1]=-1;
  53. m_dFrequency[0][0]=0;
  54. m_nCurLength=1;
  55. m_nUnknownIndex=0;
  56. m_nStartPos=0;
  57. m_nWordPosition[1]=0;
  58. m_sWords[0][0]=0;
  60. m_tagType=TT_NORMAL;//Default tagging type
  61. }
  63. CSpan::~CSpan()
  64. {
  65. }
  67. bool CSpan::Disamb()
  68. {
  69. int i,j,k,nMinCandidate;
  70. /* 
  71.      * ----- commented by huangjin@ict.ac.cn 2006-5-30 ------ 
  72.      * 
  73.  * double dMinFee,dTmp;
  74.  *
  75. */
  76. /*----Added By huangjin@ict.ac.cn 2006-5-30----*/
  77. double dMinFee=0.0, dTmp=0.0;
  78. /*---------------------------------------------*/
  79. for(i=1;i<m_nCurLength;i++)//For every word
  80. {
  81. for(j=0;m_nTags[i][j]>=0;j++)//For every word
  82. {
  83. nMinCandidate=MAX_POS_PER_WORD+1;
  84. /*----Added By huangjin@ict.ac.cn 2006-7-26----*/
  85. dMinFee=10000000.00;//Init
  86. /*---------------------------------------------*/
  87. for(k=0;m_nTags[i-1][k]>=0;k++)
  88. {
  89. //ConvertPOS(m_nTags[i-1][k],&nKey,&nPrevPOS);
  90. //ConvertPOS(m_nTags[i][j],&nKey,&nCurPOS);
  91. //dTmp=m_context.GetContextPossibility(nKey,nPrevPOS,nCurPOS);
  92. dTmp=-log(m_context.GetContextPossibility(0,m_nTags[i-1][k],m_nTags[i][j]));
  93. dTmp+=m_dFrequency[i-1][k];//Add the fees
  94. /* 
  95. * ----- commented by huangjin@ict.ac.cn 2006-7-26 ------ 
  96. *
  97. *  if(nMinCandidate>10||dTmp<dMinFee)//Get the minimum fee
  98. *
  99. */
  100. /*----Added By huangjin@ict.ac.cn 2006-7-26----*/
  101. if(nMinCandidate>MAX_POS_PER_WORD+1||dTmp<dMinFee)//Get the minimum fee
  102. /*---------------------------------------------*/
  103. {
  104. nMinCandidate=k;
  105. dMinFee=dTmp;
  106. }
  107. }
  108. m_nBestPrev[i][j]=nMinCandidate;//The best previous for j
  109. m_dFrequency[i][j]=m_dFrequency[i][j]+dMinFee;
  110. }
  111. }
  113. return true;
  114. }
  116. bool CSpan::Reset(bool bContinue)
  117. {
  118. if(!bContinue)
  119. {//||CC_Find("。！”〕〉》」〗】",m_sWords[m_nCurLength-1])
  120. if(m_tagType!=TT_NORMAL)//Get the last POS in the last sentence
  121.       m_nTags[0][0]=100;//Begin tag
  122. else
  123.       m_nTags[0][0]=0;//Begin tag
  124. m_nUnknownIndex=0;
  125. m_dFrequency[0][0]=0;
  126. m_nStartPos=0;
  127. }
  128. else
  129. {
  130. m_nTags[0][0]=m_nTags[m_nCurLength-1][0];//Get the last POS in the last sentence
  131. m_dFrequency[0][0]=m_dFrequency[m_nCurLength-1][0];
  132. }
  133.     m_nTags[0][1]=-1;//Get the last POS in the last sentence,set the -1 as end flag
  134. m_nCurLength=1;
  135. m_nWordPosition[1]=m_nStartPos;
  136. m_sWords[0][0]=0;
  138. return true;
  139. }
  142. bool CSpan::LoadContext(char *sFilename)
  143. {
  144. return m_context.Load(sFilename);
  145. }
  146. bool CSpan::GetBestPOS()
  147. {
  148.   Disamb();
  149.   for(int i=m_nCurLength-1,j=0;i>0;i--)//,j>=0
  150.   {
  151.  if(m_sWords[i][0])
  152.  {//Not virtual ending
  153.  m_nBestTag[i]=m_nTags[i][j];//Record the best POS and its possibility
  154.  }  
  155.  j=m_nBestPrev[i][j];
  156.  /*----Added By huangjin@ict.ac.cn 2006-9-12----*/
  157.  if ( j >= MAX_WORDS_PER_SENTENCE )
  158.  {
  159.  j = 0;
  160.  }
  161.  /*---------------------------------------------*/
  162.   }
  163.   int nEnd=m_nCurLength;//Set the end of POS tagging
  165. if(m_sWords[m_nCurLength-1][0]==0)
  166. nEnd=m_nCurLength-1;
  167.  
  168.   m_nBestTag[nEnd]=-1;
  169.   return true;
  170. }
  171. bool CSpan::PersonRecognize(CDictionary &personDict)
  172. {
  173.   char sPOS[MAX_WORDS_PER_SENTENCE]="z",sPersonName[100];
  174.                           //0     1    2    3    4   5   
  175.   char sPatterns[][5]={ "BBCD","BBC","BBE","BBZ","BCD","BEE","BE","BG",
  176.                     "BXD","BZ", "CDCD","CD","EE", "FB", "Y","XD",""};
  177.                     //BBCD        BBC       BBE     BBZ    BCD         BEE      BE         BG
  178.   double dFactor[]={0.003606,0.000021,0.001314,0.000315,0.656624, 0.000021,0.146116,0.009136,
  179.                // BXD      BZ   CDCD     CD      EE      FB       Y         XD  
  180. 0.000042,0.038971,0,0.090367,0.000273,0.009157,0.034324,0.009735,0
  181.   };
  182.   //About parameter:
  183. /*
  184. BBCD 343 0.003606
  185. BBC 2 0.000021
  186. BBE 125 0.001314
  187. BBZ 30 0.000315
  188. BCD 62460 0.656624
  189. BEE 0 0.000000
  190. BE 13899 0.146116
  191. BG 869 0.009136
  192. BXD 4 0.000042
  193. BZ 3707 0.038971
  194. CD 8596 0.090367
  195. EE 26 0.000273
  196. FB 871 0.009157
  197. Y 3265 0.034324
  198. XD 926 0.009735
  199.  */
  200.   //The person recognition patterns set
  201.   //BBCD:姓+姓+名1+名2;
  202.   //BBE: 姓+姓+单名;
  203.   //BBZ: 姓+姓+双名成词;
  204.   //BCD: 姓+名1+名2;
  205.   //BE:  姓+单名;
  206.   //BEE: 姓+单名+单名;韩磊磊
  207.   //BG:  姓+后缀
  208.   //BXD: 姓+姓双名首字成词+双名末字
  209.   //BZ:  姓+双名成词;
  210.   //B:  姓
  211.   //CD:  名1+名2;
  212.   //EE:  单名+单名;
  213.   //FB:  前缀+姓
  214.   //XD:  姓双名首字成词+双名末字
  215.   //Y:   姓单名成词
  216.   int nPatternLen[]={4,3,3,3,3,3,2,2,3,2,4,2,2,2,1,2,0};
  218.   /* 
  219. * ----- commented by huangjin@ict.ac.cn 2006-7-27 ------ 
  220. *
  221. * for(int i=1;m_nBestTag[i]>-1;i++)//Convert to string from POS
  222. *
  223. */
  224. /*----Added By huangjin@ict.ac.cn 2006-7-27----*/
  225.   int i=1;
  226.   for( i=1;m_nBestTag[i]>-1;i++)//Convert to string from POS
  227. /*---------------------------------------------*/  
  228. sPOS[i]=m_nBestTag[i]+'A';
  229.   sPOS[i]=0;
  230.   int j=1,k,nPos;//Find the proper pattern from the first POS
  231.   int nLittleFreqCount;//Counter for the person name role with little frequecy
  232.   bool bMatched=false;   
  233.   while(j<i)
  234.   {
  235. bMatched=false;   
  236. for(k=0;!bMatched&&nPatternLen[k]>0;k++)
  237. {
  238. if(strncmp(sPatterns[k],sPOS+j,nPatternLen[k])==0&&strcmp(m_sWords[j-1],"·")!=0&&strcmp(m_sWords[j+nPatternLen[k]],"·")!=0)
  239. {//Find the proper pattern k
  240. if(strcmp(sPatterns[k],"FB")==0&&(sPOS[j+2]=='E'||sPOS[j+2]=='C'||sPOS[j+2]=='G'))
  241. {//Rule 1 for exclusion:前缀+姓+名1(名2): 规则(前缀+姓)失效；
  242. continue;
  243. }
  244. /* if((strcmp(sPatterns[k],"BEE")==0||strcmp(sPatterns[k],"EE")==0)&&strcmp(m_sWords[j+nPatternLen[k]-1],m_sWords[j+nPatternLen[k]-2])!=0)
  245. {//Rule 2 for exclusion:姓+单名+单名:单名+单名 若EE对应的字不同，规则失效.如：韩磊磊
  246. continue;
  247. }
  249. if(strcmp(sPatterns[k],"B")==0&&m_nBestTag[j+1]!=12)
  250. {//Rule 3 for exclusion: 若姓后不是后缀，规则失效.如：江主席、刘大娘
  251. continue;
  252. }
  253. */ //Get the possible name
  254. nPos=j;//Record the person position in the tag sequence
  255. sPersonName[0]=0;
  256. nLittleFreqCount=0;//Record the number of role with little frequency
  257. while(nPos<j+nPatternLen[k])
  258. {//Get the possible person name
  259.  //
  260. if(m_nBestTag[nPos]<4&&personDict.GetFrequency(m_sWords[nPos],m_nBestTag[nPos])<LITTLE_FREQUENCY)
  261. nLittleFreqCount++;//The counter increase
  262. strcat(sPersonName,m_sWords[nPos]);
  263. nPos+=1;
  264. }
  265. /*
  266. if(IsAllForeign(sPersonName)&&personDict.GetFrequency(m_sWords[j],1)<LITTLE_FREQUENCY)
  267. {//Exclusion foreign name
  268.  //Rule 2 for exclusion:若均为外国人名用字 规则(名1+名2)失效
  269. j+=nPatternLen[k]-1;
  270. continue;
  271. }
  272. */ if(strcmp(sPatterns[k],"CDCD")==0)
  273. {//Rule for exclusion
  274.  //规则(名1+名2+名1+名2)本身是排除规则:女高音歌唱家迪里拜尔演唱
  275.    //Rule 3 for exclusion:含外国人名用字 规则适用
  276.  //否则，排除规则失效:黑妞白妞姐俩拔了头筹。
  277. if(GetForeignCharCount(sPersonName)>0)
  278. j+=nPatternLen[k]-1;
  279. continue;
  280. }
  281. /* if(strcmp(sPatterns[k],"CD")==0&&IsAllForeign(sPersonName))
  282. {//
  283. j+=nPatternLen[k]-1;
  284. continue;
  285. }
  286. if(nLittleFreqCount==nPatternLen[k]||nLittleFreqCount==3)
  287. //马哈蒂尔;小扎耶德与他的中国阿姨胡彩玲受华黎明大使之邀，
  288. //The all roles appear with two lower frequecy,we will ignore them
  289. continue;
  290. */ m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[j];
  291. m_nUnknownWords[m_nUnknownIndex][1]=m_nWordPosition[j+nPatternLen[k]];
  292. m_dWordsPossibility[m_nUnknownIndex]=-log(dFactor[k])+ComputePossibility(j,nPatternLen[k],personDict);
  293. //Mutiply the factor 
  294. m_nUnknownIndex+=1;
  295. j+=nPatternLen[k];
  296. bMatched=true;
  297. }
  298. }
  299.     if(!bMatched)//Not matched, add j by 1
  300. j+=1;
  301.   }
  302.   return true;
  303. }
  305. int CSpan::GetFrom(PWORD_RESULT pWordItems,int nIndex,CDictionary &dictCore, CDictionary &dictUnknown)
  306. {
  307. int nCount,aPOS[MAX_POS_PER_WORD],aFreq[MAX_POS_PER_WORD];
  308. int nFreq=0,j,nRetPos=0,nWordsIndex=0;
  309. bool bSplit=false;//Need to split in Transliteration recognition 
  310.     int i=1,nPOSCount;
  311. char sCurWord[WORD_MAXLENGTH];//Current word
  312. nWordsIndex=i+nIndex-1;
  313. /* 
  314. * ----- commented by huangjin@ict.ac.cn 2006-9-12 ------ 
  315. *
  316. *  for(;i<MAX_WORDS_PER_SENTENCE&&pWordItems[nWordsIndex].sWord[0]!=0;i++)
  317. *
  318. */
  319. /*----Added By huangjin@ict.ac.cn 2006-9-12----*/
  320. for(;i<MAX_WORDS_PER_SENTENCE-1&&pWordItems[nWordsIndex].sWord[0]!=0;i++)
  321. /*---------------------------------------------*/
  322. {
  323. if(m_tagType==TT_NORMAL||!dictUnknown.IsExist(pWordItems[nWordsIndex].sWord,44))
  324.         {
  325. strcpy(m_sWords[i],pWordItems[nWordsIndex].sWord);//store current word
  326.         m_nWordPosition[i+1]=m_nWordPosition[i]+strlen(m_sWords[i]);
  327. }
  328. else
  329. {
  330. if(!bSplit)
  331. {
  332. strncpy(m_sWords[i],pWordItems[nWordsIndex].sWord,2);//store current word
  333. m_sWords[i][2]=0;
  334. bSplit=true;
  335. }
  336. else
  337. {
  338. unsigned int nLen=strlen(pWordItems[nWordsIndex].sWord+2);
  339. strncpy(m_sWords[i],pWordItems[nWordsIndex].sWord+2,nLen);//store current word
  340. m_sWords[i][nLen]=0;
  341. bSplit=false;
  342. }
  343.         m_nWordPosition[i+1]=m_nWordPosition[i]+strlen(m_sWords[i]);
  344. }
  345. //Record the position of current word
  346. m_nStartPos=m_nWordPosition[i+1];
  347. //Move the Start POS to the ending
  348. if(m_tagType!=TT_NORMAL)
  349. {
  350. //Get the POSs from the unknown recognition dictionary
  351. strcpy(sCurWord,m_sWords[i]);
  352. if(m_tagType==TT_TRANS_PERSON&&i>0&&charType((unsigned char*)m_sWords[i-1])==CT_CHINESE)
  353. {
  354. if(m_sWords[i][0]=='.'&&m_sWords[i][1]==0)
  355. strcpy(sCurWord,"．");
  356. else if(m_sWords[i][0]=='-'&&m_sWords[i][1]==0)
  357. strcpy(sCurWord,"－");
  358. }
  359. dictUnknown.GetHandle(sCurWord,&nCount,aPOS,aFreq);
  360. nPOSCount=nCount+1;
  361. for(j=0;j<nCount;j++) 
  362. {//Get the POS set of sCurWord in the unknown dictionary
  363. m_nTags[i][j]=aPOS[j];
  364.     m_dFrequency[i][j]=-log((double)(1+aFreq[j]))+log((double)(m_context.GetFrequency(0,aPOS[j])+nPOSCount));
  365. }
  366. //Get the POS set of sCurWord in the core dictionary
  367. //We ignore the POS in the core dictionary and recognize them as other (0).
  368. //We add their frequency to get the possibility as POS 0
  369. /* 
  370. * ----- commented by huangjin@ict.ac.cn 2006-6-8 ------ 
  371. *
  372. * if(strcmp(m_sWords[i],"始##始")==0)
  373. * {
  374. * m_nTags[i][j]=100;
  375. * m_dFrequency[i][j]=0;
  376. * j++;
  377. * }
  378. * else if(strcmp(m_sWords[i],"末##末")==0)
  379. * {
  380. * m_nTags[i][j]=101;
  381. * m_dFrequency[i][j]=0;
  382. * j++;
  383. * }
  384. *
  385. */
  386. /*----Added By huangjin@ict.ac.cn 2006-6-8----*/
  387. if(strcmp(m_sWords[i],SENTENCE_BEGIN)==0&&j==1)
  388. {//Sentence Begin
  389. m_nTags[i][j-1]=100;
  390. m_dFrequency[i][j-1]=0;
  391. }
  392. else if(strcmp(m_sWords[i],SENTENCE_END)==0&&j==1)
  393. {//Sentence Ending
  394. m_nTags[i][j]=101;
  395. m_dFrequency[i][j]=0;
  396. }
  397. /*---------------------------------------------*/
  398. else
  399. {
  400. dictCore.GetHandle(m_sWords[i],&nCount,aPOS,aFreq);
  401. nFreq=0;
  402. for(int k=0;k<nCount;k++) 
  403. {
  404. nFreq+=aFreq[k];
  405. }
  406. if(nCount>0)
  407. {
  408. m_nTags[i][j]=0;
  409. //m_dFrequency[i][j]=(double)(1+nFreq)/(double)(m_context.GetFrequency(0,0)+1);
  410. m_dFrequency[i][j]=-log((double)(1+nFreq))+log((double)(m_context.GetFrequency(0,0)+nPOSCount));
  411. j++;
  412. }
  413. }
  414. }
  415. else//For normal POS tagging
  416. {
  417. j=0;
  418. //Get the POSs from the unknown recognition dictionary
  419. if(pWordItems[nWordsIndex].nHandle>0)
  420. {//The word has  is only one POS value
  421.  //We have record its POS and nFrequncy in the items.
  422. m_nTags[i][j]=pWordItems[nWordsIndex].nHandle;
  423. /*----Added By huangjin@ict.ac.cn 2006-9-12----*/
  424. if ( pWordItems[nWordsIndex].dValue <= 0.0 )
  425. m_dFrequency[i][j] = 0.0;
  426. else
  427. /*---------------------------------------------*/
  428. m_dFrequency[i][j]=-log(pWordItems[nWordsIndex].dValue)+log((double)(m_context.GetFrequency(0,m_nTags[i][j])+1));
  429. if(m_dFrequency[i][j]<0)//Not permit the value less than 0
  430. m_dFrequency[i][j]=0;
  431. j++;
  432. }
  433. else
  434. {//The word has multiple POSs, we should retrieve the information from Core Dictionary 
  436. if(pWordItems[nWordsIndex].nHandle<0)
  437. {//The word has  is only one POS value
  438.  //We have record its POS and nFrequncy in the items.
  439. /*
  440. if(pWordItems[nWordsIndex].nHandle==-'t'*256-'t')//tt
  441. {
  442. char sWordOrg[100],sPostfix[10];
  443. double dRatio=0.6925;//The ratio which transliteration as a person name 
  444. PostfixSplit(pWordItems[nWordsIndex].sWord,sWordOrg,sPostfix);
  445. if(sPostfix[0]!=0)
  446. dRatio=0.01;
  447. m_nTags[i][j]='n'*256+'r';
  448. m_dFrequency[i][j]=-log(dRatio)+pWordItems[nWordsIndex].dValue;
  449. //m_dFrequency[i][j]=log(dRatio)+pWordItems[nWordsIndex].dValue-log(m_context.GetFrequency(0,m_nTags[i][j]))+log(MAX_FREQUENCE);
  450. //P(W|R)=P(WRT)/P(RT)=P(R)*P(W|T)/P(R|T)
  451. j++;
  452. m_nTags[i][j]='n'*256+'s';
  453. m_dFrequency[i][j]=-log(1-dRatio)+pWordItems[nWordsIndex].dValue;
  454. //m_dFrequency[i][j]=log(1-dRatio)+pWordItems[nWordsIndex].dValue-log(m_context.GetFrequency(0,m_nTags[i][j]))+log(MAX_FREQUENCE);
  455. j++;
  456. }
  457. else//Unknown words such as Chinese person name or place name
  458. {
  459. */
  460. m_nTags[i][j]=-pWordItems[nWordsIndex].nHandle;
  461. m_dFrequency[i][j++]=pWordItems[nWordsIndex].dValue;
  462. //}
  463. }
  464. dictCore.GetHandle(m_sWords[i],&nCount,aPOS,aFreq);
  465. nPOSCount=nCount;
  466. for(;j<nCount;j++) 
  467. {//Get the POS set of sCurWord in the unknown dictionary
  468. m_nTags[i][j]=aPOS[j];
  469. /* 
  470. * ----- commented by huangjin@ict.ac.cn 2006-5-29 ------ 
  471. * 
  472. *  m_dFrequency[i][j]=-log(1+aFreq[j])+log(m_context.GetFrequency(0,m_nTags[i][j])+nPOSCount);
  473. *
  474. */
  475. /*----Added By huangjin@ict.ac.cn 2006-5-29----*/
  476. m_dFrequency[i][j]=-log(double(1+aFreq[j]))+log(double(m_context.GetFrequency(0,m_nTags[i][j])+nPOSCount));
  477. /*---------------------------------------------*/
  478. }
  479. }
  480. }
  481. if(j==0)
  482. {//We donot know the POS, so we have to guess them according lexical knowledge
  483. GuessPOS(i,&j);// the POS of current word
  484. }
  485. m_nTags[i][j]=-1;//Set the ending POS 
  486. if(j==1&&m_nTags[i][j]!=CT_SENTENCE_BEGIN)//No ambuguity
  487. {//No ambuguity, so we can break from the loop
  488. i++;
  489. m_sWords[i][0]=0;
  490. break;
  491. }
  492. if(!bSplit)
  493. nWordsIndex++;
  494. }
  495. if(pWordItems[nWordsIndex].sWord[0]==0)
  496. nRetPos=-1;//Reaching ending
  498. if(m_nTags[i-1][1]!=-1)//||m_sWords[i][0]==0
  499. {//Set end for words like "张/华/平"
  500. if(m_tagType!=TT_NORMAL)
  501. m_nTags[i][0]=101;
  502. else
  503. m_nTags[i][0]=1;
  504. m_dFrequency[i][0]=0;
  505.     m_sWords[i][0]=0;//Set virtual ending
  506. /*----Added By huangjin@ict.ac.cn 2006-9-12----*/
  507. if ( i == MAX_WORDS_PER_SENTENCE - 1 )
  508. {//backfoward one word
  509. nWordsIndex--;
  510. }
  511. /*---------------------------------------------*/
  512.         m_nTags[i++][1]=-1;
  513. }
  514. m_nCurLength=i;//The current word count
  515. if(nRetPos!=-1)
  516. return nWordsIndex+1;//Next start position
  517. return -1;//Reaching ending
  518. }
  521. //Set the tag type
  522. void CSpan::SetTagType(enum TAG_TYPE  nType)
  523. {
  524. m_tagType=nType;
  525. }
  526. //POS tagging with Hidden Markov Model
  527. bool CSpan::POSTagging(PWORD_RESULT pWordItems,CDictionary &dictCore,CDictionary &dictUnknown)
  528. {
  529. //pWordItems: Items; nItemCount: the count of items;core dictionary and unknown recognition dictionary
  530.     int i=0,j,nStartPos;
  531. Reset(false);
  532.     while(i>-1&&pWordItems[i].sWord[0]!=0)
  533. {
  534. nStartPos=i;//Start Position
  535. i=GetFrom(pWordItems,nStartPos,dictCore,dictUnknown);
  536. GetBestPOS();
  537. switch(m_tagType)
  538. {
  539. case TT_NORMAL://normal POS tagging
  540. j=1;
  541. while(m_nBestTag[j]!=-1&&j<m_nCurLength)
  542. {//Store the best POS tagging
  543. pWordItems[j+nStartPos-1].nHandle=m_nBestTag[j];
  544. //Let 。be 0
  545. if(pWordItems[j+nStartPos-1].dValue>0&&dictCore.IsExist(pWordItems[j+nStartPos-1].sWord,-1))//Exist and update its frequncy as a POS value
  546. pWordItems[j+nStartPos-1].dValue=dictCore.GetFrequency(pWordItems[j+nStartPos-1].sWord,m_nBestTag[j]);
  547. j+=1;
  548. }
  549. break;
  550. case TT_PERSON://Person recognition
  551. PersonRecognize(dictUnknown);
  552. break;
  553. case TT_PLACE://Place name recognition
  554. case TT_TRANS_PERSON://Transliteration Person
  555. PlaceRecognize(dictCore,dictUnknown);
  556. break;
  557. default:
  558. break;
  559. }
  560. Reset();
  561. }
  562. return true;
  563. }
  564. //Guess the POS of No. nIndex word item
  565. bool CSpan::GuessPOS(int nIndex,int *pSubIndex)
  566. {
  567. int j=0,i=nIndex,nCharType;
  568. unsigned int nLen;
  569. switch(m_tagType)
  570. {
  571. case TT_NORMAL:
  572. /*----Added By huangjin@ict.ac.cn 2006-9-12----*/
  573. m_nTags[i][j]='x'*256;
  574. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,'x'*256)+1);
  575. /*---------------------------------------------*/
  576. break;
  577. case TT_PERSON:
  578. j=0;
  579. if(CC_Find("××",m_sWords[nIndex]))
  580. {
  581. m_nTags[i][j]=6;
  582. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,6)+1);
  583. }
  584. else
  585. {
  586. m_nTags[i][j]=0;
  587. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  588. nLen=strlen(m_sWords[nIndex]);
  589. if(nLen>=4)
  590. {
  591. m_nTags[i][j]=0;
  592. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  593. m_nTags[i][j]=11;
  594. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  595. m_nTags[i][j]=12;
  596. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  597. m_nTags[i][j]=13;
  598. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  599. }
  600. else if(nLen==2)
  601. {
  602. m_nTags[i][j]=0;
  603. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  604. nCharType=charType((unsigned char *)m_sWords[nIndex]);
  605. if(nCharType==CT_OTHER||nCharType==CT_CHINESE)
  606. {
  607. m_nTags[i][j]=1;
  608. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)+1);
  609. m_nTags[i][j]=2;
  610. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)+1);
  611. m_nTags[i][j]=3;
  612. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)+1);
  613. m_nTags[i][j]=4;
  614. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,4)+1);
  615. }
  616. m_nTags[i][j]=11;
  617. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  618. m_nTags[i][j]=12;
  619. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  620. m_nTags[i][j]=13;
  621. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  622. }
  623. }
  624. break;
  625. case TT_PLACE:
  626. j=0;
  627. m_nTags[i][j]=0;
  628. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  629. nLen=strlen(m_sWords[nIndex]);
  630. if(nLen>=4)
  631. {
  632. m_nTags[i][j]=11;
  633. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  634. m_nTags[i][j]=12;
  635. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  636. m_nTags[i][j]=13;
  637. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  638. }
  639. else if(nLen==2)
  640. {
  641. m_nTags[i][j]=0;
  642. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  643. nCharType=charType((unsigned char *)m_sWords[nIndex]);
  644. if(nCharType==CT_OTHER||nCharType==CT_CHINESE)
  645. {
  646. m_nTags[i][j]=1;
  647. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)+1);
  648. m_nTags[i][j]=2;
  649. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)+1);
  650. m_nTags[i][j]=3;
  651. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)+1);
  652. m_nTags[i][j]=4;
  653. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,4)+1);
  654. }
  655. m_nTags[i][j]=11;
  656. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  657. m_nTags[i][j]=12;
  658. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  659. m_nTags[i][j]=13;
  660. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  661. }
  662. break;
  663. case TT_TRANS_PERSON:
  664. j=0;
  665. nLen=strlen(m_sWords[nIndex]);
  667. m_nTags[i][j]=0;
  668. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  670. if(!IsAllChinese((unsigned char *)m_sWords[nIndex]))
  671. {
  672. if(IsAllLetter((unsigned char *)m_sWords[nIndex]))
  673. {
  674. m_nTags[i][j]=1;
  675. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)+1);
  676. m_nTags[i][j]=11;
  677. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)+1);
  678. m_nTags[i][j]=2;
  679. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)*2+1);
  680. m_nTags[i][j]=3;
  681. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)*2+1);
  682. m_nTags[i][j]=12;
  683. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*2+1);
  684. m_nTags[i][j]=13;
  685. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*2+1);
  686. }
  687. m_nTags[i][j]=41;
  688. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,41)*8);
  689. m_nTags[i][j]=42;
  690. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,42)*8);
  691. m_nTags[i][j]=43;
  692. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,43)*8);
  693. }
  694. else if(nLen>=4)
  695. {
  696. m_nTags[i][j]=41;
  697. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,41)*8);
  698. m_nTags[i][j]=42;
  699. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,42)*8);
  700. m_nTags[i][j]=43;
  701. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,43)*8);
  702. }
  703. else if(nLen==2)
  704. {
  705. nCharType=charType((unsigned char *)m_sWords[nIndex]);
  706. if(nCharType==CT_OTHER||nCharType==CT_CHINESE)
  707. {
  708. m_nTags[i][j]=1;
  709. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)*2+1);
  710. m_nTags[i][j]=2;
  711. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)*2+1);
  712. m_nTags[i][j]=3;
  713. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)*2+1);
  714. m_nTags[i][j]=30;
  715. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,30)*8+1);
  716. m_nTags[i][j]=11;
  717. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*4+1);
  718. m_nTags[i][j]=12;
  719. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*4+1);
  720. m_nTags[i][j]=13;
  721. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*4+1);
  722. m_nTags[i][j]=21;
  723. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,21)*2+1);
  724. m_nTags[i][j]=22;
  725. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,22)*2+1);
  726. m_nTags[i][j]=23;
  727. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,23)*2+1);
  728. }
  729. m_nTags[i][j]=41;
  730. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,41)*8);
  731. m_nTags[i][j]=42;
  732. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,42)*8);
  733. m_nTags[i][j]=43;
  734. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,43)*8);
  735. }
  736. break;
  737. default:
  738. break;
  739. }
  740. *pSubIndex=j;
  741. return true;
  742. }
  744. ELEMENT_TYPE  CSpan::ComputePossibility(int nStartPos,int nLength,CDictionary &dict)
  745. {
  746. ELEMENT_TYPE dRetValue=0,dPOSPoss;
  747. //dPOSPoss: the possibility of a POS appears
  748. //dContextPoss: The possibility of context POS appears
  749. int nFreq;
  750. for(int i=nStartPos;i<nStartPos+nLength;i++)
  751. {
  752. nFreq=dict.GetFrequency(m_sWords[i],m_nBestTag[i]);
  753. //nFreq is word being the POS
  754. dPOSPoss=log((double)(m_context.GetFrequency(0,m_nBestTag[i])+1))-log((double)(nFreq+1));
  755. dRetValue+=dPOSPoss;
  756. /* if(i<nStartPos+nLength-1)
  757. {
  758. dContextPoss=log((double)(m_context.GetContextPossibility(0,m_nBestTag[i],m_nBestTag[i+1])+1));
  759. dRetValue+=dPOSPoss-dContextPoss;
  760. }
  761. */ }
  762. return dRetValue;
  763. }
  764. //DEL bool CSpan::TransRecognize(CDictionary &dictCore,CDictionary &transDict)
  765. //DEL {
  766. //DEL   char sPOS[MAX_WORDS_PER_SENTENCE]="Z";
  767. //DEL   int nStart=1,nEnd=1,i=1;
  768. //DEL   while(m_nBestTag[i]>-1)
  769. //DEL   {
  770. //DEL    if(m_nBestTag[i]==1||m_nBestTag[i]==11||m_nBestTag[i]==21)//1,11,21 Trigger the recognition
  771. //DEL    {
  772. //DEL  nStart=i;
  773. //DEL  nEnd=nStart+1;
  774. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart])//1,11,21
  775. //DEL  nEnd++;
  776. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart]+1)//2,12,22
  777. //DEL  nEnd++;
  778. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart]+2)//3,13,23
  779. //DEL  nEnd++;
  780. //DEL  while(m_nBestTag[nEnd]==30)//3,13,23
  781. //DEL  nEnd++;
  782. //DEL    }
  783. //DEL    else if(m_nBestTag[i]==2||m_nBestTag[i]==12||m_nBestTag[i]==22)//1,11,21 Trigger the recognition
  784. //DEL    {
  785. //DEL  nStart=i;
  786. //DEL  nEnd=nStart+1;
  787. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart])//2,12,22
  788. //DEL  nEnd++;
  789. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart]+1)//2,12,22
  790. //DEL  nEnd++;
  791. //DEL  while(m_nBestTag[nEnd]==30)//3,13,23
  792. //DEL  nEnd++;
  793. //DEL    }
  794. //DEL    if(nEnd>nStart&&!IsAllNum((unsigned char *)m_sWords[nStart])&&(nEnd>nStart+2||(nEnd==nStart+2&&(m_nBestTag[nEnd-1]!=30||strlen(m_sWords[nStart])>2))||(nEnd==nStart+1&&strlen(m_sWords[nStart])>2&&!dictCore.IsExist(m_sWords[nStart],-1))))
  795. //DEL    {
  796. //DEL  m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[nStart];
  797. //DEL  m_nUnknownWords[m_nUnknownIndex][1]=m_nWordPosition[nEnd];
  798. //DEL  m_dWordsPossibility[m_nUnknownIndex++]=ComputePossibility(nStart,nEnd-nStart+1,transDict);
  799. //DEL  nStart=nEnd;
  800. //DEL    }
  801. //DEL 
  802. //DEL    if(i<nEnd)
  803. //DEL    i=nEnd;
  804. //DEL    else
  805. //DEL    i=i+1;
  806. //DEL   }
  807. //DEL   return true;
  808. //DEL }
  809. bool CSpan::PlaceRecognize(CDictionary &dictCore,CDictionary &placeDict)
  810. {
  811.   int nStart=1,nEnd=1,i=1,nTemp;
  812.   double dPanelty=1.0;//Panelty value
  813.   while(m_nBestTag[i]>-1)
  814.   {
  815.   if(m_nBestTag[i]==1)//1 Trigger the recognition procession
  816.   {
  817. nStart=i;
  818. nEnd=nStart+1;
  819. while(m_nBestTag[nEnd]==1)//
  820. {
  821. if(nEnd>nStart+1)
  822. dPanelty+=1.0;
  823. nEnd++;
  824. }
  825. while(m_nBestTag[nEnd]==2)//2,12,22
  826. nEnd++;
  827. nTemp=nEnd;
  828. while(m_nBestTag[nEnd]==3)
  829. {
  830. if(nEnd>nTemp)
  831. dPanelty+=1.0;
  832. nEnd++;
  833. }
  834.   }
  835.   else if(m_nBestTag[i]==2)//1,11,21 Trigger the recognition
  836.   {
  837. dPanelty+=1.0;
  838. nStart=i;
  839. nEnd=nStart+1;
  840. while(m_nBestTag[nEnd]==2)//2
  841. nEnd++;
  842. nTemp=nEnd;
  843. while(m_nBestTag[nEnd]==3)//2
  844. {
  845. if(nEnd>nTemp)
  846. dPanelty+=1.0;
  847. nEnd++;
  848. }
  849.   }
  850.   if(nEnd>nStart)
  851.   {
  852. m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[nStart];
  853. m_nUnknownWords[m_nUnknownIndex][1]=m_nWordPosition[nEnd];
  854. m_dWordsPossibility[m_nUnknownIndex++]=ComputePossibility(nStart,nEnd-nStart+1,placeDict)+log(dPanelty);
  855. nStart=nEnd;
  856.   }
  857.   if(i<nEnd)
  858.   i=nEnd;
  859.   else
  860.   i=i+1;
  861.   }
  862.   return true;
  863. }
  865. //DEL bool CSpan::TransPersonRecognize(CDictionary &dictCore, CDictionary &transDict)
  866. //DEL {
  867. //DEL   int nStart=1,nEnd=1,i=1;
  868. //DEL   while(m_nBestTag[i]>-1)
  869. //DEL   {
  870. //DEL    if(m_nBestTag[i]==1)//1,11,21 Trigger the recognition
  871. //DEL    {
  872. //DEL  nStart=i;
  873. //DEL  nEnd=nStart+1;
  874. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart])//1,11,21
  875. //DEL  nEnd++;
  876. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart]+1)//2,12,22
  877. //DEL  nEnd++;
  878. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart]+2)//3,13,23
  879. //DEL  nEnd++;
  880. //DEL    }
  881. //DEL    else if(m_nBestTag[i]==2)//1,11,21 Trigger the recognition
  882. //DEL    {
  883. //DEL  nStart=i;
  884. //DEL  nEnd=nStart+1;
  885. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart])//2,12,22
  886. //DEL  nEnd++;
  887. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart]+1)//2,12,22
  888. //DEL  nEnd++;
  889. //DEL    }
  890. //DEL    if(nEnd>nStart&&!IsAllNum((unsigned char *)m_sWords[nStart])&&(nEnd>nStart+2||(nEnd==nStart+2&&(m_nBestTag[nEnd-1]!=30||strlen(m_sWords[nStart])>2))||(nEnd==nStart+1&&strlen(m_sWords[nStart])>2&&!dictCore.IsExist(m_sWords[nStart],-1))))
  891. //DEL    {
  892. //DEL  m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[nStart];
  893. //DEL  m_nUnknownWords[m_nUnknownIndex][1]=m_nWordPosition[nEnd];
  894. //DEL  m_dWordsPossibility[m_nUnknownIndex++]=ComputePossibility(nStart,nEnd-nStart+1,transDict);
  895. //DEL  nStart=nEnd;
  896. //DEL    }
  897. //DEL 
  898. //DEL    if(i<nEnd)
  899. //DEL    i=nEnd;
  900. //DEL    else
  901. //DEL    i=i+1;
  902. //DEL   }
  903. //DEL   return true;
  904. //DEL }