开通博客赚积分
发布资源赚积分
分类

源码开发语言/平台
当前位置：首页 > 源码/资料 > 多国语言处理 > 查看源码
Span.cpp
资源名称：TextClassify.rar　[点击查看]
上传用户：sanxfzhen
上传日期：2014-12-28
资源大小：2324k
文件大小：25k
源码类别：

多国语言处理

开发平台：

Visual C++

Span.cpp：源码内容
  1. //////////////////////////////////////////////////////////////////////
  2. //ICTCLAS简介：计算所汉语词法分析系统ICTCLAS(Institute of Computing Technology, Chinese Lexical Analysis System)，
  3. //             功能有：中文分词；词性标注；未登录词识别。
  4. //             分词正确率高达97.58%(973专家评测结果)，
  5. //             未登录词识别召回率均高于90%，其中中国人名的识别召回率接近98%;
  6. //             处理速度为31.5Kbytes/s。
  7. //著作权：  Copyright?2002-2005中科院计算所 职务著作权人：张华平 刘群
  8. //遵循协议：自然语言处理开放资源许可证1.0
  9. //Email: zhanghp@software.ict.ac.cn
  10. //Homepage:www.nlp.org.cn;mtgroup.ict.ac.cn
  11. /****************************************************************************
  12.  *
  13.  * Copyright (c) 2000, 2001 
  14.  *     Machine Group
  15.  *     Software Research Lab.
  16.  *     Institute of Computing Tech.
  17.  *     Chinese Academy of Sciences
  18.  *     All rights reserved.
  19.  *
  20.  * This file is the confidential and proprietary property of 
  21.  * Institute of Computing Tech. and the posession or use of this file requires 
  22.  * a written license from the author.
  23.  * Filename: Span.cpp
  24.  * Abstract:
  25.  *           implementation of the CSpan class.
  26.  * Author:   Kevin Zhang 
  27.  *          (zhanghp@software.ict.ac.cn)
  28.  * Date:     2002-4-23
  29.  *
  30.  * Notes:    Tagging with Hidden Markov Model
  31.  *                
  32.  ****************************************************************************/
  34. #include "stdafx.h"
  35. #include "Span.h"
  36. #include "..\Segment\Segment.h"
  37. #include "..\Utility\Utility.h"
  38. #include <math.h>
  39. #include <string.h>
  40. #include <stdio.h>
  41. #include <time.h>
  42. //////////////////////////////////////////////////////////////////////
  43. // Construction/Destruction
  44. //////////////////////////////////////////////////////////////////////
  45. CSpan::CSpan()
  46. {
  47. if(m_tagType!=TT_NORMAL)
  48.       m_nTags[0][0]=100;//Begin tag
  49. else
  50.       m_nTags[0][0]=0;//Begin tag
  52. m_nTags[0][1]=-1;
  53. m_dFrequency[0][0]=0;
  54. m_nCurLength=1;
  55. m_nUnknownIndex=0;
  56. m_nStartPos=0;
  57. m_nWordPosition[1]=0;
  58. m_sWords[0][0]=0;
  60. m_tagType=TT_NORMAL;//Default tagging type
  61. }
  63. CSpan::~CSpan()
  64. {
  65. }
  67. bool CSpan::Disamb()
  68. {
  69. int i,j,k,nMinCandidate;
  70. double dMinFee,dTmp;
  71. for(i=1;i<m_nCurLength;i++)//For every word
  72. {
  73. for(j=0;m_nTags[i][j]>=0;j++)//For every word
  74. {
  75. nMinCandidate=MAX_POS_PER_WORD+1;
  76. for(k=0;m_nTags[i-1][k]>=0;k++)
  77. {
  78. //ConvertPOS(m_nTags[i-1][k],&nKey,&nPrevPOS);
  79. //ConvertPOS(m_nTags[i][j],&nKey,&nCurPOS);
  80. //dTmp=m_context.GetContextPossibility(nKey,nPrevPOS,nCurPOS);
  81. dTmp=-log(m_context.GetContextPossibility(0,m_nTags[i-1][k],m_nTags[i][j]));
  82. dTmp+=m_dFrequency[i-1][k];//Add the fees
  83. if(nMinCandidate>10||dTmp<dMinFee)//Get the minimum fee
  84. {
  85. nMinCandidate=k;
  86. dMinFee=dTmp;
  87. }
  88. }
  89. m_nBestPrev[i][j]=nMinCandidate;//The best previous for j
  90. m_dFrequency[i][j]=m_dFrequency[i][j]+dMinFee;
  91. }
  92. }
  94. return true;
  95. }
  97. bool CSpan::Reset(bool bContinue)
  98. {
  99. if(!bContinue)
  100. {//||CC_Find("。！”〕〉》」〗】",m_sWords[m_nCurLength-1])
  101. if(m_tagType!=TT_NORMAL)//Get the last POS in the last sentence
  102.       m_nTags[0][0]=100;//Begin tag
  103. else
  104.       m_nTags[0][0]=0;//Begin tag
  105. m_nUnknownIndex=0;
  106. m_dFrequency[0][0]=0;
  107. m_nStartPos=0;
  108. }
  109. else
  110. {
  111. m_nTags[0][0]=m_nTags[m_nCurLength-1][0];//Get the last POS in the last sentence
  112. m_dFrequency[0][0]=m_dFrequency[m_nCurLength-1][0];
  113. }
  114.     m_nTags[0][1]=-1;//Get the last POS in the last sentence,set the -1 as end flag
  115. m_nCurLength=1;
  116. m_nWordPosition[1]=m_nStartPos;
  117. m_sWords[0][0]=0;
  118. return true;
  119. }
  122. bool CSpan::LoadContext(char *sFilename)
  123. {
  124. return m_context.Load(sFilename);
  125. }
  126. bool CSpan::GetBestPOS()
  127. {
  128.   Disamb();
  129.   for(int i=m_nCurLength-1,j=0;i>0;i--)//,j>=0
  130.   {
  131.  if(m_sWords[i][0])
  132.  {//Not virtual ending
  133.  m_nBestTag[i]=m_nTags[i][j];//Record the best POS and its possibility
  134.  }
  135.  j=m_nBestPrev[i][j];
  136.   }
  137.   int nEnd=m_nCurLength;//Set the end of POS tagging
  138.   if(m_sWords[m_nCurLength-1][0]==0)
  139.   nEnd=m_nCurLength-1;
  140.   m_nBestTag[nEnd]=-1;
  141.   return true;
  142. }
  143. bool CSpan::PersonRecognize(CDictionary &personDict)
  144. {
  145.   char sPOS[MAX_WORDS_PER_SENTENCE]="z",sPersonName[100];
  146.                           //0     1    2    3    4   5   
  147.   char sPatterns[][5]={ "BBCD","BBC","BBE","BBZ","BCD","BEE","BE","BG",
  148.                     "BXD","BZ", "CDCD","CD","EE", "FB", "Y","XD",""};
  149.                     //BBCD        BBC       BBE     BBZ    BCD         BEE      BE         BG
  150.   double dFactor[]={0.003606,0.000021,0.001314,0.000315,0.656624, 0.000021,0.146116,0.009136,
  151.                // BXD      BZ   CDCD     CD      EE      FB       Y         XD  
  152. 0.000042,0.038971,0,0.090367,0.000273,0.009157,0.034324,0.009735,0
  153.   };
  154.   //About parameter:
  155. /*
  156. BBCD 343 0.003606
  157. BBC 2 0.000021
  158. BBE 125 0.001314
  159. BBZ 30 0.000315
  160. BCD 62460 0.656624
  161. BEE 0 0.000000
  162. BE 13899 0.146116
  163. BG 869 0.009136
  164. BXD 4 0.000042
  165. BZ 3707 0.038971
  166. CD 8596 0.090367
  167. EE 26 0.000273
  168. FB 871 0.009157
  169. Y 3265 0.034324
  170. XD 926 0.009735
  171.  */
  172.   //The person recognition patterns set
  173.   //BBCD:姓+姓+名1+名2;
  174.   //BBE: 姓+姓+单名;
  175.   //BBZ: 姓+姓+双名成词;
  176.   //BCD: 姓+名1+名2;
  177.   //BE:  姓+单名;
  178.   //BEE: 姓+单名+单名;韩磊磊
  179.   //BG:  姓+后缀
  180.   //BXD: 姓+姓双名首字成词+双名末字
  181.   //BZ:  姓+双名成词;
  182.   //B:  姓
  183.   //CD:  名1+名2;
  184.   //EE:  单名+单名;
  185.   //FB:  前缀+姓
  186.   //XD:  姓双名首字成词+双名末字
  187.   //Y:   姓单名成词
  188.   int nPatternLen[]={4,3,3,3,3,3,2,2,3,2,4,2,2,2,1,2,0};
  190.   for(int i=1;m_nBestTag[i]>-1;i++)//Convert to string from POS
  191. sPOS[i]=m_nBestTag[i]+'A';
  192.   sPOS[i]=0;
  193.   int j=1,k,nPos;//Find the proper pattern from the first POS
  194.   int nLittleFreqCount;//Counter for the person name role with little frequecy
  195.   bool bMatched=false;   
  196.   while(j<i)
  197.   {
  198. bMatched=false;   
  199. for(k=0;!bMatched&&nPatternLen[k]>0;k++)
  200. {
  201. if(strncmp(sPatterns[k],sPOS+j,nPatternLen[k])==0&&strcmp(m_sWords[j-1],"·")!=0&&strcmp(m_sWords[j+nPatternLen[k]],"·")!=0)
  202. {//Find the proper pattern k
  203. if(strcmp(sPatterns[k],"FB")==0&&(sPOS[j+2]=='E'||sPOS[j+2]=='C'||sPOS[j+2]=='G'))
  204. {//Rule 1 for exclusion:前缀+姓+名1(名2): 规则(前缀+姓)失效；
  205. continue;
  206. }
  207. /* if((strcmp(sPatterns[k],"BEE")==0||strcmp(sPatterns[k],"EE")==0)&&strcmp(m_sWords[j+nPatternLen[k]-1],m_sWords[j+nPatternLen[k]-2])!=0)
  208. {//Rule 2 for exclusion:姓+单名+单名:单名+单名 若EE对应的字不同，规则失效.如：韩磊磊
  209. continue;
  210. }
  212. if(strcmp(sPatterns[k],"B")==0&&m_nBestTag[j+1]!=12)
  213. {//Rule 3 for exclusion: 若姓后不是后缀，规则失效.如：江主席、刘大娘
  214. continue;
  215. }
  216. */ //Get the possible name
  217. nPos=j;//Record the person position in the tag sequence
  218. sPersonName[0]=0;
  219. nLittleFreqCount=0;//Record the number of role with little frequency
  220. while(nPos<j+nPatternLen[k])
  221. {//Get the possible person name
  222.  //
  223. if(m_nBestTag[nPos]<4&&personDict.GetFrequency(m_sWords[nPos],m_nBestTag[nPos])<LITTLE_FREQUENCY)
  224. nLittleFreqCount++;//The counter increase
  225. strcat(sPersonName,m_sWords[nPos]);
  226. nPos+=1;
  227. }
  228. /*
  229. if(IsAllForeign(sPersonName)&&personDict.GetFrequency(m_sWords[j],1)<LITTLE_FREQUENCY)
  230. {//Exclusion foreign name
  231.  //Rule 2 for exclusion:若均为外国人名用字 规则(名1+名2)失效
  232. j+=nPatternLen[k]-1;
  233. continue;
  234. }
  235. */ if(strcmp(sPatterns[k],"CDCD")==0)
  236. {//Rule for exclusion
  237.  //规则(名1+名2+名1+名2)本身是排除规则:女高音歌唱家迪里拜尔演唱
  238.    //Rule 3 for exclusion:含外国人名用字 规则适用
  239.  //否则，排除规则失效:黑妞白妞姐俩拔了头筹。
  240. if(GetForeignCharCount(sPersonName)>0)
  241. j+=nPatternLen[k]-1;
  242. continue;
  243. }
  244. /* if(strcmp(sPatterns[k],"CD")==0&&IsAllForeign(sPersonName))
  245. {//
  246. j+=nPatternLen[k]-1;
  247. continue;
  248. }
  249. if(nLittleFreqCount==nPatternLen[k]||nLittleFreqCount==3)
  250. //马哈蒂尔;小扎耶德与他的中国阿姨胡彩玲受华黎明大使之邀，
  251. //The all roles appear with two lower frequecy,we will ignore them
  252. continue;
  253. */ m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[j];
  254. m_nUnknownWords[m_nUnknownIndex][1]=m_nWordPosition[j+nPatternLen[k]];
  255. m_dWordsPossibility[m_nUnknownIndex]=-log(dFactor[k])+ComputePossibility(j,nPatternLen[k],personDict);
  256. //Mutiply the factor 
  257. m_nUnknownIndex+=1;
  258. j+=nPatternLen[k];
  259. bMatched=true;
  260. }
  261. }
  262.     if(!bMatched)//Not matched, add j by 1
  263. j+=1;
  264.   }
  265.   return true;
  266. }
  268. int CSpan::GetFrom(PWORD_RESULT pWordItems,int nIndex,CDictionary &dictCore, CDictionary &dictUnknown)
  269. {
  270. int nCount,aPOS[MAX_POS_PER_WORD],aFreq[MAX_POS_PER_WORD];
  271. int nFreq=0,j,nRetPos=0,nWordsIndex=0;
  272. bool bSplit=false;//Need to split in Transliteration recognition 
  273.     int i=1,nPOSCount;
  274. char sCurWord[WORD_MAXLENGTH];//Current word
  275. nWordsIndex=i+nIndex-1;
  276. for(;i<MAX_WORDS_PER_SENTENCE&&pWordItems[nWordsIndex].sWord[0]!=0;i++)
  277. {
  278. if(m_tagType==TT_NORMAL||!dictUnknown.IsExist(pWordItems[nWordsIndex].sWord,44))
  279.         {
  280. strcpy(m_sWords[i],pWordItems[nWordsIndex].sWord);//store current word
  281.         m_nWordPosition[i+1]=m_nWordPosition[i]+strlen(m_sWords[i]);
  282. }
  283. else
  284. {
  285. if(!bSplit)
  286. {
  287. strncpy(m_sWords[i],pWordItems[nWordsIndex].sWord,2);//store current word
  288. m_sWords[i][2]=0;
  289. bSplit=true;
  290. }
  291. else
  292. {
  293. unsigned int nLen=strlen(pWordItems[nWordsIndex].sWord+2);
  294. strncpy(m_sWords[i],pWordItems[nWordsIndex].sWord+2,nLen);//store current word
  295. m_sWords[i][nLen]=0;
  296. bSplit=false;
  297. }
  298.         m_nWordPosition[i+1]=m_nWordPosition[i]+strlen(m_sWords[i]);
  299. }
  300. //Record the position of current word
  301. m_nStartPos=m_nWordPosition[i+1];
  302. //Move the Start POS to the ending
  303. if(m_tagType!=TT_NORMAL)
  304. {
  305. //Get the POSs from the unknown recognition dictionary
  306. strcpy(sCurWord,m_sWords[i]);
  307. if(m_tagType==TT_TRANS_PERSON&&i>0&&charType((unsigned char*)m_sWords[i-1])==CT_CHINESE)
  308. {
  309. if(m_sWords[i][0]=='.'&&m_sWords[i][1]==0)
  310. strcpy(sCurWord,"．");
  311. else if(m_sWords[i][0]=='-'&&m_sWords[i][1]==0)
  312. strcpy(sCurWord,"－");
  313. }
  314. dictUnknown.GetHandle(sCurWord,&nCount,aPOS,aFreq);
  315. nPOSCount=nCount+1;
  316. for(j=0;j<nCount;j++) 
  317. {//Get the POS set of sCurWord in the unknown dictionary
  318. m_nTags[i][j]=aPOS[j];
  319.     m_dFrequency[i][j]=-log((double)(1+aFreq[j]))+log((double)(m_context.GetFrequency(0,aPOS[j])+nPOSCount));
  320. }
  321. //Get the POS set of sCurWord in the core dictionary
  322. //We ignore the POS in the core dictionary and recognize them as other (0).
  323. //We add their frequency to get the possibility as POS 0
  324. if(strcmp(m_sWords[i],"始##始")==0)
  325. {
  326. m_nTags[i][j]=100;
  327.     m_dFrequency[i][j]=0;
  328. j++;
  329. }
  330. else if(strcmp(m_sWords[i],"末##末")==0)
  331. {
  332. m_nTags[i][j]=101;
  333.     m_dFrequency[i][j]=0;
  334. j++;
  335. }
  336. else
  337. {
  338. dictCore.GetHandle(m_sWords[i],&nCount,aPOS,aFreq);
  339. nFreq=0;
  340. for(int k=0;k<nCount;k++) 
  341. {
  342. nFreq+=aFreq[k];
  343. }
  344. if(nCount>0)
  345. {
  346. m_nTags[i][j]=0;
  347. //m_dFrequency[i][j]=(double)(1+nFreq)/(double)(m_context.GetFrequency(0,0)+1);
  348. m_dFrequency[i][j]=-log((double)(1+nFreq))+log((double)(m_context.GetFrequency(0,0)+nPOSCount));
  349. j++;
  350. }
  351. }
  352. }
  353. else//For normal POS tagging
  354. {
  355. j=0;
  356. //Get the POSs from the unknown recognition dictionary
  357. if(pWordItems[nWordsIndex].nHandle>0)
  358. {//The word has  is only one POS value
  359.  //We have record its POS and nFrequncy in the items.
  360. m_nTags[i][j]=pWordItems[nWordsIndex].nHandle;
  361. m_dFrequency[i][j]=-log(pWordItems[nWordsIndex].dValue)+log((double)(m_context.GetFrequency(0,m_nTags[i][j])+1));
  362. if(m_dFrequency[i][j]<0)//Not permit the value less than 0
  363. m_dFrequency[i][j]=0;
  364. j++;
  365. }
  366. else
  367. {//The word has multiple POSs, we should retrieve the information from Core Dictionary 
  369. if(pWordItems[nWordsIndex].nHandle<0)
  370. {//The word has  is only one POS value
  371.  //We have record its POS and nFrequncy in the items.
  372. /*
  373. if(pWordItems[nWordsIndex].nHandle==-'t'*256-'t')//tt
  374. {
  375. char sWordOrg[100],sPostfix[10];
  376. double dRatio=0.6925;//The ratio which transliteration as a person name 
  377. PostfixSplit(pWordItems[nWordsIndex].sWord,sWordOrg,sPostfix);
  378. if(sPostfix[0]!=0)
  379. dRatio=0.01;
  380. m_nTags[i][j]='n'*256+'r';
  381. m_dFrequency[i][j]=-log(dRatio)+pWordItems[nWordsIndex].dValue;
  382. //m_dFrequency[i][j]=log(dRatio)+pWordItems[nWordsIndex].dValue-log(m_context.GetFrequency(0,m_nTags[i][j]))+log(MAX_FREQUENCE);
  383. //P(W|R)=P(WRT)/P(RT)=P(R)*P(W|T)/P(R|T)
  384. j++;
  385. m_nTags[i][j]='n'*256+'s';
  386. m_dFrequency[i][j]=-log(1-dRatio)+pWordItems[nWordsIndex].dValue;
  387. //m_dFrequency[i][j]=log(1-dRatio)+pWordItems[nWordsIndex].dValue-log(m_context.GetFrequency(0,m_nTags[i][j]))+log(MAX_FREQUENCE);
  388. j++;
  389. }
  390. else//Unknown words such as Chinese person name or place name
  391. {
  392. */
  393. m_nTags[i][j]=-pWordItems[nWordsIndex].nHandle;
  394. m_dFrequency[i][j++]=pWordItems[nWordsIndex].dValue;
  395. //}
  396. }
  397. dictCore.GetHandle(m_sWords[i],&nCount,aPOS,aFreq);
  398. nPOSCount=nCount;
  399. for(;j<nCount;j++) 
  400. {//Get the POS set of sCurWord in the unknown dictionary
  401. m_nTags[i][j]=aPOS[j];
  402.     m_dFrequency[i][j]=-log(1+aFreq[j])+log(m_context.GetFrequency(0,m_nTags[i][j])+nPOSCount);
  403. }
  404. }
  405. }
  406. if(j==0)
  407. {//We donot know the POS, so we have to guess them according lexical knowledge
  408. GuessPOS(i,&j);//Guess the POS of current word
  409. }
  410. m_nTags[i][j]=-1;//Set the ending POS 
  411. if(j==1&&m_nTags[i][j]!=CT_SENTENCE_BEGIN)//No ambuguity
  412. {//No ambuguity, so we can break from the loop
  413. i++;
  414. m_sWords[i][0]=0;
  415. break;
  416. }
  417. if(!bSplit)
  418. nWordsIndex++;
  419. }
  420. if(pWordItems[nWordsIndex].sWord[0]==0)
  421. nRetPos=-1;//Reaching ending
  423. if(m_nTags[i-1][1]!=-1)//||m_sWords[i][0]==0
  424. {//Set end for words like "张/华/平"
  425. if(m_tagType!=TT_NORMAL)
  426.        m_nTags[i][0]=101;
  427. else
  428.        m_nTags[i][0]=1;
  430. m_dFrequency[i][0]=0;
  431.     m_sWords[i][0]=0;//Set virtual ending
  432. m_nTags[i++][1]=-1;
  433. }
  434. m_nCurLength=i;//The current word count
  435. if(nRetPos!=-1)
  436. return nWordsIndex+1;//Next start position
  437. return -1;//Reaching ending
  438. }
  441. //Set the tag type
  442. void CSpan::SetTagType(enum TAG_TYPE  nType)
  443. {
  444. m_tagType=nType;
  445. }
  446. //POS tagging with Hidden Markov Model
  447. bool CSpan::POSTagging(PWORD_RESULT pWordItems,CDictionary &dictCore,CDictionary &dictUnknown)
  448. {
  449. //pWordItems: Items; nItemCount: the count of items;core dictionary and unknown recognition dictionary
  450.     int i=0,j,nStartPos;
  451. Reset(false);
  452.     while(i>-1&&pWordItems[i].sWord[0]!=0)
  453. {
  454. nStartPos=i;//Start Position
  455. i=GetFrom(pWordItems,nStartPos,dictCore,dictUnknown);
  456. GetBestPOS();
  457. switch(m_tagType)
  458. {
  459. case TT_NORMAL://normal POS tagging
  460. j=1;
  461. while(m_nBestTag[j]!=-1&&j<m_nCurLength)
  462. {//Store the best POS tagging
  463. pWordItems[j+nStartPos-1].nHandle=m_nBestTag[j];
  464. //Let 。be 0
  465. if(pWordItems[j+nStartPos-1].dValue>0&&dictCore.IsExist(pWordItems[j+nStartPos-1].sWord,-1))//Exist and update its frequncy as a POS value
  466. pWordItems[j+nStartPos-1].dValue=dictCore.GetFrequency(pWordItems[j+nStartPos-1].sWord,m_nBestTag[j]);
  467. j+=1;
  468. }
  469. break;
  470. case TT_PERSON://Person recognition
  471. PersonRecognize(dictUnknown);
  472. break;
  473. case TT_PLACE://Place name recognition
  474. case TT_TRANS_PERSON://Transliteration Person
  475. PlaceRecognize(dictCore,dictUnknown);
  476. break;
  477. default:
  478. break;
  479. }
  480. Reset();
  481. }
  482. return true;
  483. }
  484. //Guess the POS of No. nIndex word item
  485. bool CSpan::GuessPOS(int nIndex,int *pSubIndex)
  486. {
  487. int j=0,i=nIndex,nCharType;
  488. unsigned int nLen;
  489. switch(m_tagType)
  490. {
  491. case TT_NORMAL:
  492. break;
  493. case TT_PERSON:
  494. j=0;
  495. if(CC_Find("××",m_sWords[nIndex]))
  496. {
  497. m_nTags[i][j]=6;
  498. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,6)+1);
  499. }
  500. else
  501. {
  502. m_nTags[i][j]=0;
  503. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  504. nLen=strlen(m_sWords[nIndex]);
  505. if(nLen>=4)
  506. {
  507. m_nTags[i][j]=0;
  508. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  509. m_nTags[i][j]=11;
  510. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  511. m_nTags[i][j]=12;
  512. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  513. m_nTags[i][j]=13;
  514. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  515. }
  516. else if(nLen==2)
  517. {
  518. m_nTags[i][j]=0;
  519. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  520. nCharType=charType((unsigned char *)m_sWords[nIndex]);
  521. if(nCharType==CT_OTHER||nCharType==CT_CHINESE)
  522. {
  523. m_nTags[i][j]=1;
  524. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)+1);
  525. m_nTags[i][j]=2;
  526. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)+1);
  527. m_nTags[i][j]=3;
  528. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)+1);
  529. m_nTags[i][j]=4;
  530. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,4)+1);
  531. }
  532. m_nTags[i][j]=11;
  533. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  534. m_nTags[i][j]=12;
  535. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  536. m_nTags[i][j]=13;
  537. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  538. }
  539. }
  540. break;
  541. case TT_PLACE:
  542. j=0;
  543. m_nTags[i][j]=0;
  544. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  545. nLen=strlen(m_sWords[nIndex]);
  546. if(nLen>=4)
  547. {
  548. m_nTags[i][j]=11;
  549. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  550. m_nTags[i][j]=12;
  551. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  552. m_nTags[i][j]=13;
  553. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  554. }
  555. else if(nLen==2)
  556. {
  557. m_nTags[i][j]=0;
  558. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  559. nCharType=charType((unsigned char *)m_sWords[nIndex]);
  560. if(nCharType==CT_OTHER||nCharType==CT_CHINESE)
  561. {
  562. m_nTags[i][j]=1;
  563. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)+1);
  564. m_nTags[i][j]=2;
  565. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)+1);
  566. m_nTags[i][j]=3;
  567. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)+1);
  568. m_nTags[i][j]=4;
  569. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,4)+1);
  570. }
  571. m_nTags[i][j]=11;
  572. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*8);
  573. m_nTags[i][j]=12;
  574. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*8);
  575. m_nTags[i][j]=13;
  576. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*8);
  577. }
  578. break;
  579. case TT_TRANS_PERSON:
  580. j=0;
  581. nLen=strlen(m_sWords[nIndex]);
  583. m_nTags[i][j]=0;
  584. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,0)+1);
  586. if(!IsAllChinese((unsigned char *)m_sWords[nIndex]))
  587. {
  588. if(IsAllLetter((unsigned char *)m_sWords[nIndex]))
  589. {
  590. m_nTags[i][j]=1;
  591. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)+1);
  592. m_nTags[i][j]=11;
  593. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)+1);
  594. m_nTags[i][j]=2;
  595. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)*2+1);
  596. m_nTags[i][j]=3;
  597. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)*2+1);
  598. m_nTags[i][j]=12;
  599. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*2+1);
  600. m_nTags[i][j]=13;
  601. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*2+1);
  602. }
  603. m_nTags[i][j]=41;
  604. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,41)*8);
  605. m_nTags[i][j]=42;
  606. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,42)*8);
  607. m_nTags[i][j]=43;
  608. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,43)*8);
  609. }
  610. else if(nLen>=4)
  611. {
  612. m_nTags[i][j]=41;
  613. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,41)*8);
  614. m_nTags[i][j]=42;
  615. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,42)*8);
  616. m_nTags[i][j]=43;
  617. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,43)*8);
  618. }
  619. else if(nLen==2)
  620. {
  621. nCharType=charType((unsigned char *)m_sWords[nIndex]);
  622. if(nCharType==CT_OTHER||nCharType==CT_CHINESE)
  623. {
  624. m_nTags[i][j]=1;
  625. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,1)*2+1);
  626. m_nTags[i][j]=2;
  627. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,2)*2+1);
  628. m_nTags[i][j]=3;
  629. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,3)*2+1);
  630. m_nTags[i][j]=30;
  631. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,30)*8+1);
  632. m_nTags[i][j]=11;
  633. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,11)*4+1);
  634. m_nTags[i][j]=12;
  635. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,12)*4+1);
  636. m_nTags[i][j]=13;
  637. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,13)*4+1);
  638. m_nTags[i][j]=21;
  639. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,21)*2+1);
  640. m_nTags[i][j]=22;
  641. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,22)*2+1);
  642. m_nTags[i][j]=23;
  643. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,23)*2+1);
  644. }
  645. m_nTags[i][j]=41;
  646. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,41)*8);
  647. m_nTags[i][j]=42;
  648. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,42)*8);
  649. m_nTags[i][j]=43;
  650. m_dFrequency[i][j++]=(double)1/(double)(m_context.GetFrequency(0,43)*8);
  651. }
  652. break;
  653. default:
  654. break;
  655. }
  656. *pSubIndex=j;
  657. return true;
  658. }
  660. ELEMENT_TYPE  CSpan::ComputePossibility(int nStartPos,int nLength,CDictionary &dict)
  661. {
  662. ELEMENT_TYPE dRetValue=0,dPOSPoss;
  663. //dPOSPoss: the possibility of a POS appears
  664. //dContextPoss: The possibility of context POS appears
  665. int nFreq;
  666. for(int i=nStartPos;i<nStartPos+nLength;i++)
  667. {
  668. nFreq=dict.GetFrequency(m_sWords[i],m_nBestTag[i]);
  669. //nFreq is word being the POS
  670. dPOSPoss=log((double)(m_context.GetFrequency(0,m_nBestTag[i])+1))-log((double)(nFreq+1));
  671. dRetValue+=dPOSPoss;
  672. /* if(i<nStartPos+nLength-1)
  673. {
  674. dContextPoss=log((double)(m_context.GetContextPossibility(0,m_nBestTag[i],m_nBestTag[i+1])+1));
  675. dRetValue+=dPOSPoss-dContextPoss;
  676. }
  677. */ }
  678. return dRetValue;
  679. }
  680. //DEL bool CSpan::TransRecognize(CDictionary &dictCore,CDictionary &transDict)
  681. //DEL {
  682. //DEL   char sPOS[MAX_WORDS_PER_SENTENCE]="Z";
  683. //DEL   int nStart=1,nEnd=1,i=1;
  684. //DEL   while(m_nBestTag[i]>-1)
  685. //DEL   {
  686. //DEL    if(m_nBestTag[i]==1||m_nBestTag[i]==11||m_nBestTag[i]==21)//1,11,21 Trigger the recognition
  687. //DEL    {
  688. //DEL  nStart=i;
  689. //DEL  nEnd=nStart+1;
  690. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart])//1,11,21
  691. //DEL  nEnd++;
  692. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart]+1)//2,12,22
  693. //DEL  nEnd++;
  694. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart]+2)//3,13,23
  695. //DEL  nEnd++;
  696. //DEL  while(m_nBestTag[nEnd]==30)//3,13,23
  697. //DEL  nEnd++;
  698. //DEL    }
  699. //DEL    else if(m_nBestTag[i]==2||m_nBestTag[i]==12||m_nBestTag[i]==22)//1,11,21 Trigger the recognition
  700. //DEL    {
  701. //DEL  nStart=i;
  702. //DEL  nEnd=nStart+1;
  703. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart])//2,12,22
  704. //DEL  nEnd++;
  705. //DEL  while(m_nBestTag[nEnd]==m_nBestTag[nStart]+1)//2,12,22
  706. //DEL  nEnd++;
  707. //DEL  while(m_nBestTag[nEnd]==30)//3,13,23
  708. //DEL  nEnd++;
  709. //DEL    }
  710. //DEL    if(nEnd>nStart&&!IsAllNum((unsigned char *)m_sWords[nStart])&&(nEnd>nStart+2||(nEnd==nStart+2&&(m_nBestTag[nEnd-1]!=30||strlen(m_sWords[nStart])>2))||(nEnd==nStart+1&&strlen(m_sWords[nStart])>2&&!dictCore.IsExist(m_sWords[nStart],-1))))
  711. //DEL    {
  712. //DEL  m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[nStart];
  713. //DEL  m_nUnknownWords[m_nUnknownIndex][1]=m_nWordPosition[nEnd];
  714. //DEL  m_dWordsPossibility[m_nUnknownIndex++]=ComputePossibility(nStart,nEnd-nStart+1,transDict);
  715. //DEL  nStart=nEnd;
  716. //DEL    }
  717. //DEL 
  718. //DEL    if(i<nEnd)
  719. //DEL    i=nEnd;
  720. //DEL    else
  721. //DEL    i=i+1;
  722. //DEL   }
  723. //DEL   return true;
  724. //DEL }
  725. bool CSpan::PlaceRecognize(CDictionary &dictCore,CDictionary &placeDict)
  726. {
  727.   int nStart=1,nEnd=1,i=1,nTemp;
  728.   double dPanelty=1.0;//Panelty value
  729.   while(m_nBestTag[i]>-1)
  730.   {
  731.   if(m_nBestTag[i]==1)//1 Trigger the recognition procession
  732.   {
  733. nStart=i;
  734. nEnd=nStart+1;
  735. while(m_nBestTag[nEnd]==1)//
  736. {
  737. if(nEnd>nStart+1)
  738. dPanelty+=1.0;
  739. nEnd++;
  740. }
  741. while(m_nBestTag[nEnd]==2)//2,12,22
  742. nEnd++;
  743. nTemp=nEnd;
  744. while(m_nBestTag[nEnd]==3)
  745. {
  746. if(nEnd>nTemp)
  747. dPanelty+=1.0;
  748. nEnd++;
  749. }
  750.   }
  751.   else if(m_nBestTag[i]==2)//1,11,21 Trigger the recognition
  752.   {
  753. dPanelty+=1.0;
  754. nStart=i;
  755. nEnd=nStart+1;
  756. while(m_nBestTag[nEnd]==2)//2
  757. nEnd++;
  758. nTemp=nEnd;
  759. while(m_nBestTag[nEnd]==3)//2
  760. {
  761. if(nEnd>nTemp)
  762. dPanelty+=1.0;
  763. nEnd++;
  764. }
  765.   }
  766.   if(nEnd>nStart)
  767.   {
  768. m_nUnknownWords[m_nUnknownIndex][0]=m_nWordPosition[nStart];
  769. m_nUnknownWords[m_nUnknownIndex][1]=m_nWordPosition[nEnd];
  770. m_dWordsPossibility[m_nUnknownIndex++]=ComputePossibility(nStart,nEnd-nStart+1,placeDict)+log(dPanelty);
  771. nStart=nEnd;
  772.   }
  773.   if(i<nEnd)
  774.   i=nEnd;
  775.   else
  776.   i=i+1;
  777.   }
  778.   return true;
  779. }
  781. void CSpan::ReleaseSpan()
  782. {
  783. m_context.ReleaseContextStat();
  784. }