开通博客赚积分
发布资源赚积分
分类

源码开发语言/平台
当前位置：首页 > 源码/资料 > 多国语言处理 > 查看源码
Result.cpp
资源名称：ICTCLASV1.2.rar　[点击查看]
上传用户：sunyong76
上传日期：2021-10-03
资源大小：2236k
文件大小：25k
源码类别：

多国语言处理

开发平台：

Java

Result.cpp：源码内容
  1. //////////////////////////////////////////////////////////////////////
  2. //ICTCLAS简介：计算所汉语词法分析系统ICTCLAS(Institute of Computing Technology, Chinese Lexical Analysis System)，
  3. //             功能有：中文分词；词性标注；未登录词识别。
  4. //             分词正确率高达97.58%(973专家评测结果)，
  5. //             未登录词识别召回率均高于90%，其中中国人名的识别召回率接近98%;
  6. //             处理速度为31.5Kbytes/s。
  7. //著作权：  Copyright?2002-2005中科院计算所 职务著作权人：张华平 刘群
  8. //遵循协议：自然语言处理开放资源许可证1.0
  9. //Email: zhanghp@software.ict.ac.cn
  10. //Homepage:www.nlp.org.cn;mtgroup.ict.ac.cn
  11. // Result.cpp: implementation of the CResult class.
  12. //
  13. //////////////////////////////////////////////////////////////////////
  15. #include "stdafx.h"
  17. #include "Result.h"
  18. #include <string.h>
  19. #include <stdio.h>
  20. #include <math.h>
  21. #include "..\Utility\Utility.h"
  24. //////////////////////////////////////////////////////////////////////
  25. // Construction/Destruction
  26. //////////////////////////////////////////////////////////////////////
  28. CResult::CResult()
  29. {
  30. //malloc buffer
  31. m_pResult=new PWORD_RESULT[MAX_SEGMENT_NUM];
  32. for(int i=0;i<MAX_SEGMENT_NUM;i++)
  33. {
  34. m_pResult[i]=new WORD_RESULT[MAX_WORDS];
  35. }
  37. m_dictCore.Load("data\coreDict.dct");
  38. m_POSTagger.LoadContext("data\lexical.ctx");
  39. /*
  40. m_dictCore.Load("data\Dict.dct");
  41. m_POSTagger.LoadContext("data\trainTest.ctx");
  42. */
  43. /*
  45. m_dictCore.AddItem("十分",'d'*256,+500);
  46. m_dictCore.AddItem("十分",'m'*256,-500);
  47. m_dictCore.AddItem("我国",'n'*256,-2000);
  48. m_dictCore.AddItem("我国",'r'*256,+2000);
  49. m_dictCore.AddItem("千年",'t'*256,200);
  51.     m_dictCore.Optimum();
  52. m_dictCore.Save("data\coreDictOptimum.dct");
  53. */
  56. m_POSTagger.SetTagType();
  58. m_uPerson.Configure("data\nr",TT_PERSON);
  59. //Set the person recognition configure
  60. m_uPlace.Configure("data\ns",TT_PLACE);
  61. //Set the place recognition configure
  62. m_uTransPerson.Configure("data\tr",TT_TRANS_PERSON);
  63. //Set the transliteration person recognition configure
  65. m_nOperateType=2;//0:Only Segment;1: First Tag; 2:Second Type
  66. m_nOutputFormat=0;//0:PKU criterion;1:973 criterion; 2: XML criterion
  68. m_dSmoothingPara=0.1;//Smoothing parameter
  69. m_dictBigram.Load("data\BigramDict.dct");
  71. }
  73. CResult::~CResult()
  74. {
  75. //free buffer
  76. for(int i=0;i<MAX_SEGMENT_NUM;i++)
  77. {
  78. delete [] m_pResult[i];
  79. }
  80. /* 
  81.      * ----- commented by huangjin@ict.ac.cn 2006-5-29 ------ 
  82.      * 
  83.      *  delete m_pResult;
  84.      *
  85.      */
  86. /*----Added By huangjin@ict.ac.cn 2006-5-29----*/
  87. delete []m_pResult;
  88. m_pResult=NULL;
  89. /*---------------------------------------------*/
  91. }
  93. bool CResult::Output(PWORD_RESULT pItem, CString &sResult,bool bFirstWordIgnore)
  94. {
  95. int i=0;
  96. char sTempBuffer[WORD_MAXLENGTH],sPOS[3];
  97. sPOS[2]=0;
  98. sResult="";
  99. if(bFirstWordIgnore)//Ignore first valid
  100. i=1;
  101. while(pItem[i].sWord[0]!=0&&pItem[i].nHandle!=CT_SENTENCE_END)//Not sentence ending flag
  102. {
  103. //Get the POS string
  104. if(m_nOutputFormat!=0)//Not PKU format
  105. PKU2973POS(pItem[i].nHandle,sPOS);
  106. else//PKU format
  107. {
  108. sPOS[0]=pItem[i].nHandle/256;
  109. sPOS[1]=pItem[i].nHandle%256;
  110. }
  111. sPOS[m_nOperateType]=0;//Set the sPOS with operate type
  113. if(m_nOutputFormat==0)//PKU format
  114. {
  115. sprintf(sTempBuffer,"%s",pItem[i].sWord);
  116.     //strcat(sResult,sTempBuffer);
  117.             sResult+=sTempBuffer;
  118. if(sPOS[0]!=0)//need POS 
  119. {
  120. sprintf(sTempBuffer,"/%s",sPOS);
  121. //strcat(sResult,sTempBuffer);
  122.                 sResult+=sTempBuffer;
  123. }
  124.     //strcat(sResult,"  ");
  125.             sResult+="  ";
  126. }
  127. else if(m_nOutputFormat==1)//973 format
  128. {
  129. sprintf(sTempBuffer,"%s\",pItem[i].sWord);
  130.     //strcat(sResult,sTempBuffer);
  131.             sResult+=sTempBuffer;
  132. if(sPOS[0]!=0)//need POS 
  133. {
  134. sprintf(sTempBuffer,"[%s]",sPOS);
  135. //strcat(sResult,sTempBuffer);
  136.                 sResult+=sTempBuffer;
  137. }
  138. }
  139. else if(m_nOutputFormat==2)//XML format
  140. {
  141. if(sPOS[0]!=0)//POS
  142. {
  143. sprintf(sTempBuffer,"<any type=42%s42>",sPOS);
  144. //strcat(sResult,sTempBuffer);
  145.                 sResult+=sTempBuffer;
  146. }
  147. sprintf(sTempBuffer,"<src>%s</src>",pItem[i].sWord);
  148. //strcat(sResult,sTempBuffer);
  149.             sResult+=sTempBuffer;
  150. if(sPOS[0]!=0)
  151. {
  152. //strcat(sResult,"</any>");
  153.                 sResult+="</any>";
  154. }
  155. }
  156. i++;
  157. }
  158. return true;
  159. }
  161. bool CResult::Processing(char *sSentence,unsigned int nCount)
  162. {
  164. int nIndex;
  165. #if _ICT_DEBUG
  166. char *sSegment;
  167. sSegment=new char[MAX_SENTENCE_LEN*2];
  168. #endif
  170. /*----Added By huangjin@ict.ac.cn 2006-5-30----*/
  171. m_Seg.ClearSegmentWord();
  172. this->ResetWord();
  173. /*---------------------------------------------*/
  175. //Unigram segment
  176. //m_Seg.Segment(sSentence,m_dictCore,nCount);
  177. //Bigram segment
  179. m_Seg.BiSegment(sSentence, m_dSmoothingPara, m_dictCore, m_dictBigram, nCount);
  181. m_nResultCount=m_Seg.m_nSegmentCount;
  183. //Record the number of result
  184. for(nIndex=0;nIndex<m_Seg.m_nSegmentCount;nIndex++)
  185. {
  186. #if _ICT_DEBUG
  187. m_POSTagger.POSTagging(m_Seg.m_pWordSeg[nIndex],m_dictCore,m_dictCore);
  188. Output(m_Seg.m_pWordSeg[nIndex],sSegment);
  189. printf("POS Tag%d:%sn",nIndex+1,sSegment);
  190. #endif
  191. m_uPerson.Recognition(m_Seg.m_pWordSeg[nIndex],m_Seg.m_graphOptimum,m_Seg.m_graphSeg,m_dictCore);
  192. m_uTransPerson.Recognition(m_Seg.m_pWordSeg[nIndex],m_Seg.m_graphOptimum,m_Seg.m_graphSeg,m_dictCore);
  193. m_uPlace.Recognition(m_Seg.m_pWordSeg[nIndex],m_Seg.m_graphOptimum,m_Seg.m_graphSeg,m_dictCore);
  195. }
  197. //m_uPerson.Recognition(m_Seg.m_WordSeg[0],m_Seg.m_graphOptimum,m_Seg.m_graphSeg,m_dictCore);
  198. //Person Recognition
  199. #if _ICT_DEBUG
  200. printf("After person recognition.n");
  201. #endif
  202. //Unigram
  203. //m_Seg.OptimumSegmet(nCount);
  204. //Bigram
  205. m_Seg.BiOptimumSegment(nCount,m_dSmoothingPara,m_dictBigram,m_dictCore);
  209. for(nIndex=0;nIndex<m_Seg.m_nSegmentCount;nIndex++)
  210. {
  211. m_POSTagger.POSTagging(m_Seg.m_pWordSeg[nIndex],m_dictCore,m_dictCore);
  212. #if _ICT_DEBUG
  213. Output(m_Seg.m_pWordSeg[nIndex],sSegment);
  214. printf("POS Tag%d:%sn",nIndex+1,sSegment);
  215. #endif
  216. }
  217. #if _ICT_DEBUG
  218. printf("After Sorting.n");
  219. #endif
  220. Sort();//Sort the ending 
  221. #if _ICT_DEBUG
  222. for(nIndex=0;nIndex<m_Seg.m_nSegmentCount;nIndex++)
  223. {
  224. Output(m_pResult[nIndex],sSegment);
  225. printf("POS Tag%d(P=Exp(%f)):%sn",nIndex+1,m_dResultPossibility[nIndex],sSegment);
  226. }
  227. delete [] sSegment;
  228. #endif
  229. return true;
  230. }
  231. //Sort the segmentation and POS result according its possibility
  232. bool CResult::Sort()
  233. {
  234. ELEMENT_TYPE dPossibility[MAX_SEGMENT_NUM],dTemp;
  235. int nIndex[MAX_SEGMENT_NUM],nTemp;//Index
  237. memset(dPossibility,0,sizeof(dPossibility));
  238. //Init the possibility
  240. for(int i=0;i<m_Seg.m_nSegmentCount;i++)
  241. {//Computing the possibility
  242. dPossibility[i]=ComputePossibility(m_Seg.m_pWordSeg[i]);
  243. nIndex[i]=i;//Record the index
  244. }
  246. //Sort with Bubble sort algorithm
  247. for(i=0;i<m_Seg.m_nSegmentCount;i++)
  248. for(int j=i+1;j<m_Seg.m_nSegmentCount;j++)
  249. {
  250. if(dPossibility[i]<dPossibility[j])
  251. {//Swap the possition and value
  252. nTemp=nIndex[i];
  253. dTemp=dPossibility[i];
  254. nIndex[i]=nIndex[j];
  255. dPossibility[i]=dPossibility[j];
  256. nIndex[j]=nTemp;
  257. dPossibility[j]=dTemp;
  258. }
  259. }
  261. for(i=0;i<m_Seg.m_nSegmentCount;i++)
  262. {//Adjust the segmentation and POS result and store them in the final result array
  263.  //Store them according their possibility ascendly
  264. Adjust(m_Seg.m_pWordSeg[nIndex[i]],m_pResult[i]);
  265. m_dResultPossibility[i]=dPossibility[i];
  266. }
  267. return true;
  268. }
  269. //Compute the possibility of current segmentation and POS result
  270. ELEMENT_TYPE CResult::ComputePossibility(PWORD_RESULT pItem)
  271. {
  272. int i=0;
  273. ELEMENT_TYPE dResultPossibility=0;
  274. while(pItem[i].sWord[0]!=0)
  275. {
  276. dResultPossibility+=pItem[i].dValue;
  277. //Compute the possibility of logP(Wi|Ti)
  278. if(pItem[i+1].sWord[0]!=0)//Not the last one
  279. {//Compute the possibility of logP(Ti|Ti-1)
  280. dResultPossibility+=log((double)(m_POSTagger.m_context.GetContextPossibility(0,pItem[i].nHandle,pItem[i+1].nHandle)+1));
  281. dResultPossibility-=log((double)(m_POSTagger.m_context.GetFrequency(0,pItem[i].nHandle)+1));
  282. }
  283. i++;
  284. }
  285. return dResultPossibility;
  286. }
  288. //Adjust the result with some rules
  289. bool CResult::Adjust(PWORD_RESULT pItem,PWORD_RESULT pItemRet)
  290. {
  291. int i=0,j=0;
  292. unsigned int nLen;
  293. char sSurName[10],sSurName2[10],sGivenName[10];
  294. bool bProcessed=false;//Have been processed
  295. while(pItem[i].sWord[0]!=0)
  296. {
  297. nLen=strlen(pItem[i].sWord);
  298. bProcessed=false;
  300. //Rule1: adjust person name
  301. if(pItem[i].nHandle==28274&&ChineseNameSplit(pItem[i].sWord,sSurName,sSurName2,sGivenName,m_uPerson.m_dict)&&strcmp(pItem[i].sWord,"叶利钦")!=0)//'nr'
  302. {//Divide name into surname and given name
  304. if(sSurName[0])
  305. {
  306. strcpy(pItemRet[j].sWord,sSurName);
  307. pItemRet[j++].nHandle=28274;
  308. }
  309. if(sSurName2[0])
  310. {
  311. strcpy(pItemRet[j].sWord,sSurName2);
  312. pItemRet[j++].nHandle=28274;
  313. }
  314. if(sGivenName[0])
  315. {
  316. strcpy(pItemRet[j].sWord,sGivenName);
  317. pItemRet[j++].nHandle=28274;
  318. }
  319. bProcessed=true;
  320. }
  321. //Rule2 for overlap words ABB 一段段、一片片
  322. /* 
  323. * ----- commented by huangjin@ict.ac.cn 2006-7-17 ------ 
  324. *
  325. *  else if(pItem[i].nHandle==27904&&strlen(pItem[i+1].sWord)==2&&strcmp(pItem[i+1].sWord,pItem[i+2].sWord)==0)
  326. *
  327. */
  328. /*----Added By huangjin@ict.ac.cn 2006-7-17----*/
  329. else if(pItem[i].nHandle==27904&&
  330. strlen(pItem[i].sWord)<=2&&
  331. strlen(pItem[i+1].sWord)==2&&
  332. charType((unsigned char*)(pItem[i+1].sWord))==CT_CHINESE&&
  333. strcmp(pItem[i+1].sWord,pItem[i+2].sWord)==0)
  334. /*---------------------------------------------*/
  335. {//(pItem[i+1].nHandle/256=='q'||pItem[i+1].nHandle/256=='a')&&
  336. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  337. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  338. strcat(pItemRet[j].sWord,pItem[i+2].sWord);
  339. pItemRet[j].nHandle=27904;
  340. j+=1;
  341. i+=2;
  342. bProcessed=true;
  343. }
  344. //Rule3 for overlap words AA
  345. /* 
  346. * ----- commented by huangjin@ict.ac.cn 2006-7-17 ------ 
  347. *
  348. *  else if(nLen==2&&strcmp(pItem[i].sWord,pItem[i+1].sWord)==0)
  349. *
  350. */
  351. /*----Added By huangjin@ict.ac.cn 2006-7-17----*/
  352. else if(nLen==2&&
  353. charType((unsigned char*)(pItem[i].sWord))==CT_CHINESE&&
  354. strcmp(pItem[i].sWord,pItem[i+1].sWord)==0)
  355. /*---------------------------------------------*/
  356. {
  357. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  358. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  359.      //24832=='a'*256
  360. pItemRet[j].nHandle=24832;//a
  361. if(pItem[i].nHandle/256=='v'||pItem[i+1].nHandle/256=='v')//30208='v'8256
  362. {
  363. pItemRet[j].nHandle=30208;
  364. }
  365. if(pItem[i].nHandle/256=='n'||pItem[i+1].nHandle/256=='n')//30208='v'8256
  366. {
  367. pItemRet[j].nHandle='n'*256;
  368. }
  369. i+=1;
  370. if(strlen(pItem[i+1].sWord)==2)
  371. {//AAB:洗/洗/脸、蒙蒙亮
  372. if((pItemRet[j].nHandle==30208&&pItem[i+1].nHandle/256=='n')||
  373.    (pItemRet[j].nHandle==24832&&pItem[i+1].nHandle/256=='a')
  374.    )
  375. {
  376. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  377. i+=1;
  378. }
  379. }
  380. j+=1;
  381. bProcessed=true;
  382. }
  384. //Rule 4: AAB 洗/洗澡
  385. else if(nLen==2&&strncmp(pItem[i].sWord,pItem[i+1].sWord,2)==0&&strlen(pItem[i+1].sWord)==4&&(pItem[i].nHandle/256=='v'||pItem[i].nHandle==24832))//v,a
  386. {
  387. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  388. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  389.      //24832=='a'*256
  390. pItemRet[j].nHandle=24832;//'a'
  391. if(pItem[i].nHandle/256=='v'||pItem[i+1].nHandle/256=='v')//30208='v'8256
  392. {
  393. pItemRet[j].nHandle=30208;
  394. }
  396. i+=1;
  397. j+=1;
  398. bProcessed=true;
  399. }
  400. else if(pItem[i].nHandle/256=='u'&&pItem[i].nHandle%256)//uj,ud,uv,uz,ul,ug->u
  401. pItem[i].nHandle='u'*256;
  402. else if(nLen==2&&strncmp(pItem[i].sWord,pItem[i+1].sWord,2)==0&&strlen(pItem[i+1].sWord)==4&&strncmp(pItem[i+1].sWord+2,pItem[i+2].sWord,2)==0)
  403. {//AABB 朴朴素素 枝枝叶叶
  404. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  405. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  406. strcat(pItemRet[j].sWord,pItem[i+2].sWord);
  407. pItemRet[j].nHandle=pItem[i+1].nHandle;
  408. i+=2;
  409. j+=1;
  410. bProcessed=true;
  411. }
  412. else if(pItem[i].nHandle==28275)//PostFix
  413. {
  414. if(m_uPlace.m_dict.IsExist(pItem[i+1].sWord,4))
  415. {
  416. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  417. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  418. pItemRet[j].nHandle=28275;
  419. i+=1;
  420. j+=1;
  421. bProcessed=true;
  422. }
  423. else if(strlen(pItem[i+1].sWord)==2&&CC_Find("队",pItem[i+1].sWord))
  424. {
  425. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  426. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  427. pItemRet[j].nHandle=28276;
  428. i+=1;
  429. j+=1;
  430. bProcessed=true;
  431. }
  432. else if(strlen(pItem[i+1].sWord)==2&&CC_Find("语文字杯",pItem[i+1].sWord))
  433. {
  434. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  435. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  436. pItemRet[j].nHandle=28282;
  437. i+=1;
  438. j+=1;
  439. bProcessed=true;
  440. }
  441. else if(strlen(pItem[i+1].sWord)==2&&CC_Find("裔",pItem[i+1].sWord))
  442. {
  443. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  444. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  445. pItemRet[j].nHandle=28160;
  446. i+=1;
  447. j+=1;
  448. bProcessed=true;
  449. }
  450. }
  451. else if(pItem[i].nHandle==30208||pItem[i].nHandle==28160)//v
  452. {
  453. if(strlen(pItem[i+1].sWord)==2&&CC_Find("员",pItem[i+1].sWord))
  454. {
  455. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  456. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  457. pItemRet[j].nHandle=28160;
  458. i+=1;
  459. j+=1;
  460. bProcessed=true;
  461. }
  462. }
  463. else if(pItem[i].nHandle==28280)
  464. {//www/nx ./w sina/nx; ＥＩＭ/nx  -６０１/m 
  465. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  466. pItemRet[j].nHandle=28280;
  468. /* 
  469. * ----- commented by huangjin@ict.ac.cn 2006-5-29 ------ 
  470. * 
  471. *  while(pItem[i+1].nHandle==28280||strstr(".．",pItem[i+1].sWord)||(pItem[i+1].nHandle==27904&&IsAllNum((unsigned char *)pItem[i+1].sWord)))
  472. *
  473. */
  474. /*----Added By huangjin@ict.ac.cn 2006-5-30----*/
  475. while(pItem[i+1].sWord[0]!=0 
  476.   && ( pItem[i+1].nHandle==28280
  477. ||strstr(".．",pItem[i+1].sWord)
  478. ||(pItem[i+1].nHandle==27904&&IsAllNum((unsigned char *)pItem[i+1].sWord))))
  479. /*---------------------------------------------*/
  480. {
  481. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  482. i+=1;
  483. }
  484. j+=1;
  485. bProcessed=true;
  486. }
  487. /*----Added By huangjin@ict.ac.cn 2006-9-12----*/
  488. //additional rule for 三点/m　十五分
  489. else if(pItem[i].nHandle==27904&&
  490. !strncmp(pItem[i].sWord+strlen(pItem[i].sWord)-2,"点",2)&&
  491. pItem[i+1].sWord[0]!=0&&
  492. pItem[i+1].nHandle==29696&&
  493. !strncmp(pItem[i+1].sWord+strlen(pItem[i+1].sWord)-2,"分",2))
  494. {
  495. strcpy(pItemRet[j].sWord, pItem[i].sWord);
  496. pItemRet[j++].nHandle = pItem[i+1].nHandle;
  497. bProcessed=true;
  498. }
  500. /*---------------------------------------------*/
  501. if(!bProcessed)
  502. {//If not processed,that's mean: not need to adjust;
  503.  //just copy to the final result
  504. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  505. pItemRet[j++].nHandle=pItem[i].nHandle;
  506. }
  507. i++;
  508. }
  509. pItemRet[j].sWord[0]=0;//Set ending
  510. return true;
  511. }
  513. //Paragraph Segment and POS Tagging
  514. bool CResult::ParagraphProcessing(char *sParagraph,CString &sResult)
  515. {
  516. char *sSentence,sChar[3];
  517. // char *sSentenceResult;
  518.     CString sSentenceResult;
  519. unsigned int nLen=strlen(sParagraph)+13;
  521. sSentence=new char[nLen];//malloc buffer
  522. /* 
  523.      * ----- commented by huangjin@ict.ac.cn 2006-5-29 ------ 
  524.      * 
  525.      *  sSentenceResult=new char[nLen*3];//malloc buffer
  526.      *
  527.     */
  528. /*----Added By huangjin@ict.ac.cn 2006-5-31----*/
  529. //single letter + '/' + postag + ' ',so multiply 4 
  530. //sSentenceResult=new char[nLen*4];//malloc buffer
  531. /*---------------------------------------------*/
  533. sSentence[0]=0;
  534. /*----Added By huangjin@ict.ac.cn 2006-5-29----*/
  535. //sSentenceResult[0]=0;
  536.     sSentenceResult="";
  537. /*---------------------------------------------*/
  538. unsigned int nPosIndex=0,nParagraphLen=strlen(sParagraph),nSentenceIndex=0;
  539. sChar[2]=0;
  540. //sResult[0]=0;//Init the result
  541.     sResult="";
  542. bool bFirstIgnore=true;
  543. strcpy(sSentence,SENTENCE_BEGIN);//Add a sentence begin flag
  544. while(nPosIndex<nParagraphLen)
  545. {//Find a whole sentence which separated by ! . n r
  546. sChar[0]=sParagraph[nPosIndex];//Get a char
  547. sChar[1]=0;
  548. if(sParagraph[nPosIndex]<0)
  549. {//double byte char
  550. nPosIndex+=1;
  551. sChar[1]=sParagraph[nPosIndex];
  552. }
  553. nPosIndex+=1;
  554. /*
  555. #define  SEPERATOR_C_SENTENCE "。！？：；…"
  556. #define  SEPERATOR_C_SUB_SENTENCE "、，（）“”‘’"
  557. #define  SEPERATOR_E_SENTENCE "!?:;"
  558. #define  SEPERATOR_E_SUB_SENTENCE ",()42'"
  559. #define  SEPERATOR_LINK "nr 　"
  560. */
  561. if(CC_Find(SEPERATOR_C_SENTENCE,sChar)||CC_Find(SEPERATOR_C_SUB_SENTENCE,sChar)||strstr(SEPERATOR_E_SENTENCE,sChar)||strstr(SEPERATOR_E_SUB_SENTENCE,sChar)||strstr(SEPERATOR_LINK,sChar))
  562. {//Reach end of a sentence.Get a whole sentence
  563. if(!strstr(SEPERATOR_LINK,sChar))//Not link seperator
  564. {
  565. strcat(sSentence,sChar);
  566. }
  567. /*----Added By huangjin@ict.ac.cn 2006-7-17----*/
  568. //for ……, do not split them
  569. if(!strcmp(sChar,"…")&&nPosIndex<nParagraphLen-1)
  570. {
  571. sChar[0]=sParagraph[nPosIndex];
  572. sChar[1]=sParagraph[nPosIndex+1];
  573.                 if(strcmp(sChar,"…")==0)
  574. {
  575. nPosIndex+=2;
  576. strcat(sSentence,sChar);
  577. }
  578. else
  579. {
  580. strcpy(sChar,"…");//back off
  581. }
  582. }
  583. /*---------------------------------------------*/
  584. if(sSentence[0]!=0&&strcmp(sSentence,SENTENCE_BEGIN)!=0)
  585. {
  586. if(!strstr(SEPERATOR_C_SUB_SENTENCE,sChar)&&!strstr(SEPERATOR_E_SUB_SENTENCE,sChar))
  587. strcat(sSentence,SENTENCE_END);//Add sentence ending flag
  589. Processing(sSentence,1);//Processing and output the result of current sentence.
  590. Output(m_pResult[0],sSentenceResult,bFirstIgnore);//Output to the imediate result
  591. //bFirstIgnore=true;
  592. //strcat(sResult,sSentenceResult);//Store in the result buffer
  593.                 sResult+=sSentenceResult;
  594. }
  595. if(strstr(SEPERATOR_LINK,sChar))//Link the result with the SEPERATOR_LINK
  596. {
  597. //strcat(sResult,sChar);
  598.                 sResult+=sChar;
  599. strcpy(sSentence,SENTENCE_BEGIN);//Add a sentence begin flag
  601. //sSentence[0]=0;//New sentence, and begin new segmentation
  602. //bFirstIgnore=false;
  603. }
  604. /* 
  605. * ----- commented by huangjin@ict.ac.cn 2006-7-17 ------ 
  606. *
  607. *  else if(strstr(SEPERATOR_C_SENTENCE,sChar)||strstr(SEPERATOR_E_SENTENCE,sChar))
  608. *
  609. */
  610. /*----Added By huangjin@ict.ac.cn 2006-7-17----*/
  611. else if(CC_Find(SEPERATOR_C_SENTENCE,sChar)||strstr(SEPERATOR_E_SENTENCE,sChar))
  612. /*---------------------------------------------*/
  613. {
  614. strcpy(sSentence,SENTENCE_BEGIN);//Add a sentence begin flag
  615. //sSentence[0]=0;//New sentence, and begin new segmentation
  616. //bFirstIgnore=false;
  617. }
  618. else
  619. {
  620. strcpy(sSentence,sChar);//reset current sentence, and add the previous end at begin position
  621. }
  622. }
  623. else //Other chars and store in the sentence buffer
  624. strcat(sSentence,sChar);
  625. }
  626. if(sSentence[0]!=0&&strcmp(sSentence,SENTENCE_BEGIN)!=0)
  627. {
  628. strcat(sSentence,SENTENCE_END);//Add sentence ending flag
  629. Processing(sSentence,1);//Processing and output the result of current sentence.
  630. Output(m_pResult[0],sSentenceResult,bFirstIgnore);//Output to the imediate result
  631. //strcat(sResult,sSentenceResult);//Store in the result buffer
  632.         sResult+=sSentenceResult;
  633. }
  634. delete []  sSentence;//FREE sentence buffer 
  635. // delete []  sSentenceResult;//free buffer
  636. return true;
  637. }
  639. bool CResult::FileProcessing(char *sSourceFile,char *sResultFile)
  640. {
  641. FILE *fpSource,*fpResult;//The file pointer of read and write
  642. char *sParagraph;//,*sParagraphResult;
  643.     CString sParagraphResult;
  644. int nLineIndex=1;
  645. /* 
  646.      * ----- commented by huangjin@ict.ac.cn 2006-5-29 ------ 
  647.      * 
  648.      *  sParagraph=new char[4*1024];
  649.  * sParagraphResult=new char[8*1024];
  650.  * if((fpSource=fopen(sSourceFile,"rt"))==NULL)
  651.  * return false;//Cannot open the source file to read
  652.  * if((fpResult=fopen(sResultFile,"wt"))==NULL) 
  653.  * return false;//Cannot open the result  file to write
  654.      *
  655.     */
  656. /*----Added By huangjin@ict.ac.cn 2006-5-29----*/
  657. //malloc memory after the files are opened successfully
  658.  if((fpSource=fopen(sSourceFile,"rt"))==NULL)
  659.  return false;//Cannot open the source file to read
  660.  if((fpResult=fopen(sResultFile,"wt"))==NULL) 
  661.  return false;//Cannot open the result  file to write
  663.  sParagraph=new char[MAX_SENTENCE_LEN*2];
  664.  //sParagraphResult=new char[MAX_SENTENCE_LEN*4];
  665. /*--------------------------------------------*/
  667. if(m_nOutputFormat==2)//XML format
  668. fprintf(fpResult,"<?xml version=42 1.042 encoding=42gb231242?><result>");
  669. while(!feof(fpSource))
  670. {
  671. /* 
  672. * ----- commented by huangjin@ict.ac.cn 2006-7-23 ------ 
  673. *
  674. *  if(fgets(sParagraph,4*1024,fpSource)==0)//Get a paragrah
  675. *
  676. */
  677. /*----Added By huangjin@ict.ac.cn 2006-7-23----*/
  678. if(fgets(sParagraph,MAX_SENTENCE_LEN,fpSource)==0)//Get a paragrah
  679. /*---------------------------------------------*/
  680. continue;
  682. /*----Added By huangjin@ict.ac.cn 2006-7-23----*/
  683. //如果读到尾正好把一个全角字符截断了会特别处理一下，采取回退而不是多读
  684. if(strlen(sParagraph)==MAX_SENTENCE_LEN-1)
  685. {
  686. bool bTrunc=false;
  687. for(int i=0;i<MAX_SENTENCE_LEN-1;i++)
  688. {
  689. if(sParagraph[i]<0)
  690. {
  691. if(i==MAX_SENTENCE_LEN-2)//末尾是被截断的字符
  692. {
  693. bTrunc=true;
  694. break;
  695. }
  696. i++;
  697. }
  698. }
  699. if(bTrunc)
  700. {
  701. ungetc(sParagraph[MAX_SENTENCE_LEN-2],fpSource);
  702. sParagraph[MAX_SENTENCE_LEN-2]='';
  703. }
  704. }
  705. /*---------------------------------------------*/
  706. //TRACE("%dn",nLineIndex++);
  707. ParagraphProcessing(sParagraph,sParagraphResult);
  708. fprintf(fpResult,"%s",(LPCTSTR)sParagraphResult);
  709. }
  710. delete [] sParagraph;
  711. //delete [] sParagraphResult;
  712. fclose(fpSource);
  713. if(m_nOutputFormat==2)//XML format
  714. fprintf(fpResult,"</result>");
  715. fclose(fpResult);
  716. return true;
  717. }
  719. bool CResult::PKU2973POS(int nHandle, char *sPOS973)
  720. {
  721. int nHandleSet[46]={24832,24932,24935,24942,25088,25344,25600,25703,25856,26112,26368,26624,26880,27136,27392,27648,27904,28160,28263,28274,28275,28276,28280,28282,28416,28672,28928,29184,29440,29696,29799,29952,30052,30055,30058,30060,30070,30074,30208,30308,30311,30318,30464,30720,30976,31232};
  722. //   "a", "ad","ag","an","b", "c", "d", "dg","e", "f","g", "h", "i", "j", "k", "l", "m", "n", "ng","nr","ns","nt","nx","nz","o", "p", "q", "r", "s", "t", "tg","u", "ud","ug","uj","ul","uv","uz","v", "vd","vg","vn","w", "x", "y", "z"
  723. char sPOSRelated[46][3]={"a", "ad","ga","an","f", "c", "d", "d", "e","nd","g", "h", "i", "j", "k", "l", "m", "n", "gn","nh","ns","ni","ws", "nz","o", "p", "q", "r", "nl","nt","gt","u", "ud","ug","uj","ul","uv","uz","v", "vd","gv","vn","w", "x", "u", "a"};
  724. /* 
  725.  "Bg","gf",
  726.  "Rg","gr",
  727.  "Mg","gm",
  728.  "Yg","u",
  729.  "Ug","u",
  730.  "Qg","q",
  731. */
  733. int nIndex=BinarySearch(nHandle,nHandleSet,46);
  734. if(nIndex==-1)
  735. strcpy(sPOS973,"@");
  736. else
  737. strcpy(sPOS973,sPOSRelated[nIndex]);
  738. return true;
  739. }
  741. bool CResult::ChineseNameSplit(char *sPersonName, char *sSurname, char *sSurname2, char *sGivenName, CDictionary &personDict)
  742. {
  743. int nSurNameLen=4,nLen=strlen(sPersonName),nFreq,i=0,nCharType,nFreqGiven;
  744. char sTemp[3];
  745. if(nLen<3||nLen>8)//Not a traditional Chinese person name
  746. return false;
  747. while(i<nLen)//No Including non-CHinese char
  748. {
  749. nCharType=charType((unsigned char*)sPersonName+i);
  750. if(nCharType!=CT_CHINESE&&nCharType!=CT_OTHER)
  751. return false;
  752. i+=2;
  753. }
  754. sSurname2[0]=0;//init 
  755. strncpy(sSurname,sPersonName,nSurNameLen);
  756. sSurname[nSurNameLen]=0;
  757. if(!personDict.IsExist(sSurname,1))
  758. {
  759. nSurNameLen=2;
  760. sSurname[nSurNameLen]=0;
  761. if(!personDict.IsExist(sSurname,1))
  762. {
  763. nSurNameLen=0;
  764. sSurname[nSurNameLen]=0;
  765. }
  766. }
  767. strcpy(sGivenName,sPersonName+nSurNameLen);
  768. if(nLen>6)
  769. {
  770. strncpy(sTemp,sPersonName+nSurNameLen,2);
  771. sTemp[2]=0;//Get the second possible surname
  772. if(personDict.IsExist(sTemp,1))
  773. {//Hongkong women's name: Surname+surname+given name
  774. strcpy(sSurname2,sTemp);
  775. strcpy(sGivenName,sPersonName+nSurNameLen+2);
  776. }
  777. }
  778. nFreq=personDict.GetFrequency(sSurname,1);
  779. strncpy(sTemp,sGivenName,2);
  780. sTemp[2]=0;
  781. nFreqGiven=personDict.GetFrequency(sTemp,2);
  782. if(nSurNameLen!=4&&((nSurNameLen==0&&nLen>4)||strlen(sGivenName)>4||(GetForeignCharCount(sPersonName)>=3&&nFreq<personDict.GetFrequency("张",1)/40&&nFreqGiven<personDict.GetFrequency("华",2)/20)||(nFreq<10&&GetForeignCharCount(sGivenName)==(nLen-nSurNameLen)/2)))
  783. return false;
  785. if(nLen==4&&m_uPerson.IsGivenName(sPersonName))
  786. {//Single Surname+given name
  787. return false;
  788. }
  789. return true;
  790. }
  792. /*----Added By huangjin@ict.ac.cn 2006-5-30----*/
  793. void CResult::ResetWord(void)
  794. {
  795. if(m_pResult)
  796. {
  797. for(int i=0;i<MAX_SEGMENT_NUM;i++)
  798. {
  799. if(m_pResult[i])
  800. {
  801. /*---------------------------------------------*/
  802. /* 
  803. * ----- commented by huangjin@ict.ac.cn 2006-9-12 ------ 
  804. *
  805. *  for(int j=0;j<MAX_WORDS;j++)
  806. *
  807. */
  808. /*----Added By huangjin@ict.ac.cn 2006-9-12----*/
  809. for(int j=0;j<MAX_WORDS&&m_pResult[i][j].sWord[0];j++)
  810. /*---------------------------------------------*/
  811. {
  812. m_pResult[i][j].sWord[0]='';
  813. m_pResult[i][j].dValue=0.0;
  814. m_pResult[i][j].nHandle=0;
  815. }
  816. }
  817. }
  818. }
  819. }
  820. /*---------------------------------------------*/