开通博客赚积分
发布资源赚积分
分类

源码开发语言/平台
当前位置：首页 > 源码/资料 > 多国语言处理 > 查看源码
Result.cpp
资源名称：Codes and Application.rar　[点击查看]
上传用户：yxl0916
上传日期：2007-05-25
资源大小：2245k
文件大小：19k
源码类别：

多国语言处理

开发平台：

Visual C++

Result.cpp：源码内容
  1. //////////////////////////////////////////////////////////////////////
  2. //ICTCLAS简介：计算所汉语词法分析系统ICTCLAS(Institute of Computing Technology, Chinese Lexical Analysis System)，
  3. //             功能有：中文分词；词性标注；未登录词识别。
  4. //             分词正确率高达97.58%(973专家评测结果)，
  5. //             未登录词识别召回率均高于90%，其中中国人名的识别召回率接近98%;
  6. //             处理速度为31.5Kbytes/s。
  7. //著作权：  Copyright?2002-2005中科院计算所 职务著作权人：张华平 刘群
  8. //遵循协议：自然语言处理开放资源许可证1.0
  9. //Email: zhanghp@software.ict.ac.cn
  10. //Homepage:www.nlp.org.cn;mtgroup.ict.ac.cn
  11. // Result.cpp: implementation of the CResult class.
  12. //
  13. //////////////////////////////////////////////////////////////////////
  15. #include "stdafx.h"
  16. #include "Result.h"
  17. #include <string.h>
  18. #include <stdio.h>
  19. #include <math.h>
  20. #include "..\Utility\Utility.h"
  21. //////////////////////////////////////////////////////////////////////
  22. // Construction/Destruction
  23. //////////////////////////////////////////////////////////////////////
  25. CResult::CResult()
  26. {
  27. //malloc buffer
  28. m_pResult=new PWORD_RESULT[MAX_SEGMENT_NUM];
  29. for(int i=0;i<MAX_SEGMENT_NUM;i++)
  30. {
  31. m_pResult[i]=new WORD_RESULT[MAX_WORDS];
  32. }
  34. m_dictCore.Load("data\coreDict.dct");
  35. m_POSTagger.LoadContext("data\lexical.ctx");
  36. /*
  37. m_dictCore.Load("data\Dict.dct");
  38. m_POSTagger.LoadContext("data\trainTest.ctx");
  39. */
  40. /*
  42. m_dictCore.AddItem("十分",'d'*256,+500);
  43. m_dictCore.AddItem("十分",'m'*256,-500);
  44. m_dictCore.AddItem("我国",'n'*256,-2000);
  45. m_dictCore.AddItem("我国",'r'*256,+2000);
  46. m_dictCore.AddItem("千年",'t'*256,200);
  48.     m_dictCore.Optimum();
  49. m_dictCore.Save("data\coreDictOptimum.dct");
  50. */
  53. m_POSTagger.SetTagType();
  55. m_uPerson.Configure("data\nr",TT_PERSON);
  56. //Set the person recognition configure
  57. m_uPlace.Configure("data\ns",TT_PLACE);
  58. //Set the place recognition configure
  59. m_uTransPerson.Configure("data\tr",TT_TRANS_PERSON);
  60. //Set the transliteration person recognition configure
  62. m_nOperateType=2;//0:Only Segment;1: First Tag; 2:Second Type
  63. m_nOutputFormat=0;//0:PKU criterion;1:973 criterion; 2: XML criterion
  65. m_dSmoothingPara=0.1;//Smoothing parameter
  66. m_dictBigram.Load("data\BigramDict.dct");
  68. }
  70. CResult::~CResult()
  71. {
  72. //free buffer
  73. for(int i=0;i<MAX_SEGMENT_NUM;i++)
  74. {
  75. delete [] m_pResult[i];
  76. }
  77. delete m_pResult;
  79. }
  81. bool CResult::Output(PWORD_RESULT pItem, char *sResult,bool bFirstWordIgnore)
  82. {
  83. int i=0;
  84. char sTempBuffer[WORD_MAXLENGTH],sPOS[3];
  85. sPOS[2]=0;
  86. sResult[0]=0;
  87. if(bFirstWordIgnore)//Ignore first valid
  88. i=1;
  89. while(pItem[i].sWord[0]!=0&&pItem[i].nHandle!=CT_SENTENCE_END)//Not sentence ending flag
  90. {
  91. //Get the POS string
  92. if(m_nOutputFormat!=0)//Not PKU format
  93. PKU2973POS(pItem[i].nHandle,sPOS);
  94. else//PKU format
  95. {
  96. sPOS[0]=pItem[i].nHandle/256;
  97. sPOS[1]=pItem[i].nHandle%256;
  98. }
  99. sPOS[m_nOperateType]=0;//Set the sPOS with operate type
  101. if(m_nOutputFormat==0)//PKU format
  102. {
  103. sprintf(sTempBuffer,"%s",pItem[i].sWord);
  104.     strcat(sResult,sTempBuffer);
  105. if(sPOS[0]!=0)//need POS 
  106. {
  107. sprintf(sTempBuffer,"/%s",sPOS);
  108. strcat(sResult,sTempBuffer);
  109. }
  110.     strcat(sResult,"  ");
  111. }
  112. else if(m_nOutputFormat==1)//973 format
  113. {
  114. sprintf(sTempBuffer,"%s\",pItem[i].sWord);
  115.     strcat(sResult,sTempBuffer);
  116. if(sPOS[0]!=0)//need POS 
  117. {
  118. sprintf(sTempBuffer,"[%s]",sPOS);
  119. strcat(sResult,sTempBuffer);
  120. }
  121. }
  122. else if(m_nOutputFormat==2)//XML format
  123. {
  124. if(sPOS[0]!=0)//POS
  125. {
  126. sprintf(sTempBuffer,"<any type=42%s42>",sPOS);
  127. strcat(sResult,sTempBuffer);
  128. }
  129. sprintf(sTempBuffer,"<src>%s</src>",pItem[i].sWord);
  130. strcat(sResult,sTempBuffer);
  131. if(sPOS[0]!=0)
  132. {
  133. strcat(sResult,"</any>");
  134. }
  135. }
  136. i++;
  137. }
  138. return true;
  139. }
  141. bool CResult::Processing(char *sSentence,unsigned int nCount)
  142. {
  143. int nIndex;
  144. #if _ICT_DEBUG
  145. char *sSegment;
  146. sSegment=new char[MAX_SENTENCE_LEN*2];
  147. #endif
  148. //Unigram segment
  149. //m_Seg.Segment(sSentence,m_dictCore,nCount);
  150. //Bigram segment
  151. m_Seg.BiSegment(sSentence, m_dSmoothingPara,m_dictCore,m_dictBigram,nCount);
  153. m_nResultCount=m_Seg.m_nSegmentCount;
  154. //Record the number of result
  155. for(nIndex=0;nIndex<m_Seg.m_nSegmentCount;nIndex++)
  156. {
  157. #if _ICT_DEBUG
  158. m_POSTagger.POSTagging(m_Seg.m_pWordSeg[nIndex],m_dictCore,m_dictCore);
  159. Output(m_Seg.m_pWordSeg[nIndex],sSegment);
  160. printf("POS Tag%d:%sn",nIndex+1,sSegment);
  161. #endif
  162. m_uPerson.Recognition(m_Seg.m_pWordSeg[nIndex],m_Seg.m_graphOptimum,m_Seg.m_graphSeg,m_dictCore);
  163. m_uTransPerson.Recognition(m_Seg.m_pWordSeg[nIndex],m_Seg.m_graphOptimum,m_Seg.m_graphSeg,m_dictCore);
  164. m_uPlace.Recognition(m_Seg.m_pWordSeg[nIndex],m_Seg.m_graphOptimum,m_Seg.m_graphSeg,m_dictCore);
  165. }
  166. //m_uPerson.Recognition(m_Seg.m_WordSeg[0],m_Seg.m_graphOptimum,m_Seg.m_graphSeg,m_dictCore);
  167. //Person Recognition
  168. #if _ICT_DEBUG
  169. printf("After person recognition.n");
  170. #endif
  171. //Unigram
  172. //m_Seg.OptimumSegmet(nCount);
  173. //Bigram
  174. m_Seg.BiOptimumSegment(nCount,m_dSmoothingPara,m_dictBigram,m_dictCore);
  176. for(nIndex=0;nIndex<m_Seg.m_nSegmentCount;nIndex++)
  177. {
  178. m_POSTagger.POSTagging(m_Seg.m_pWordSeg[nIndex],m_dictCore,m_dictCore);
  179. #if _ICT_DEBUG
  180. Output(m_Seg.m_pWordSeg[nIndex],sSegment);
  181. printf("POS Tag%d:%sn",nIndex+1,sSegment);
  182. #endif
  183. }
  184. #if _ICT_DEBUG
  185. printf("After Sorting.n");
  186. #endif
  187. Sort();//Sort the ending 
  188. #if _ICT_DEBUG
  189. for(nIndex=0;nIndex<m_Seg.m_nSegmentCount;nIndex++)
  190. {
  191. Output(m_pResult[nIndex],sSegment);
  192. printf("POS Tag%d(P=Exp(%f)):%sn",nIndex+1,m_dResultPossibility[nIndex],sSegment);
  193. }
  194. delete [] sSegment;
  195. #endif
  196. return true;
  197. }
  198. //Sort the segmentation and POS result according its possibility
  199. bool CResult::Sort()
  200. {
  201. ELEMENT_TYPE dPossibility[MAX_SEGMENT_NUM],dTemp;
  202. int nIndex[MAX_SEGMENT_NUM],nTemp;//Index
  204. memset(dPossibility,0,sizeof(dPossibility));
  205. //Init the possibility
  207. for(int i=0;i<m_Seg.m_nSegmentCount;i++)
  208. {//Computing the possibility
  209. dPossibility[i]=ComputePossibility(m_Seg.m_pWordSeg[i]);
  210. nIndex[i]=i;//Record the index
  211. }
  213. //Sort with Bubble sort algorithm
  214. for(i=0;i<m_Seg.m_nSegmentCount;i++)
  215. for(int j=i+1;j<m_Seg.m_nSegmentCount;j++)
  216. {
  217. if(dPossibility[i]<dPossibility[j])
  218. {//Swap the possition and value
  219. nTemp=nIndex[i];
  220. dTemp=dPossibility[i];
  221. nIndex[i]=nIndex[j];
  222. dPossibility[i]=dPossibility[j];
  223. nIndex[j]=nTemp;
  224. dPossibility[j]=dTemp;
  225. }
  226. }
  228. for(i=0;i<m_Seg.m_nSegmentCount;i++)
  229. {//Adjust the segmentation and POS result and store them in the final result array
  230.  //Store them according their possibility ascendly
  231. Adjust(m_Seg.m_pWordSeg[nIndex[i]],m_pResult[i]);
  232. m_dResultPossibility[i]=dPossibility[i];
  233. }
  234. return true;
  235. }
  236. //Compute the possibility of current segmentation and POS result
  237. ELEMENT_TYPE CResult::ComputePossibility(PWORD_RESULT pItem)
  238. {
  239. int i=0;
  240. ELEMENT_TYPE dResultPossibility=0;
  241. while(pItem[i].sWord[0]!=0)
  242. {
  243. dResultPossibility+=pItem[i].dValue;
  244. //Compute the possibility of logP(Wi|Ti)
  245. if(pItem[i+1].sWord[0]!=0)//Not the last one
  246. {//Compute the possibility of logP(Ti|Ti-1)
  247. dResultPossibility+=log((double)(m_POSTagger.m_context.GetContextPossibility(0,pItem[i].nHandle,pItem[i+1].nHandle)+1));
  248. dResultPossibility-=log((double)(m_POSTagger.m_context.GetFrequency(0,pItem[i].nHandle)+1));
  249. }
  250. i++;
  251. }
  252. return dResultPossibility;
  253. }
  255. //Adjust the result with some rules
  256. bool CResult::Adjust(PWORD_RESULT pItem,PWORD_RESULT pItemRet)
  257. {
  258. int i=0,j=0;
  259. unsigned int nLen;
  260. char sSurName[10],sSurName2[10],sGivenName[10];
  261. bool bProcessed=false;//Have been processed
  262. while(pItem[i].sWord[0]!=0)
  263. {
  264. nLen=strlen(pItem[i].sWord);
  265. bProcessed=false;
  267. //Rule1: adjust person name
  268. if(pItem[i].nHandle==28274&&ChineseNameSplit(pItem[i].sWord,sSurName,sSurName2,sGivenName,m_uPerson.m_dict)&&strcmp(pItem[i].sWord,"叶利钦")!=0)//'nr'
  269. {//Divide name into surname and given name
  271. if(sSurName[0])
  272. {
  273. strcpy(pItemRet[j].sWord,sSurName);
  274. pItemRet[j++].nHandle=28274;
  275. }
  276. if(sSurName2[0])
  277. {
  278. strcpy(pItemRet[j].sWord,sSurName2);
  279. pItemRet[j++].nHandle=28274;
  280. }
  281. if(sGivenName[0])
  282. {
  283. strcpy(pItemRet[j].sWord,sGivenName);
  284. pItemRet[j++].nHandle=28274;
  285. }
  286. bProcessed=true;
  287. }
  288. //Rule2 for overlap words ABB 一段段、一片片
  289. else if(pItem[i].nHandle==27904&&strlen(pItem[i+1].sWord)==2&&strcmp(pItem[i+1].sWord,pItem[i+2].sWord)==0)
  290. {//(pItem[i+1].nHandle/256=='q'||pItem[i+1].nHandle/256=='a')&&
  291. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  292. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  293. strcat(pItemRet[j].sWord,pItem[i+2].sWord);
  294. pItemRet[j].nHandle=27904;
  295. j+=1;
  296. i+=2;
  297. bProcessed=true;
  298. }
  299. //Rule3 for overlap words AA
  300. else if(nLen==2&&strcmp(pItem[i].sWord,pItem[i+1].sWord)==0)
  301. {
  302. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  303. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  304.      //24832=='a'*256
  305. pItemRet[j].nHandle=24832;//a
  306. if(pItem[i].nHandle/256=='v'||pItem[i+1].nHandle/256=='v')//30208='v'8256
  307. {
  308. pItemRet[j].nHandle=30208;
  309. }
  310. if(pItem[i].nHandle/256=='n'||pItem[i+1].nHandle/256=='n')//30208='v'8256
  311. {
  312. pItemRet[j].nHandle='n'*256;
  313. }
  314. i+=1;
  315. if(strlen(pItem[i+1].sWord)==2)
  316. {//AAB:洗/洗/脸、蒙蒙亮
  317. if((pItemRet[j].nHandle==30208&&pItem[i+1].nHandle/256=='n')||
  318.    (pItemRet[j].nHandle==24832&&pItem[i+1].nHandle/256=='a')
  319.    )
  320. {
  321. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  322. i+=1;
  323. }
  324. }
  325. j+=1;
  326. bProcessed=true;
  327. }
  329. //Rule 4: AAB 洗/洗澡
  330. else if(nLen==2&&strncmp(pItem[i].sWord,pItem[i+1].sWord,2)==0&&strlen(pItem[i+1].sWord)==4&&(pItem[i].nHandle/256=='v'||pItem[i].nHandle==24832))//v,a
  331. {
  332. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  333. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  334.      //24832=='a'*256
  335. pItemRet[j].nHandle=24832;//'a'
  336. if(pItem[i].nHandle/256=='v'||pItem[i+1].nHandle/256=='v')//30208='v'8256
  337. {
  338. pItemRet[j].nHandle=30208;
  339. }
  341. i+=1;
  342. j+=1;
  343. bProcessed=true;
  344. }
  345. else if(pItem[i].nHandle/256=='u'&&pItem[i].nHandle%256)//uj,ud,uv,uz,ul,ug->u
  346. pItem[i].nHandle='u'*256;
  347. else if(nLen==2&&strncmp(pItem[i].sWord,pItem[i+1].sWord,2)==0&&strlen(pItem[i+1].sWord)==4&&strncmp(pItem[i+1].sWord+2,pItem[i+2].sWord,2)==0)
  348. {//AABB 朴朴素素 枝枝叶叶
  349. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  350. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  351. strcat(pItemRet[j].sWord,pItem[i+2].sWord);
  352. pItemRet[j].nHandle=pItem[i+1].nHandle;
  353. i+=2;
  354. j+=1;
  355. bProcessed=true;
  356. }
  357. else if(pItem[i].nHandle==28275)//PostFix
  358. {
  359. if(m_uPlace.m_dict.IsExist(pItem[i+1].sWord,4))
  360. {
  361. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  362. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  363. pItemRet[j].nHandle=28275;
  364. i+=1;
  365. j+=1;
  366. bProcessed=true;
  367. }
  368. else if(strlen(pItem[i+1].sWord)==2&&CC_Find("队",pItem[i+1].sWord))
  369. {
  370. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  371. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  372. pItemRet[j].nHandle=28276;
  373. i+=1;
  374. j+=1;
  375. bProcessed=true;
  376. }
  377. else if(strlen(pItem[i+1].sWord)==2&&CC_Find("语文字杯",pItem[i+1].sWord))
  378. {
  379. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  380. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  381. pItemRet[j].nHandle=28282;
  382. i+=1;
  383. j+=1;
  384. bProcessed=true;
  385. }
  386. else if(strlen(pItem[i+1].sWord)==2&&CC_Find("裔",pItem[i+1].sWord))
  387. {
  388. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  389. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  390. pItemRet[j].nHandle=28160;
  391. i+=1;
  392. j+=1;
  393. bProcessed=true;
  394. }
  395. }
  396. else if(pItem[i].nHandle==30208||pItem[i].nHandle==28160)//v
  397. {
  398. if(strlen(pItem[i+1].sWord)==2&&CC_Find("员",pItem[i+1].sWord))
  399. {
  400. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  401. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  402. pItemRet[j].nHandle=28160;
  403. i+=1;
  404. j+=1;
  405. bProcessed=true;
  406. }
  407. }
  408. else if(pItem[i].nHandle==28280)
  409. {//www/nx ./w sina/nx; ＥＩＭ/nx  -６０１/m 
  410. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  411. pItemRet[j].nHandle=28280;
  412. while(pItem[i+1].nHandle==28280||strstr(".．",pItem[i+1].sWord)||(pItem[i+1].nHandle==27904&&IsAllNum((unsigned char *)pItem[i+1].sWord)))
  413. {
  414. strcat(pItemRet[j].sWord,pItem[i+1].sWord);
  415. i+=1;
  416. }
  417. j+=1;
  418. bProcessed=true;
  419. }
  421. if(!bProcessed)
  422. {//If not processed,that's mean: not need to adjust;
  423.  //just copy to the final result
  424. strcpy(pItemRet[j].sWord,pItem[i].sWord);
  425. pItemRet[j++].nHandle=pItem[i].nHandle;
  426. }
  427. i++;
  428. }
  429. pItemRet[j].sWord[0]=0;//Set ending
  430. return true;
  431. }
  433. //Paragraph Segment and POS Tagging
  434. bool CResult::ParagraphProcessing(char *sParagraph,char *sResult)
  435. {
  436. char *sSentence,sChar[3];
  437. char *sSentenceResult;
  438. unsigned int nLen=strlen(sParagraph)+13;
  439. sSentence=new char[nLen];//malloc buffer
  440. sSentenceResult=new char[nLen*3];//malloc buffer
  441. sSentence[0]=0;
  442. unsigned int nPosIndex=0,nParagraphLen=strlen(sParagraph),nSentenceIndex=0;
  443. sChar[2]=0;
  444. sResult[0]=0;//Init the result
  445. bool bFirstIgnore=true;
  446. strcpy(sSentence,SENTENCE_BEGIN);//Add a sentence begin flag
  447. while(nPosIndex<nParagraphLen)
  448. {//Find a whole sentence which separated by ! . n r
  449. sChar[0]=sParagraph[nPosIndex];//Get a char
  450. sChar[1]=0;
  451. if(sParagraph[nPosIndex]<0)
  452. {//double byte char
  453. nPosIndex+=1;
  454. sChar[1]=sParagraph[nPosIndex];
  455. }
  456. nPosIndex+=1;
  457. /*
  458. #define  SEPERATOR_C_SENTENCE "。！？：；…"
  459. #define  SEPERATOR_C_SUB_SENTENCE "、，（）“”‘’"
  460. #define  SEPERATOR_E_SENTENCE "!?:;"
  461. #define  SEPERATOR_E_SUB_SENTENCE ",()42'"
  462. #define  SEPERATOR_LINK "nr 　"
  463. */
  464. if(CC_Find(SEPERATOR_C_SENTENCE,sChar)||CC_Find(SEPERATOR_C_SUB_SENTENCE,sChar)||strstr(SEPERATOR_E_SENTENCE,sChar)||strstr(SEPERATOR_E_SUB_SENTENCE,sChar)||strstr(SEPERATOR_LINK,sChar))
  465. {//Reach end of a sentence.Get a whole sentence
  466. if(!strstr(SEPERATOR_LINK,sChar))//Not link seperator
  467. {
  468. strcat(sSentence,sChar);
  469. }
  470. if(sSentence[0]!=0&&strcmp(sSentence,SENTENCE_BEGIN)!=0)
  471. {
  472. if(!strstr(SEPERATOR_C_SUB_SENTENCE,sChar)&&!strstr(SEPERATOR_E_SUB_SENTENCE,sChar))
  473. strcat(sSentence,SENTENCE_END);//Add sentence ending flag
  475. Processing(sSentence,1);//Processing and output the result of current sentence.
  476. Output(m_pResult[0],sSentenceResult,bFirstIgnore);//Output to the imediate result
  477. //bFirstIgnore=true;
  478. strcat(sResult,sSentenceResult);//Store in the result buffer
  479. }
  480. if(strstr(SEPERATOR_LINK,sChar))//Link the result with the SEPERATOR_LINK
  481. {
  482. strcat(sResult,sChar);
  483. strcpy(sSentence,SENTENCE_BEGIN);//Add a sentence begin flag
  485. //sSentence[0]=0;//New sentence, and begin new segmentation
  486. //bFirstIgnore=false;
  487. }
  488. else if(strstr(SEPERATOR_C_SENTENCE,sChar)||strstr(SEPERATOR_E_SENTENCE,sChar))
  489. {
  490. strcpy(sSentence,SENTENCE_BEGIN);//Add a sentence begin flag
  491. //sSentence[0]=0;//New sentence, and begin new segmentation
  492. //bFirstIgnore=false;
  493. }
  494. else
  495. {
  496. strcpy(sSentence,sChar);//reset current sentence, and add the previous end at begin position
  497. }
  498. }
  499. else //Other chars and store in the sentence buffer
  500. strcat(sSentence,sChar);
  501. }
  502. if(sSentence[0]!=0&&strcmp(sSentence,SENTENCE_BEGIN)!=0)
  503. {
  504. strcat(sSentence,SENTENCE_END);//Add sentence ending flag
  505. Processing(sSentence,1);//Processing and output the result of current sentence.
  506. Output(m_pResult[0],sSentenceResult,bFirstIgnore);//Output to the imediate result
  507. strcat(sResult,sSentenceResult);//Store in the result buffer
  508. }
  509. delete []  sSentence;//FREE sentence buffer 
  510. delete []  sSentenceResult;//free buffer
  511. return true;
  512. }
  514. bool CResult::FileProcessing(char *sSourceFile,char *sResultFile)
  515. {
  516. FILE *fpSource,*fpResult;//The file pointer of read and write
  517. char *sParagraph,*sParagraphResult;
  518. int nLineIndex=1;
  519. sParagraph=new char[4*1024];
  520. sParagraphResult=new char[8*1024];
  521.     if((fpSource=fopen(sSourceFile,"rt"))==NULL)
  522. return false;//Cannot open the source file to read
  523.     if((fpResult=fopen(sResultFile,"wt"))==NULL) 
  524. return false;//Cannot open the result  file to write
  525. if(m_nOutputFormat==2)//XML format
  526. fprintf(fpResult,"<?xml version=42 1.042 encoding=42gb231242?><result>");
  527. while(!feof(fpSource))
  528. {
  529. if(fgets(sParagraph,4*1024,fpSource)==0)//Get a paragrah
  530. continue;
  531. TRACE("%dn",nLineIndex++);
  532. ParagraphProcessing(sParagraph,sParagraphResult);
  533. fprintf(fpResult,"%s",sParagraphResult);
  534. }
  535. delete [] sParagraph;
  536. delete [] sParagraphResult;
  537. fclose(fpSource);
  538. if(m_nOutputFormat==2)//XML format
  539. fprintf(fpResult,"</result>");
  540. fclose(fpResult);
  541. return true;
  542. }
  544. bool CResult::PKU2973POS(int nHandle, char *sPOS973)
  545. {
  546. int nHandleSet[46]={24832,24932,24935,24942,25088,25344,25600,25703,25856,26112,26368,26624,26880,27136,27392,27648,27904,28160,28263,28274,28275,28276,28280,28282,28416,28672,28928,29184,29440,29696,29799,29952,30052,30055,30058,30060,30070,30074,30208,30308,30311,30318,30464,30720,30976,31232};
  547. //   "a", "ad","ag","an","b", "c", "d", "dg","e", "f","g", "h", "i", "j", "k", "l", "m", "n", "ng","nr","ns","nt","nx","nz","o", "p", "q", "r", "s", "t", "tg","u", "ud","ug","uj","ul","uv","uz","v", "vd","vg","vn","w", "x", "y", "z"
  548. char sPOSRelated[46][3]={"a", "ad","ga","an","f", "c", "d", "d", "e","nd","g", "h", "i", "j", "k", "l", "m", "n", "gn","nh","ns","ni","ws", "nz","o", "p", "q", "r", "nl","nt","gt","u", "ud","ug","uj","ul","uv","uz","v", "vd","gv","vn","w", "x", "u", "a"};
  549. /* 
  550.  "Bg","gf",
  551.  "Rg","gr",
  552.  "Mg","gm",
  553.  "Yg","u",
  554.  "Ug","u",
  555.  "Qg","q",
  556. */
  558. int nIndex=BinarySearch(nHandle,nHandleSet,46);
  559. if(nIndex==-1)
  560. strcpy(sPOS973,"@");
  561. else
  562. strcpy(sPOS973,sPOSRelated[nIndex]);
  563. return true;
  564. }
  566. bool CResult::ChineseNameSplit(char *sPersonName, char *sSurname, char *sSurname2, char *sGivenName, CDictionary &personDict)
  567. {
  568. int nSurNameLen=4,nLen=strlen(sPersonName),nFreq,i=0,nCharType,nFreqGiven;
  569. char sTemp[3];
  570. if(nLen<3||nLen>8)//Not a traditional Chinese person name
  571. return false;
  572. while(i<nLen)//No Including non-CHinese char
  573. {
  574. nCharType=charType((unsigned char*)sPersonName+i);
  575. if(nCharType!=CT_CHINESE&&nCharType!=CT_OTHER)
  576. return false;
  577. i+=2;
  578. }
  579. sSurname2[0]=0;//init 
  580. strncpy(sSurname,sPersonName,nSurNameLen);
  581. sSurname[nSurNameLen]=0;
  582. if(!personDict.IsExist(sSurname,1))
  583. {
  584. nSurNameLen=2;
  585. sSurname[nSurNameLen]=0;
  586. if(!personDict.IsExist(sSurname,1))
  587. {
  588. nSurNameLen=0;
  589. sSurname[nSurNameLen]=0;
  590. }
  591. }
  592. strcpy(sGivenName,sPersonName+nSurNameLen);
  593. if(nLen>6)
  594. {
  595. strncpy(sTemp,sPersonName+nSurNameLen,2);
  596. sTemp[2]=0;//Get the second possible surname
  597. if(personDict.IsExist(sTemp,1))
  598. {//Hongkong women's name: Surname+surname+given name
  599. strcpy(sSurname2,sTemp);
  600. strcpy(sGivenName,sPersonName+nSurNameLen+2);
  601. }
  602. }
  603. nFreq=personDict.GetFrequency(sSurname,1);
  604. strncpy(sTemp,sGivenName,2);
  605. sTemp[2]=0;
  606. nFreqGiven=personDict.GetFrequency(sTemp,2);
  607. if(nSurNameLen!=4&&((nSurNameLen==0&&nLen>4)||strlen(sGivenName)>4||(GetForeignCharCount(sPersonName)>=3&&nFreq<personDict.GetFrequency("张",1)/40&&nFreqGiven<personDict.GetFrequency("华",2)/20)||(nFreq<10&&GetForeignCharCount(sGivenName)==(nLen-nSurNameLen)/2)))
  608. return false;
  609. if(nLen==4&&m_uPerson.IsGivenName(sPersonName))
  610. {//Single Surname+given name
  611. return false;
  612. }
  613. return true;
  614. }