北大天网搜索引擎TSE分析及完全注释[6]倒排索引的建立的程序分析(2)

面的DocIndex程序输入一个Tianwang.raw.*****文件，会产生一下三个文件 Doc.idx, Url.idx, DocId2Url.idx，我们这里对DocSegment程序进行分析。

这里输入 Tianwang.raw.*****，Doc.idx，Url.idx.sort_uniq等三个文件，输出一个Tianwang.raw.***.seg 分词完毕的文件

intmain(intargc,char*argv[]) {stringstrLine,strFileName=argv[1]; CUrliUrl; vector<CUrl>vecCUrl; CDocumentiDocument; vector<CDocument>vecCDocument; unsignedintdocId=0; //ifstreamifs("Tianwang.raw.2559638448"); ifstreamifs(strFileName.c_str());//DocSegmentTianwang.raw.**** if(!ifs) {cerr<<"Cannotopentianwang.img.infoforinput/n"; return-1; } ifstreamifsUrl("Url.idx.sort_uniq");//排序并消重后的url字典 if(!ifsUrl) {cerr<<"CannotopenUrl.idx.sort_uniqforinput/n"; return-1; } ifstreamifsDoc("Doc.idx");//字典文件 if(!ifsDoc) {cerr<<"CannotopenDoc.idxforinput/n"; return-1; } while(getline(ifsUrl,strLine))//偏离url字典存入一个向量内存中 {charchksum[33]; intdocid; memset(chksum,0,33); sscanf(strLine.c_str(),"%s%d",chksum,&docid); iUrl.m_sChecksum=chksum; iUrl.m_nDocId=docid; vecCUrl.push_back(iUrl); } while(getline(ifsDoc,strLine))//偏离字典文件将其放入一个向量内存中 {intdocid,pos,length; charchksum[33]; memset(chksum,0,33); sscanf(strLine.c_str(),"%d%d%d%s",&docid,&pos,&length,chksum); iDocument.m_nDocId=docid; iDocument.m_nPos=pos; iDocument.m_nLength=length; iDocument.m_sChecksum=chksum; vecCDocument.push_back(iDocument); } strFileName+=".seg"; ofstreamfout(strFileName.c_str(),ios::in|ios::out|ios::trunc|ios::binary);//设置完成分词后的数据输出文件 for(docId=0;docId<MAX_DOC_ID;docId++) {//finddocumentaccordingtodocId intlength=vecCDocument[docId+1].m_nPos-vecCDocument[docId].m_nPos-1; char*pContent=newchar[length+1]; memset(pContent,0,length+1); ifs.seekg(vecCDocument[docId].m_nPos); ifs.read(pContent,length); char*s; s=pContent; //skipHead intbytesRead=0,newlines=0; while(newlines!=2&&bytesRead!=HEADER_BUF_SIZE-1) {if(*s=='/n') newlines++; else newlines=0; s++; bytesRead++; } if(bytesRead==HEADER_BUF_SIZE-1)continue; //skipheader bytesRead=0,newlines=0; while(newlines!=2&&bytesRead!=HEADER_BUF_SIZE-1) {if(*s=='/n') newlines++; else newlines=0; s++; bytesRead++; } if(bytesRead==HEADER_BUF_SIZE-1)continue; //iDocument.m_sBody=s; iDocument.RemoveTags(s);//去除<> iDocument.m_sBodyNoTags=s; delete[]pContent; stringstrLine=iDocument.m_sBodyNoTags; CStrFun::ReplaceStr(strLine,"",""); CStrFun::EmptyStr(strLine);//set"/t/r/n"to"" //segmentthedocument具体分词处理 CHzSegiHzSeg; strLine=iHzSeg.SegmentSentenceMM(iDict,strLine); fout<<docId<<endl<<strLine; fout<<endl; } return(0); }

这里只是浮光掠影式的过一遍大概的代码，后面我会有专题详细讲解 parse html 和 segment docment 等技术