sphinx源码分析之搜索(search)

此处分析用的源码为最新版本1.0 beta版的。

//search.cpp int main ( int argc, char ** argv )

让我们从程序入口点开始进行旅程。search的入口点在文件search.cpp中，打开后定位到int main ( int argc, char ** argv )开始我们的分析：

在main中开始部分进行参数检查和配置信息的load，先进行命令行参数的检查和设置，如下：

/// // get query and other commandline options /// CSphQuery tQuery; char sQuery [ 1024 ]; sQuery[0] = '/0'; const char * sOptConfig = NULL; const char * sIndex = NULL; bool bNoInfo = false; bool bStdin = false; int iStart = 0; int iLimit = 20; #define OPT(_a1,_a2) else if ( !strcmp(argv[i],_a1) || !strcmp(argv[i],_a2) ) #define OPT1(_a1) else if ( !strcmp(argv[i],_a1) ) int i; for ( i=1; i<argc; i++ ) { if ( argv[i][0]=='-' ) { // this is an option if ( i==0 ); OPT ( "-a", "--any" ) tQuery.m_eMode = SPH_MATCH_ANY; OPT ( "-b", "--boolean" ) tQuery.m_eMode = SPH_MATCH_BOOLEAN; OPT ( "-p", "--phrase" ) tQuery.m_eMode = SPH_MATCH_PHRASE; OPT ( "-e", "--ext" ) tQuery.m_eMode = SPH_MATCH_EXTENDED; OPT ( "-e2", "--ext2" ) tQuery.m_eMode = SPH_MATCH_EXTENDED2; OPT ( "-q", "--noinfo" ) bNoInfo = true; OPT1 ( "--sort=date" ) tQuery.m_eSort = SPH_SORT_ATTR_DESC; OPT1 ( "--rsort=date" ) tQuery.m_eSort = SPH_SORT_ATTR_ASC; OPT1 ( "--sort=ts" ) tQuery.m_eSort = SPH_SORT_TIME_SEGMENTS; OPT1 ( "--stdin" ) bStdin = true; else if ( (i+1)>=argc ) break; OPT ( "-o", "--offset" ) iStart = atoi ( argv[++i] ); OPT ( "-l", "--limit" ) iLimit = atoi ( argv[++i] ); OPT ( "-c", "--config" ) sOptConfig = argv[++i]; OPT ( "-i", "--index" ) sIndex = argv[++i]; OPT ( "-g", "--group" ) { tQuery.m_eGroupFunc = SPH_GROUPBY_ATTR; tQuery.m_sGroupBy = argv[++i]; } OPT ( "-gs","--groupsort" ) { tQuery.m_sGroupSortBy = argv[++i]; } // NOLINT OPT ( "-s", "--sortby" ) { tQuery.m_eSort = SPH_SORT_EXTENDED; tQuery.m_sSortBy = argv[++i]; } OPT ( "-S", "--sortexpr" ) { tQuery.m_eSort = SPH_SORT_EXPR; tQuery.m_sSortBy = argv[++i]; } else if ( (i+2)>=argc ) break; OPT ( "-f", "--filter" ) { DWORD uVal = strtoul ( argv[i+2], NULL, 10 ); tQuery.m_dFilters.Reset (); tQuery.m_dFilters.Resize ( 1 ); tQuery.m_dFilters[0].m_eType = SPH_FILTER_VALUES; tQuery.m_dFilters[0].m_dValues.Reset (); tQuery.m_dFilters[0].m_dValues.Add ( uVal ); tQuery.m_dFilters[0].m_sAttrName = argv[i+1]; i += 2; } else break; // unknown option } else if ( strlen(sQuery) + strlen(argv[i]) + 1 < sizeof(sQuery) ) { // this is a search term strcat ( sQuery, argv[i] ); // NOLINT strcat ( sQuery, " " ); // NOLINT } } iStart = Max ( iStart, 0 ); iLimit = Max ( iLimit, 0 ); if ( i!=argc ) { fprintf ( stdout, "ERROR: malformed or unknown option near '%s'./n", argv[i] ); return 1; } #undef OPT if ( bStdin ) { int iPos = 0, iLeft = sizeof(sQuery)-1; char sThrowaway [ 256 ]; while ( !feof(stdin) ) { if ( iLeft>0 ) { int iLen = fread ( sQuery, 1, iLeft, stdin ); iPos += iLen; iLeft -= iLen; } else { int iDummy; // to avoid gcc unused result warning iDummy = fread ( sThrowaway, 1, sizeof(sThrowaway), stdin ); } } assert ( iPos<(int)sizeof(sQuery) ); sQuery[iPos] = '/0'; }

所有的查询信息都封装在CSphQuery tQuery;中，结构CSphQuery定义可以查看原文件，此处不知道也无关系影响不大，只要清楚它只是个容器的作用，用于盛放所以查询参数就ok了。

 

继续旅程，下面进入config的加载部分：

/ // configure / tQuery.m_iMaxMatches = Max ( 1000, iStart + iLimit ); CSphConfigParser cp; CSphConfig & hConf = cp.m_tConf; sphLoadConfig ( sOptConfig, false, cp );

可以看到通过类CSphConfigParser对配置文件进行解析，最终解析结果存放在结构CSphConfig中，其实CSphConfig就是一个hashtable一样的东东，config文件中的source、index、searchd等等所有节的东西都被加载进入了CSphConfig中。

 

现在所有的参数信息不管是命令行传入的还是从config文件加载进入的都已经就绪，下面就该开始真正的工作了。

首先说明一下查询的流程，查询就是一个打开索引文件，用输入的查询词在索引文件中挨个进行比较，找到满足关系的文档的过程，并读出文档，给每个文件打分，最后打分完成后进行排序，随后获取到排序后的文档列表的过程。

// // search // tQuery.m_sQuery = sQuery; CSphQueryResult * pResult = NULL; //查询结果存放指针 CSphIndex * pIndex = sphCreateIndexPhrase ( hIndex["path"].cstr() ); //用索引文件存放路径构造CSphIndex对象 pIndex->m_bEnableStar = ( hIndex.GetInt("enable_star")!=0 ); pIndex->SetWordlistPreload ( hIndex.GetInt("ondisk_dict")==0 );

 

数据库中可能存在多个索引文件，而查询一次在一个索引中进行。

for ( ; pIndex; ) { if ( !pIndex->Prealloc ( false, false, sWarning ) || !pIndex->Preread() ) { sError = pIndex->GetLastError (); break; } const CSphSchema * pSchema = &pIndex->GetMatchSchema();

pIndex->Prealloc ( false, false, sWarning )预先分配知足够内存存放cache数据

!pIndex->Preread() 预先读取所有的需要cache起来的数据，存放在Prealloc 分配的空间中
获取索引关联的数据模式const CSphSchema * pSchema = &pIndex->GetMatchSchema();

查询：

// do querying ISphMatchSorter * pTop = sphCreateQueue ( &tQuery, pIndex->GetMatchSchema(), sError );//创建一个优先级队列用于对文档集过滤，存放分值最高的文档doc if ( !pTop ) { sError.SetSprintf ( "failed to create sorting queue: %s", sError.cstr() ); break; } pResult = new CSphQueryResult(); //实例化结果对象CSphQueryResult if ( !pIndex->MultiQuery ( &tQuery, pResult, 1, &pTop, NULL ) ) //执行最终的查询动作 { // failure; pull that error message sError = pIndex->GetLastError(); SafeDelete ( pResult ); } else { // success; fold them matches pResult->m_dMatches.Reset (); pResult->m_iTotalMatches += pTop->GetTotalCount(); pResult->m_tSchema = pTop->GetSchema(); sphFlattenQueue ( pTop, pResult, 0 ); //用查询结果填充pResult，以便返回使用 } SafeDelete ( pTop );

pIndex->MultiQuery ( &tQuery, pResult, 1, &pTop, NULL ) )进行最终的查询过程。

函数MultiQuery的原型为virtual bool                MultiQuery ( const CSphQuery * pQuery, CSphQueryResult * pResult, int iSorters, ISphMatchSorter ** ppSorters, const CSphVector<CSphFilterSettings> * pExtraFilters, int iTag=0 ) const = 0;

在后面还将对其进行分析。

MultiQuery的查询结果由pResult和pTop返回，在pTop中存放的是docs数据，里面的数据需要进行sphFlattenQueue ( pTop, pResult, 0 )操作后才能copy到pResult中，sphFlattenQueue()的运算过程就是一个优先队列的出队操作，最后的结果就是rank从高到底的排列。pResult返回的结果是一些对全体文档都有效的数据，如字符串属性的字符串值、多值属性的值等。

 

查询得到结果后下面的代码只是进行简单的终端输出了，没什么好分析的了：

/ // print / if ( !pResult ) { fprintf ( stdout, "index '%s': search error: %s./n", sIndexName, sError.cstr() ); return 1; } fprintf ( stdout, "index '%s': query '%s': returned %d matches of %d total in %d.%03d sec/n", sIndexName, sQuery, pResult->m_dMatches.GetLength(), pResult->m_iTotalMatches, pResult->m_iQueryTime/1000, pResult->m_iQueryTime%1000 ); if ( !pResult->m_sWarning.IsEmpty() ) fprintf ( stdout, "WARNING: %s/n", pResult->m_sWarning.cstr() ); if ( pResult->m_dMatches.GetLength() ) { fprintf ( stdout, "/ndisplaying matches:/n" ); int iMaxIndex = Min ( iStart+iLimit, pResult->m_dMatches.GetLength() ); for ( int i=iStart; i<iMaxIndex; i++ ) { CSphMatch & tMatch = pResult->m_dMatches[i]; fprintf ( stdout, "%d. document=" DOCID_FMT ", weight=%d", 1+i, tMatch.m_iDocID, tMatch.m_iWeight ); for ( int j=0; j<pResult->m_tSchema.GetAttrsCount(); j++ ) { const CSphColumnInfo & tAttr = pResult->m_tSchema.GetAttr(j); fprintf ( stdout, ", %s=", tAttr.m_sName.cstr() ); if ( tAttr.m_eAttrType & SPH_ATTR_MULTI ) { fprintf ( stdout, "(" ); SphAttr_t iIndex = tMatch.GetAttr ( tAttr.m_tLocator ); if ( iIndex ) { const DWORD * pValues = pResult->m_pMva + iIndex; int iValues = *pValues++; for ( int k=0; k<iValues; k++ ) fprintf ( stdout, k ? ",%u" : "%u", *pValues++ ); } fprintf ( stdout, ")" ); } else switch ( tAttr.m_eAttrType ) { case SPH_ATTR_INTEGER: case SPH_ATTR_ORDINAL: case SPH_ATTR_BOOL: fprintf ( stdout, "%u", (DWORD)tMatch.GetAttr ( tAttr.m_tLocator ) ); break; case SPH_ATTR_TIMESTAMP: fprintf ( stdout, "%s", myctime ( (DWORD)tMatch.GetAttr ( tAttr.m_tLocator ) ) ); break; case SPH_ATTR_FLOAT: fprintf ( stdout, "%f", tMatch.GetAttrFloat ( tAttr.m_tLocator ) ); break; case SPH_ATTR_BIGINT: fprintf ( stdout, INT64_FMT, tMatch.GetAttr ( tAttr.m_tLocator ) ); break; case SPH_ATTR_STRING: { const BYTE * pStr; int iLen = sphUnpackStr ( pResult->m_pStrings + tMatch.GetAttr ( tAttr.m_tLocator ), &pStr );//从tMatch.GetAttr ( tAttr.m_tLocator )得到字符串属性的存放地址偏移加上pResult->m_pStrings就是对于的字符串值 fwrite ( pStr, 1, iLen, stdout ); break; } default: fprintf ( stdout, "(unknown-type-%d)", tAttr.m_eAttrType ); } } fprintf ( stdout, "/n" );

 

 

下面对文档排序器(ISphMatchSorter )的创建过程简单分析：

//sphinxsort.cpp ISphMatchSorter * sphCreateQueue ( const CSphQuery * pQuery, const CSphSchema & tSchema, CSphString & sError, bool bComputeItems )

此函数中大部分的代码段都是根据pQuery 中的配置对ISphMatchSorte人 进行定制的，源程序中的注释已经很详细，具体可以参考源代码。在进行了所有的需要定制的功能的设定后，最终如下面代码片段，进行实际的排序器构造：

switch ( eMatchFunc ) { case FUNC_REL_DESC: pTop = new CSphMatchQueue<MatchRelevanceLt_fn> ( pQuery->m_iMaxMatches, bUsesAttrs ); break; case FUNC_ATTR_DESC:pTop = new CSphMatchQueue<MatchAttrLt_fn> ( pQuery->m_iMaxMatches, bUsesAttrs ); break; case FUNC_ATTR_ASC: pTop = new CSphMatchQueue<MatchAttrGt_fn> ( pQuery->m_iMaxMatches, bUsesAttrs ); break; case FUNC_TIMESEGS: pTop = new CSphMatchQueue<MatchTimeSegments_fn> ( pQuery->m_iMaxMatches, bUsesAttrs ); break; case FUNC_GENERIC2: pTop = new CSphMatchQueue<MatchGeneric2_fn> ( pQuery->m_iMaxMatches, bUsesAttrs ); break; case FUNC_GENERIC3: pTop = new CSphMatchQueue<MatchGeneric3_fn> ( pQuery->m_iMaxMatches, bUsesAttrs ); break; case FUNC_GENERIC4: pTop = new CSphMatchQueue<MatchGeneric4_fn> ( pQuery->m_iMaxMatches, bUsesAttrs ); break; case FUNC_GENERIC5: pTop = new CSphMatchQueue<MatchGeneric5_fn> ( pQuery->m_iMaxMatches, bUsesAttrs ); break; case FUNC_CUSTOM: pTop = new CSphMatchQueue<MatchCustom_fn> ( pQuery->m_iMaxMatches, bUsesAttrs ); break; case FUNC_EXPR: pTop = new CSphMatchQueue<MatchExpr_fn> ( pQuery->m_iMaxMatches, bUsesAttrs ); break; default: pTop = NULL; }

 

 

还是按着查询的路线一直走下去，下面分析查询过程。

//sphinx.cpp bool CSphIndex_VLN::MultiQuery ( const CSphQuery * pQuery, CSphQueryResult * pResult, int iSorters, ISphMatchSorter ** ppSorters, const CSphVector<CSphFilterSettings> * pExtraFilters, int iTag ) const

bool CSphIndex_VLN::MultiQuery ( const CSphQuery * pQuery, CSphQueryResult * pResult, int iSorters, ISphMatchSorter ** ppSorters, const CSphVector<CSphFilterSettings> * pExtraFilters, int iTag ) const { assert ( pQuery ); MEMORY ( SPH_MEM_IDX_DISK_MULTY_QUERY ); // to avoid the checking of a ppSorters's element for NULL on every next step, just filter out all nulls right here CSphVector<ISphMatchSorter*> dSorters; dSorters.Reserve ( iSorters ); for ( int i=0; i<iSorters; i++ ) if ( ppSorters[i] ) dSorters.Add ( ppSorters[i] ); iSorters = dSorters.GetLength(); // if we have anything to work with if ( iSorters==0 ) return false; // non-random at the start, random at the end dSorters.Sort ( CmpPSortersByRandom_fn() ); // fast path for scans if ( pQuery->m_sQuery.IsEmpty() ) return MultiScan ( pQuery, pResult, iSorters, &dSorters[0], pExtraFilters, iTag ); ISphTokenizer * pTokenizer = m_pTokenizer->Clone ( false ); //复制一个分词器，也许是为了多线程环境了高效和安全吧 CSphScopedPtr<CSphDict> tDict ( NULL ); CSphDict * pDict = SetupStarDict ( tDict, *pTokenizer );//给字典加个修饰以处理星号(*)查询 CSphScopedPtr<CSphDict> tDict2 ( NULL ); pDict = SetupExactDict ( tDict2, pDict, *pTokenizer ); // parse query XQQuery_t tParsed; /* 用输入的查询词(由pQuery->m_sQuery.cstr()得到)解析成词条(term)后，最终构造一棵查询树, * 这树结构纯属本人幻想的，事实是否如此还得以后分析，只是觉得二叉树好玩而已 * 如：a b c * and * // * and c * // * a b */ if ( !sphParseExtendedQuery ( tParsed, pQuery->m_sQuery.cstr(), pTokenizer, &m_tSchema, pDict ) ) { pResult->m_sError = tParsed.m_sParseError; SafeDelete ( pTokenizer ); return false; } // fixup stat's order sphDoStatsOrder ( tParsed.m_pRoot, *pResult ); // transform query if needed (quorum transform, keyword expansion, etc.) TransformQuorum ( &tParsed.m_pRoot ); if ( m_bExpandKeywords ) tParsed.m_pRoot = ExpandKeywords ( tParsed.m_pRoot, m_tSettings ); // flag common subtrees CSphVector<XQNode_t*> dTrees; dTrees.Add ( tParsed.m_pRoot ); int iCommonSubtrees = sphMarkCommonSubtrees ( dTrees ); CSphQueryNodeCache tNodeCache ( iCommonSubtrees, m_iMaxCachedDocs, m_iMaxCachedHits ); bool bResult = ParsedMultiQuery ( pQuery, pResult, iSorters, &dSorters[0], tParsed.m_pRoot, pDict, pExtraFilters, &tNodeCache, iTag );//用解析出的查询树进行最终的search工作 SafeDelete ( pTokenizer ); return bResult; }

 

//sphinx.cpp bool CSphIndex_VLN::ParsedMultiQuery ( const CSphQuery * pQuery, CSphQueryResult * pResult, int iSorters, ISphMatchSorter ** ppSorters, const XQNode_t * pRoot, CSphDict * pDict, const CSphVector<CSphFilterSettings> * pExtraFilters, CSphQueryNodeCache * pNodeCache, int iTag ) const

开头部分只是一些索引查询前的铺垫工作，为下一层接口提供需要的参数的封装：

// start counting int64_t tmQueryStart = sphMicroTimer(); /// // setup searching /// PROFILER_INIT (); PROFILE_BEGIN ( query_init ); // non-ready index, empty response! if ( m_bPreread.IsEmpty() || !m_bPreread[0] ) { pResult->m_sError = "index not preread"; return false; } // setup calculations and result schema CSphQueryContext tCtx; if ( !tCtx.SetupCalc ( pResult, ppSorters[0]->GetSchema(), m_tSchema, GetMVAPool() ) ) return false; // open files CSphAutofile tDoclist, tHitlist, tWordlist, tDummy; if ( !m_bKeepFilesOpen ) { if ( tDoclist.Open ( GetIndexFileName("spd"), SPH_O_READ, pResult->m_sError ) < 0 ) return false; if ( tHitlist.Open ( GetIndexFileName ( m_uVersion>=3 ? "spp" : "spd" ), SPH_O_READ, pResult->m_sError ) < 0 ) return false; if ( tWordlist.Open ( GetIndexFileName ( "spi" ), SPH_O_READ, pResult->m_sError ) < 0 ) return false; } // setup search terms DiskIndexQwordSetup_c tTermSetup ( m_bKeepFilesOpen ? m_tDoclistFile : tDoclist, m_bKeepFilesOpen ? m_tHitlistFile : tHitlist, m_bPreloadWordlist ? tDummy : ( m_bKeepFilesOpen ? m_tWordlistFile : tWordlist ) ); tTermSetup.m_pDict = pDict; tTermSetup.m_pIndex = this; tTermSetup.m_eDocinfo = m_tSettings.m_eDocinfo; tTermSetup.m_tMin.m_iDocID = m_tMin.m_iDocID; if ( m_tSettings.m_eDocinfo==SPH_DOCINFO_INLINE ) { tTermSetup.m_tMin.Clone ( m_tMin, m_tSchema.GetRowSize() ); tTermSetup.m_iInlineRowitems = m_tSchema.GetRowSize(); } tTermSetup.m_iDynamicRowitems = pResult->m_tSchema.GetDynamicSize(); if ( pQuery->m_uMaxQueryMsec>0 ) tTermSetup.m_iMaxTimer = sphMicroTimer() + pQuery->m_uMaxQueryMsec*1000; // max_query_time tTermSetup.m_pWarning = &pResult->m_sWarning; tTermSetup.m_bSetupReaders = true; tTermSetup.m_pCtx = &tCtx; tTermSetup.m_pNodeCache = pNodeCache; // bind weights tCtx.BindWeights ( pQuery, m_tSchema );

 

 

// setup query // must happen before index-level reject, in order to build proper keyword stats CSphScopedPtr<ISphRanker> pRanker ( sphCreateRanker ( pRoot, pQuery->m_eRanker, pResult, tTermSetup ) ); //创建一个打分器(ranker)，此打分器为下面find的每一个doc都进行打分处理 if ( !pRanker.Ptr() ) return false;

 

// // find and weight matching documents 查找和给每一个doc打分 // PROFILE_BEGIN ( query_match ); switch ( pQuery->m_eMode ) { case SPH_MATCH_ALL: case SPH_MATCH_PHRASE: case SPH_MATCH_ANY: case SPH_MATCH_EXTENDED: case SPH_MATCH_EXTENDED2: case SPH_MATCH_BOOLEAN: if ( !MatchExtended ( &tCtx, pQuery, iSorters, ppSorters, pRanker.Ptr(), iTag ) ) //从索引中进行最终的查询 return false; break; default: sphDie ( "INTERNAL ERROR: unknown matching mode (mode=%d)", pQuery->m_eMode ); } PROFILE_END ( query_match );

最终的查找，并对找到的doc进行ranking

 

现在我们已经从index中获取到docs，此时的doc缺乏docinfo信息，下面给它们安装上docinfo：

// adjust result sets for ( int iSorter=0; iSorter<iSorters; iSorter++ ) { ISphMatchSorter * pTop = ppSorters[iSorter]; // final lookup and/or calc bool bFinalLookup = !tCtx.m_bLookupFilter && !tCtx.m_bLookupSort; if ( pTop->GetLength() && ( bFinalLookup || tCtx.m_dCalcFinal.GetLength() ) ) { const int iCount = pTop->GetLength (); CSphMatch * const pHead = pTop->First(); CSphMatch * const pTail = pHead + iCount; for ( CSphMatch * pCur=pHead; pCur<pTail; pCur++ ) { if ( bFinalLookup ) CopyDocinfo ( &tCtx, *pCur, FindDocinfo ( pCur->m_iDocID ) );//给找到的doc安装docinfo tCtx.CalcFinal ( *pCur ); } } // mva and string pools ptrs pResult->m_pMva = m_pMva.GetWritePtr(); pResult->m_pStrings = m_pStrings.GetWritePtr(); //给查询结果对象设置全局字符串数据引用 }

 

//sphinx.cpp bool CSphIndex_VLN::MatchExtended ( CSphQueryContext * pCtx, const CSphQuery * pQuery, int iSorters, ISphMatchSorter ** ppSorters, ISphRanker * pRanker, int iTag ) const

bool CSphIndex_VLN::MatchExtended ( CSphQueryContext * pCtx, const CSphQuery * pQuery, int iSorters, ISphMatchSorter ** ppSorters, ISphRanker * pRanker, int iTag ) const { int iCutoff = pQuery->m_iCutoff; if ( iCutoff<=0 ) iCutoff = -1; // do searching CSphMatch * pMatch = pRanker->GetMatchesBuffer(); for ( ;; ) { int iMatches = pRanker->GetMatches ( pCtx->m_iWeights, pCtx->m_dWeights ); //从index获取到匹配的docs if ( iMatches<=0 ) break; for ( int i=0; i<iMatches; i++ ) { if ( pCtx->m_bLookupSort ) CopyDocinfo ( pCtx, pMatch[i], FindDocinfo ( pMatch[i].m_iDocID ) ); pCtx->CalcSort ( pMatch[i] ); if ( pCtx->m_pWeightFilter && !pCtx->m_pWeightFilter->Eval ( pMatch[i] ) ) continue; pMatch[i].m_iTag = iTag; bool bRand = false; bool bNewMatch = false; for ( int iSorter=0; iSorter<iSorters; iSorter++ ) { // all non-random sorters are in the beginning, // so we can avoid the simple 'first-element' assertion if ( !bRand && ppSorters[iSorter]->m_bRandomize ) { bRand = true; pMatch[i].m_iWeight = ( sphRand() & 0xffff ); if ( pCtx->m_pWeightFilter && !pCtx->m_pWeightFilter->Eval ( pMatch[i] ) ) break; } bNewMatch |= ppSorters[iSorter]->Push ( pMatch[i] ); //把doc放入排序队列中 } if ( bNewMatch ) if ( --iCutoff==0 ) break; } if ( iCutoff==0 ) break; } return true; }  

 

***********************************************************************************************************************************************

一直到现在，都停留在index抽象层中，没有涉及到任何具体的磁盘IO，从下面开始我们将进入的外存空间，进行具体的读磁盘索引操作。

***********************************************************************************************************************************************

//sphinxsearch.cpp template < typename STATE > int ExtRanker_T<STATE>::GetMatches ( int iFields, const int * pWeights )

从磁盘中读取匹配到的docs块和hits块

int ExtRanker_T<STATE>::GetMatches ( int iFields, const int * pWeights ) { STATE tState ( iFields, pWeights, this ); if ( !m_pRoot ) return 0; int iMatches = 0; const ExtHit_t * pHlist = m_pHitlist; const ExtDoc_t * pDocs = m_pDoclist; // warmup if necessary if ( !pHlist ) { if ( !pDocs ) pDocs = GetFilteredDocs (); //从spd文件中获取docs if ( !pDocs ) return iMatches; pHlist = m_pRoot->GetHitsChunk ( pDocs, m_uMaxID );//从spp中获取hits if ( !pHlist ) return iMatches; } // main matching loop const ExtDoc_t * pDoc = pDocs; for ( SphDocID_t uCurDocid=0; iMatches<ExtNode_i::MAX_DOCS; ) { // keep ranking while ( pHlist->m_uDocid==uCurDocid ) tState.Update ( pHlist++ ); // if hits block is over, get next block, but do *not* flush current doc if ( pHlist->m_uDocid==DOCID_MAX ) { assert ( pDocs ); pHlist = m_pRoot->GetHitsChunk ( pDocs, m_uMaxID ); //获取下一块 if ( pHlist ) continue; } // otherwise (new match or no next hits block), flush current doc if ( uCurDocid ) { assert ( uCurDocid==pDoc->m_uDocid ); Swap ( m_dMatches[iMatches], m_dMyMatches[pDoc-m_dMyDocs] ); m_dMatches[iMatches].m_iWeight = tState.Finalize ( m_dMatches[iMatches].m_iWeight ); iMatches++; } // boundary checks if ( !pHlist ) { // there are no more hits for current docs block; do we have a next one? assert ( pDocs ); pDoc = pDocs = GetFilteredDocs (); // we don't, so bail out if ( !pDocs ) break; // we do, get some hits pHlist = m_pRoot->GetHitsChunk ( pDocs, m_uMaxID ); assert ( pHlist ); // fresh docs block, must have hits } // skip until next good doc/hit pair assert ( pDoc->m_uDocid<=pHlist->m_uDocid ); while ( pDoc->m_uDocid<pHlist->m_uDocid ) pDoc++; //跳过不完整的docs，就是没有hits存在的doc assert ( pDoc->m_uDocid==pHlist->m_uDocid ); uCurDocid = pHlist->m_uDocid; } m_pDoclist = pDocs; //设置docs块，当 m_pHitlist = pHlist;//设置hits块 return iMatches; //返回结果数量 }

 

//sphinxsearch.cpp const ExtDoc_t * ExtRanker_c::GetFilteredDocs ()

这函数没什么好说的，对查询树找到的结果进行过滤，因为有些match是不想要的

const ExtDoc_t * ExtRanker_c::GetFilteredDocs () { for ( ;; ) { // get another chunk m_uMaxID = 0; const ExtDoc_t * pCand = m_pRoot->GetDocsChunk ( &m_uMaxID ); //通过查询树进行查找 if ( !pCand ) return NULL; // create matches, and filter them int iDocs = 0; while ( pCand->m_uDocid!=DOCID_MAX ) { m_tTestMatch.m_iDocID = pCand->m_uDocid; if ( pCand->m_pDocinfo ) memcpy ( m_tTestMatch.m_pDynamic, pCand->m_pDocinfo, m_iInlineRowitems*sizeof(CSphRowitem) ); if ( m_pIndex->EarlyReject ( m_pCtx, m_tTestMatch ) ) //丢掉不合格的match { pCand++; continue; } m_dMyDocs[iDocs] = *pCand; m_tTestMatch.m_iWeight = (int)( (pCand->m_fTFIDF+0.5f)*SPH_BM25_SCALE ); // FIXME! bench bNeedBM25 BM25算法微调 Swap ( m_tTestMatch, m_dMyMatches[iDocs] ); iDocs++; pCand++; } if ( iDocs ) { m_dMyDocs[iDocs].m_uDocid = DOCID_MAX; return m_dMyDocs; } } }

 

//sphinxsearch.cpp const ExtDoc_t * ExtAnd_c::GetDocsChunk ( SphDocID_t * pMaxID )

前面已经提到过查询是分解到一颗查询树上进行的，树叶子节点为原子词条，而分支节点为and、or等孩子节点的组合操作，从下面的函数可以看到这点，这里是一个and节点，它将两个孩子节点的查询结果进行集合运算中的and运算：

const ExtDoc_t * ExtAnd_c::GetDocsChunk ( SphDocID_t * pMaxID ) { m_uMaxID = 0; const ExtDoc_t * pCur0 = m_pCurDoc[0]; const ExtDoc_t * pCur1 = m_pCurDoc[1]; int iDoc = 0; CSphRowitem * pDocinfo = m_pDocinfo; for ( ;; ) { // if any of the pointers is empty, *and* there is no data yet, process next child chunk // if there is data, we can't advance, because child hitlist offsets would be lost if ( !pCur0 || !pCur1 ) { if ( iDoc!=0 ) break; if ( !pCur0 ) pCur0 = m_pChildren[0]->GetDocsChunk ( NULL ); //从第一个孩子获取doc if ( !pCur1 ) pCur1 = m_pChildren[1]->GetDocsChunk ( NULL ); //从第二个孩子获取doc if ( !pCur0 || !pCur1 ) { m_pCurDoc[0] = NULL; m_pCurDoc[1] = NULL; return NULL; } } // find common matches assert ( pCur0 && pCur1 ); while ( iDoc<MAX_DOCS-1 ) { //将两个孩子节点获取到的doc合并起来，过滤掉docId不相同的，这是一个and节点 // find next matching docid while ( pCur0->m_uDocid < pCur1->m_uDocid ) pCur0++; if ( pCur0->m_uDocid==DOCID_MAX ) { pCur0 = NULL; break; } while ( pCur1->m_uDocid < pCur0->m_uDocid ) pCur1++; if ( pCur1->m_uDocid==DOCID_MAX ) { pCur1 = NULL; break; } if ( pCur0->m_uDocid!=pCur1->m_uDocid ) continue; // emit it ExtDoc_t & tDoc = m_dDocs[iDoc++]; tDoc.m_uDocid = pCur0->m_uDocid; tDoc.m_uFields = pCur0->m_uFields | pCur1->m_uFields; tDoc.m_uHitlistOffset = -1; tDoc.m_fTFIDF = pCur0->m_fTFIDF + pCur1->m_fTFIDF; //将两个term的分值进行合并,也就是将两个独立的查询词进行加起来统一得到总的rank值 CopyExtDocinfo ( tDoc, *pCur0, &pDocinfo, m_iStride ); // skip it pCur0++; if ( pCur0->m_uDocid==DOCID_MAX ) pCur0 = NULL; pCur1++; if ( pCur1->m_uDocid==DOCID_MAX ) pCur1 = NULL; if ( !pCur0 || !pCur1 ) break; } } m_pCurDoc[0] = pCur0; m_pCurDoc[1] = pCur1; return ReturnDocsChunk ( iDoc, pMaxID ); }

 

//sphinxsearch.cpp const ExtDoc_t * ExtTerm_c::GetDocsChunk ( SphDocID_t * pMaxID )

下面看看对于原子词条是怎么读取和它相对应的docs列表的：

const ExtDoc_t * ExtTerm_c::GetDocsChunk ( SphDocID_t * pMaxID ) { if ( !m_pQword->m_iDocs ) return NULL; m_uMaxID = 0; // max_query_time if ( m_iMaxTimer>0 && sphMicroTimer()>=m_iMaxTimer ) { if ( m_pWarning ) *m_pWarning = "query time exceeded max_query_time"; return NULL; } // interrupt by sitgerm if ( m_bInterruptNow ) { if ( m_pWarning ) *m_pWarning = "Server shutdown in progress"; return NULL; } int iDoc = 0; CSphRowitem * pDocinfo = m_pDocinfo; while ( iDoc<MAX_DOCS-1 ) { const CSphMatch & tMatch = m_pQword->GetNextDoc ( pDocinfo ); if ( !tMatch.m_iDocID ) { m_pQword->m_iDocs = 0; break; } if (!( m_pQword->m_uFields & m_uFields )) continue; ExtDoc_t & tDoc = m_dDocs[iDoc++]; tDoc.m_uDocid = tMatch.m_iDocID; tDoc.m_pDocinfo = pDocinfo; tDoc.m_uHitlistOffset = m_pQword->m_iHitlistPos; //hit文件的偏移量 tDoc.m_uFields = m_pQword->m_uFields & m_uFields; // OPTIMIZE: only needed for phrase node tDoc.m_fTFIDF = float(m_pQword->m_uMatchHits) / float(m_pQword->m_uMatchHits+SPH_BM25_K1) * m_fIDF; //BM25进行权值计算 pDocinfo += m_iStride; } m_pHitDoc = NULL; return ReturnDocsChunk ( iDoc, pMaxID ); }

 

//sphinx.cpp virtual const CSphMatch & GetNextDoc ( DWORD * pDocinfo )

virtual const CSphMatch & GetNextDoc ( DWORD * pDocinfo ) { SphDocID_t iDelta = m_rdDoclist.UnzipDocid(); //读取文档ID差值 if ( iDelta ) { m_tDoc.m_iDocID += iDelta; //计算文档ID if ( INLINE_DOCINFO ) { assert ( pDocinfo ); for ( int i=0; i<m_iInlineAttrs; i++ ) pDocinfo[i] = m_rdDoclist.UnzipInt() + m_pInlineFixup[i]; } if ( INLINE_HITS ) { m_uMatchHits = m_rdDoclist.UnzipInt(); const DWORD uFirst = m_rdDoclist.UnzipInt(); if ( m_uMatchHits==1 ) { const DWORD uField = m_rdDoclist.UnzipInt(); // field and end marker m_iHitlistPos = uFirst | ( uField << 23 ) | ( U64C(1)<<63 ); m_uFields = 1 << ( uField >> 1 ); } else { m_uFields = uFirst; m_uHitPosition += m_rdDoclist.UnzipOffset(); m_iHitlistPos = m_uHitPosition; } } else { SphOffset_t iDeltaPos = m_rdDoclist.UnzipOffset(); assert ( iDeltaPos>=0 ); m_iHitlistPos += iDeltaPos; //计算hitlist偏移值，此偏移量等以后读取hitlist的时候用到 m_uFields = m_rdDoclist.UnzipInt(); //获取字段数 m_uMatchHits = m_rdDoclist.UnzipInt();//获取hits计数 } } else { m_tDoc.m_iDocID = 0; } return m_tDoc; }

 

对于从文件中读取hitslist和读docs文件差不多，这里不再给出分析过程。

 

分析了这么大半天终于接近尾声，从输入一个查询短语，一直到从index文件中读出相关的文档列表，走过了查询的主要过程，但之间还有什么忽略掉了的地方，一些细节的地方，但它们对弄懂search流程并不那么重要，要具体弄懂每一个细节的地方还得花大量的时间精力去分析它，但这里就不在深入下去了。