func (engine *Engine) Init(options types.EngineInitOptions) { // 将线程数设置为CPU数 runtime.GOMAXPROCS(runtime.NumCPU()) // 初始化初始参数 if engine.initialized { log.Fatal("请勿重复初始化引擎") } options.Init() engine.initOptions = options engine.initialized = true if !options.NotUsingSegmenter { // 载入分词器词典 engine.segmenter.LoadDictionary(options.SegmenterDictionaries) // 初始化停用词 engine.stopTokens.Init(options.StopTokenFile) } // 初始化索引器和排序器 for shard := 0; shard < options.NumShards; shard++ { engine.indexers = append(engine.indexers, core.Indexer{}) engine.indexers[shard].Init(*options.IndexerInitOptions) engine.rankers = append(engine.rankers, core.Ranker{}) engine.rankers[shard].Init() } // 初始化分词器通道 engine.segmenterChannel = make( chan segmenterRequest, options.NumSegmenterThreads) // 初始化索引器通道 engine.indexerAddDocumentChannels = make( []chan indexerAddDocumentRequest, options.NumShards) engine.indexerLookupChannels = make( []chan indexerLookupRequest, options.NumShards) for shard := 0; shard < options.NumShards; shard++ { engine.indexerAddDocumentChannels[shard] = make( chan indexerAddDocumentRequest, options.IndexerBufferLength) engine.indexerLookupChannels[shard] = make( chan indexerLookupRequest, options.IndexerBufferLength) } // 初始化排序器通道 engine.rankerAddDocChannels = make( []chan rankerAddDocRequest, options.NumShards) engine.rankerRankChannels = make( []chan rankerRankRequest, options.NumShards) engine.rankerRemoveDocChannels = make( []chan rankerRemoveDocRequest, options.NumShards) for shard := 0; shard < options.NumShards; shard++ { engine.rankerAddDocChannels[shard] = make( chan rankerAddDocRequest, options.RankerBufferLength) engine.rankerRankChannels[shard] = make( chan rankerRankRequest, options.RankerBufferLength) engine.rankerRemoveDocChannels[shard] = make( chan rankerRemoveDocRequest, options.RankerBufferLength) } // 初始化持久化存储通道 if engine.initOptions.UsePersistentStorage { engine.persistentStorageIndexDocumentChannels = make([]chan persistentStorageIndexDocumentRequest, engine.initOptions.PersistentStorageShards) for shard := 0; shard < engine.initOptions.PersistentStorageShards; shard++ { engine.persistentStorageIndexDocumentChannels[shard] = make( chan persistentStorageIndexDocumentRequest) } engine.persistentStorageInitChannel = make( chan bool, engine.initOptions.PersistentStorageShards) } // 启动分词器 for iThread := 0; iThread < options.NumSegmenterThreads; iThread++ { go engine.segmenterWorker() } // 启动索引器和排序器 for shard := 0; shard < options.NumShards; shard++ { go engine.indexerAddDocumentWorker(shard) go engine.rankerAddDocWorker(shard) go engine.rankerRemoveDocWorker(shard) for i := 0; i < options.NumIndexerThreadsPerShard; i++ { go engine.indexerLookupWorker(shard) } for i := 0; i < options.NumRankerThreadsPerShard; i++ { go engine.rankerRankWorker(shard) } } // 启动持久化存储工作协程 if engine.initOptions.UsePersistentStorage { err := os.MkdirAll(engine.initOptions.PersistentStorageFolder, 0700) if err != nil { log.Fatal("无法创建目录", engine.initOptions.PersistentStorageFolder) } // 打开或者创建数据库 engine.dbs = make([]storage.Storage, engine.initOptions.PersistentStorageShards) for shard := 0; shard < engine.initOptions.PersistentStorageShards; shard++ { dbPath := engine.initOptions.PersistentStorageFolder + "/" + PersistentStorageFilePrefix + "." + strconv.Itoa(shard) db, err := storage.OpenStorage(dbPath) if db == nil || err != nil { log.Fatal("无法打开数据库", dbPath, ": ", err) } engine.dbs[shard] = db } // 从数据库中恢复 for shard := 0; shard < engine.initOptions.PersistentStorageShards; shard++ { go engine.persistentStorageInitWorker(shard) } // 等待恢复完成 for shard := 0; shard < engine.initOptions.PersistentStorageShards; shard++ { <-engine.persistentStorageInitChannel } for { runtime.Gosched() if engine.numIndexingRequests == engine.numDocumentsIndexed { break } } // 关闭并重新打开数据库 for shard := 0; shard < engine.initOptions.PersistentStorageShards; shard++ { engine.dbs[shard].Close() dbPath := engine.initOptions.PersistentStorageFolder + "/" + PersistentStorageFilePrefix + "." + strconv.Itoa(shard) db, err := storage.OpenStorage(dbPath) if db == nil || err != nil { log.Fatal("无法打开数据库", dbPath, ": ", err) } engine.dbs[shard] = db } for shard := 0; shard < engine.initOptions.PersistentStorageShards; shard++ { go engine.persistentStorageIndexDocumentWorker(shard) } } atomic.AddUint64(&engine.numDocumentsStored, engine.numIndexingRequests) }
func (engine *Engine) Init(options types.EngineInitOptions) { // 将线程数设置为CPU数 runtime.GOMAXPROCS(runtime.NumCPU()) // 初始化初始参数 if engine.initialized { log.Fatal("请勿重复初始化引擎") } options.Init() engine.initOptions = options engine.initialized = true // 载入分词器词典 engine.segmenter.LoadDictionary(options.SegmenterDictionaries) // 初始化停用词 engine.stopTokens.Init(options.StopTokenFile) // 初始化索引器和排序器 for shard := 0; shard < options.NumShards; shard++ { engine.indexers = append(engine.indexers, core.Indexer{}) engine.indexers[shard].Init(*options.IndexerInitOptions) engine.rankers = append(engine.rankers, core.Ranker{}) engine.rankers[shard].Init() } // 初始化分词器通道 engine.segmenterChannel = make( chan segmenterRequest, options.NumSegmenterThreads) // 初始化索引器通道 engine.indexerAddDocumentChannels = make( []chan indexerAddDocumentRequest, options.NumShards) engine.indexerLookupChannels = make( []chan indexerLookupRequest, options.NumShards) for shard := 0; shard < options.NumShards; shard++ { engine.indexerAddDocumentChannels[shard] = make( chan indexerAddDocumentRequest, options.IndexerBufferLength) engine.indexerLookupChannels[shard] = make( chan indexerLookupRequest, options.IndexerBufferLength) } // 初始化排序器通道 engine.rankerAddScoringFieldsChannels = make( []chan rankerAddScoringFieldsRequest, options.NumShards) engine.rankerRankChannels = make( []chan rankerRankRequest, options.NumShards) engine.rankerRemoveScoringFieldsChannels = make( []chan rankerRemoveScoringFieldsRequest, options.NumShards) for shard := 0; shard < options.NumShards; shard++ { engine.rankerAddScoringFieldsChannels[shard] = make( chan rankerAddScoringFieldsRequest, options.RankerBufferLength) engine.rankerRankChannels[shard] = make( chan rankerRankRequest, options.RankerBufferLength) engine.rankerRemoveScoringFieldsChannels[shard] = make( chan rankerRemoveScoringFieldsRequest, options.RankerBufferLength) } // 启动分词器 for iThread := 0; iThread < options.NumSegmenterThreads; iThread++ { go engine.segmenterWorker() } // 启动索引器和排序器 for shard := 0; shard < options.NumShards; shard++ { go engine.indexerAddDocumentWorker(shard) go engine.rankerAddScoringFieldsWorker(shard) go engine.rankerRemoveScoringFieldsWorker(shard) for i := 0; i < options.NumIndexerThreadsPerShard; i++ { go engine.indexerLookupWorker(shard) } for i := 0; i < options.NumRankerThreadsPerShard; i++ { go engine.rankerRankWorker(shard) } } }