func ValidateBucket(cluster, bucket string, uuids []string) bool { var cinfo *common.ClusterInfoCache url, err := common.ClusterAuthUrl(cluster) if err == nil { cinfo, err = common.NewClusterInfoCache(url, DEFAULT_POOL) } if err != nil { logging.Fatalf("Indexer::Fail to init ClusterInfoCache : %v", err) common.CrashOnError(err) } cinfo.Lock() defer cinfo.Unlock() if err := cinfo.Fetch(); err != nil { logging.Errorf("Indexer::Fail to init ClusterInfoCache : %v", err) common.CrashOnError(err) } if nids, err := cinfo.GetNodesByBucket(bucket); err == nil && len(nids) != 0 { // verify UUID currentUUID := cinfo.GetBucketUUID(bucket) for _, uuid := range uuids { if uuid != currentUUID { return false } } return true } else { logging.Fatalf("Indexer::Error Fetching Bucket Info: %v Nids: %v", err, nids) return false } }
func (fdb *fdbSlice) deletePrimaryIndex(docid []byte, workerId int) { //logging.Tracef("ForestDBSlice::delete \n\tSliceId %v IndexInstId %v. Delete Key - %s", // fdb.id, fdb.idxInstId, docid) if docid == nil { common.CrashOnError(errors.New("Nil Primary Key")) return } //docid -> key format entry, err := NewPrimaryIndexEntry(docid) common.CrashOnError(err) //delete from main index t0 := time.Now() if err := fdb.main[workerId].DeleteKV(entry.Bytes()); err != nil { fdb.checkFatalDbError(err) logging.Errorf("ForestDBSlice::delete \n\tSliceId %v IndexInstId %v. Error deleting "+ "entry from main index for Doc %s. Error %v", fdb.id, fdb.idxInstId, docid, err) return } fdb.idxStats.Timings.stKVDelete.Put(time.Now().Sub(t0)) platform.AddInt64(&fdb.delete_bytes, int64(len(entry.Bytes()))) }
func (c *clustMgrAgent) handleGetGlobalTopology(cmd Message) { logging.Debugf("ClustMgr:handleGetGlobalTopology %v", cmd) //get the latest topology from manager metaIter, err := c.mgr.NewIndexDefnIterator() if err != nil { common.CrashOnError(err) } defer metaIter.Close() indexInstMap := make(common.IndexInstMap) for _, defn, err := metaIter.Next(); err == nil; _, defn, err = metaIter.Next() { var idxDefn common.IndexDefn idxDefn = *defn t, e := c.mgr.GetTopologyByBucket(idxDefn.Bucket) if e != nil { common.CrashOnError(e) } inst := t.GetIndexInstByDefn(idxDefn.DefnId) if inst == nil { logging.Warnf("ClustMgr:handleGetGlobalTopology Index Instance Not "+ "Found For Index Definition %v. Ignored.", idxDefn) continue } //for indexer, Ready state doesn't matter. Till index build, //the index stays in Created state. var state common.IndexState instState := common.IndexState(inst.State) if instState == common.INDEX_STATE_READY { state = common.INDEX_STATE_CREATED } else { state = instState } idxInst := common.IndexInst{InstId: common.IndexInstId(inst.InstId), Defn: idxDefn, State: state, Stream: common.StreamId(inst.StreamId), } indexInstMap[idxInst.InstId] = idxInst } c.supvCmdch <- &MsgClustMgrTopology{indexInstMap: indexInstMap} }
func (meta *metaNotifier) OnIndexBuild(indexDefnList []common.IndexDefnId, buckets []string) map[common.IndexInstId]error { logging.Infof("clustMgrAgent::OnIndexBuild Notification "+ "Received for Build Index %v", indexDefnList) respCh := make(MsgChannel) var indexInstList []common.IndexInstId for _, defnId := range indexDefnList { indexInstList = append(indexInstList, common.IndexInstId(defnId)) } meta.adminCh <- &MsgBuildIndex{indexInstList: indexInstList, respCh: respCh, bucketList: buckets} //wait for response if res, ok := <-respCh; ok { switch res.GetMsgType() { case CLUST_MGR_BUILD_INDEX_DDL_RESPONSE: errMap := res.(*MsgBuildIndexResponse).GetErrorMap() logging.Infof("clustMgrAgent::OnIndexBuild returns "+ "for Build Index %v", indexDefnList) return errMap case MSG_ERROR: logging.Errorf("clustMgrAgent::OnIndexBuild Error "+ "for Build Index %v. Error %v.", indexDefnList, res) err := res.(*MsgError).GetError() errMap := make(map[common.IndexInstId]error) for _, instId := range indexDefnList { errMap[common.IndexInstId(instId)] = errors.New(err.String()) } return errMap default: logging.Fatalf("clustMgrAgent::OnIndexBuild Unknown Response "+ "Received for Build Index %v. Response %v", indexDefnList, res) common.CrashOnError(errors.New("Unknown Response")) } } else { logging.Fatalf("clustMgrAgent::OnIndexBuild Unexpected Channel Close "+ "for Create Index %v", indexDefnList) common.CrashOnError(errors.New("Unknown Response")) } return nil }
func (meta *metaNotifier) OnIndexCreate(indexDefn *common.IndexDefn) error { logging.Infof("clustMgrAgent::OnIndexCreate Notification "+ "Received for Create Index %v", indexDefn) pc := meta.makeDefaultPartitionContainer() idxInst := common.IndexInst{InstId: common.IndexInstId(indexDefn.DefnId), Defn: *indexDefn, State: common.INDEX_STATE_CREATED, Pc: pc, } respCh := make(MsgChannel) meta.adminCh <- &MsgCreateIndex{mType: CLUST_MGR_CREATE_INDEX_DDL, indexInst: idxInst, respCh: respCh} //wait for response if res, ok := <-respCh; ok { switch res.GetMsgType() { case MSG_SUCCESS: logging.Infof("clustMgrAgent::OnIndexCreate Success "+ "for Create Index %v", indexDefn) return nil case MSG_ERROR: logging.Errorf("clustMgrAgent::OnIndexCreate Error "+ "for Create Index %v. Error %v.", indexDefn, res) err := res.(*MsgError).GetError() return err.cause default: logging.Fatalf("clustMgrAgent::OnIndexCreate Unknown Response "+ "Received for Create Index %v. Response %v", indexDefn, res) common.CrashOnError(errors.New("Unknown Response")) } } else { logging.Fatalf("clustMgrAgent::OnIndexCreate Unexpected Channel Close "+ "for Create Index %v", indexDefn) common.CrashOnError(errors.New("Unknown Response")) } return nil }
func siSplitEntry(entry []byte, tmp []byte) ([]byte, []byte) { e := secondaryIndexEntry(entry) sk, err := e.ReadSecKey(tmp) c.CrashOnError(err) docid, err := e.ReadDocId(sk) return sk, docid[len(sk):] }
func (c *clustMgrAgent) handleUpdateTopologyForIndex(cmd Message) { logging.Debugf("ClustMgr:handleUpdateTopologyForIndex %v", cmd) indexList := cmd.(*MsgClustMgrUpdate).GetIndexList() updatedFields := cmd.(*MsgClustMgrUpdate).GetUpdatedFields() updatedState := common.INDEX_STATE_NIL updatedStream := common.NIL_STREAM updatedError := "" for _, index := range indexList { if updatedFields.state { updatedState = index.State } if updatedFields.stream { updatedStream = index.Stream } if updatedFields.err { updatedError = index.Error } updatedBuildTs := index.BuildTs err := c.mgr.UpdateIndexInstance(index.Defn.Bucket, index.Defn.DefnId, updatedState, updatedStream, updatedError, updatedBuildTs) common.CrashOnError(err) } c.supvCmdch <- &MsgSuccess{} }
func addPartnInfoToProtoInst(cfg c.Config, cinfo *c.ClusterInfoCache, indexInst c.IndexInst, streamId c.StreamId, protoInst *protobuf.IndexInst) { switch partn := indexInst.Pc.(type) { case *c.KeyPartitionContainer: //Right now the fill the SinglePartition as that is the only //partition structure supported partnDefn := partn.GetAllPartitions() //TODO move this to indexer init. These addresses cannot change. //Better to get these once and store. cinfo.Lock() defer cinfo.Unlock() err := cinfo.Fetch() c.CrashOnError(err) nid := cinfo.GetCurrentNode() streamMaintAddr, err := cinfo.GetServiceAddress(nid, "indexStreamMaint") c.CrashOnError(err) streamInitAddr, err := cinfo.GetServiceAddress(nid, "indexStreamInit") c.CrashOnError(err) streamCatchupAddr, err := cinfo.GetServiceAddress(nid, "indexStreamCatchup") c.CrashOnError(err) var endpoints []string for _, p := range partnDefn { for _, e := range p.Endpoints() { //Set the right endpoint based on streamId switch streamId { case c.MAINT_STREAM: e = c.Endpoint(streamMaintAddr) case c.CATCHUP_STREAM: e = c.Endpoint(streamCatchupAddr) case c.INIT_STREAM: e = c.Endpoint(streamInitAddr) } endpoints = append(endpoints, string(e)) } } protoInst.SinglePartn = &protobuf.SinglePartition{ Endpoints: endpoints, } } }
func (meta *metaNotifier) OnIndexDelete(defnId common.IndexDefnId, bucket string) error { logging.Infof("clustMgrAgent::OnIndexDelete Notification "+ "Received for Drop IndexId %v", defnId) respCh := make(MsgChannel) //Treat DefnId as InstId for now meta.adminCh <- &MsgDropIndex{mType: CLUST_MGR_DROP_INDEX_DDL, indexInstId: common.IndexInstId(defnId), respCh: respCh, bucket: bucket} //wait for response if res, ok := <-respCh; ok { switch res.GetMsgType() { case MSG_SUCCESS: logging.Infof("clustMgrAgent::OnIndexDelete Success "+ "for Drop IndexId %v", defnId) return nil case MSG_ERROR: logging.Errorf("clustMgrAgent::OnIndexDelete Error "+ "for Drop IndexId %v. Error %v", defnId, res) err := res.(*MsgError).GetError() return err.cause default: logging.Fatalf("clustMgrAgent::OnIndexDelete Unknown Response "+ "Received for Drop IndexId %v. Response %v", defnId, res) common.CrashOnError(errors.New("Unknown Response")) } } else { logging.Fatalf("clustMgrAgent::OnIndexDelete Unexpected Channel Close "+ "for Drop IndexId %v", defnId) common.CrashOnError(errors.New("Unknown Response")) } return nil }
func (c *clustMgrAgent) handleIndexerReady(cmd Message) { logging.Debugf("ClustMgr:handleIndexerReady %v", cmd) err := c.mgr.NotifyIndexerReady() common.CrashOnError(err) c.supvCmdch <- &MsgSuccess{} }
func (s *fdbSnapshot) newIndexEntry(b []byte) IndexEntry { var entry IndexEntry var err error if s.slice.isPrimary { entry, err = BytesToPrimaryIndexEntry(b) } else { entry, err = BytesToSecondaryIndexEntry(b) } common.CrashOnError(err) return entry }
func (c *clustMgrAgent) handleDeleteBucket(cmd Message) { logging.Debugf("ClustMgr:handleDeleteBucket %v", cmd) bucket := cmd.(*MsgClustMgrUpdate).GetBucket() streamId := cmd.(*MsgClustMgrUpdate).GetStreamId() err := c.mgr.DeleteIndexForBucket(bucket, streamId) common.CrashOnError(err) c.supvCmdch <- &MsgSuccess{} }
//handleSupervisorCommands handles the messages from Supervisor //Each operation acquires the mutex to make the itself atomic. func (m *mutationMgr) handleSupervisorCommands(cmd Message) { switch cmd.GetMsgType() { case OPEN_STREAM: m.handleOpenStream(cmd) case ADD_INDEX_LIST_TO_STREAM: m.handleAddIndexListToStream(cmd) case REMOVE_INDEX_LIST_FROM_STREAM: m.handleRemoveIndexListFromStream(cmd) case REMOVE_BUCKET_FROM_STREAM: m.handleRemoveBucketFromStream(cmd) case CLOSE_STREAM: m.handleCloseStream(cmd) case CLEANUP_STREAM: m.handleCleanupStream(cmd) case MUT_MGR_PERSIST_MUTATION_QUEUE: m.handlePersistMutationQueue(cmd) case MUT_MGR_DRAIN_MUTATION_QUEUE: m.handleDrainMutationQueue(cmd) case MUT_MGR_GET_MUTATION_QUEUE_HWT: m.handleGetMutationQueueHWT(cmd) case MUT_MGR_GET_MUTATION_QUEUE_LWT: m.handleGetMutationQueueLWT(cmd) case UPDATE_INDEX_INSTANCE_MAP: m.handleUpdateIndexInstMap(cmd) case UPDATE_INDEX_PARTITION_MAP: m.handleUpdateIndexPartnMap(cmd) case MUT_MGR_ABORT_PERSIST: m.handleAbortPersist(cmd) case CONFIG_SETTINGS_UPDATE: m.handleConfigUpdate(cmd) default: logging.Fatalf("MutationMgr::handleSupervisorCommands Received Unknown Command %v", cmd) common.CrashOnError(errors.New("Unknown Command On Supervisor Channel")) } }
func main() { platform.HideConsole(true) defer platform.HideConsole(false) common.SeedProcess() logging.Infof("Indexer started with command line: %v\n", os.Args) flag.Parse() logging.SetLogLevel(logging.Level(*logLevel)) forestdb.Log = &logging.SystemLogger // setup cbauth if *auth != "" { up := strings.Split(*auth, ":") logging.Tracef("Initializing cbauth with user %v for cluster %v\n", up[0], *cluster) if _, err := cbauth.InternalRetryDefaultInit(*cluster, up[0], up[1]); err != nil { logging.Fatalf("Failed to initialize cbauth: %s", err) } } go platform.DumpOnSignal() go common.ExitOnStdinClose() config := common.SystemConfig config.SetValue("indexer.clusterAddr", *cluster) config.SetValue("indexer.numVbuckets", *numVbuckets) config.SetValue("indexer.enableManager", *enableManager) config.SetValue("indexer.adminPort", *adminPort) config.SetValue("indexer.scanPort", *scanPort) config.SetValue("indexer.httpPort", *httpPort) config.SetValue("indexer.streamInitPort", *streamInitPort) config.SetValue("indexer.streamCatchupPort", *streamCatchupPort) config.SetValue("indexer.streamMaintPort", *streamMaintPort) config.SetValue("indexer.storage_dir", *storageDir) storage_dir := config["indexer.storage_dir"].String() if err := os.MkdirAll(storage_dir, 0755); err != nil { common.CrashOnError(err) } _, msg := indexer.NewIndexer(config) if msg.GetMsgType() != indexer.MSG_SUCCESS { logging.Warnf("Indexer Failure to Init %v", msg) } logging.Infof("Indexer exiting normally\n") }
//If a snapshot marker has been received but no mutation for that snapshot, //the repairTs seqno will be outside the snapshot marker range and //DCP will refuse to accept such seqno for restart. Such VBs need to //use lastFlushTs or restartTs. func (ss *StreamState) adjustNonSnapAlignedVbs(repairTs *common.TsVbuuid, streamId common.StreamId, bucket string) { for _, vbno := range repairTs.GetVbnos() { seqno := repairTs.Seqnos[vbno] snapBegin := repairTs.Snapshots[vbno][0] snapEnd := repairTs.Snapshots[vbno][1] if !(seqno >= snapBegin && seqno <= snapEnd) { // First, use the last flush TS seqno if avaliable if fts, ok := ss.streamBucketLastFlushedTsMap[streamId][bucket]; ok && fts != nil { repairTs.Snapshots[vbno][0] = fts.Snapshots[vbno][0] repairTs.Snapshots[vbno][1] = fts.Snapshots[vbno][1] repairTs.Seqnos[vbno] = fts.Seqnos[vbno] snapBegin = repairTs.Snapshots[vbno][0] snapEnd = repairTs.Snapshots[vbno][1] } // If last flush TS is still out-of-bound, use last Snap-aligned flushed TS if available if !(repairTs.Seqnos[vbno] >= snapBegin && repairTs.Seqnos[vbno] <= snapEnd) { if fts, ok := ss.streamBucketLastSnapAlignFlushedTsMap[streamId][bucket]; ok && fts != nil { repairTs.Snapshots[vbno][0] = fts.Snapshots[vbno][0] repairTs.Snapshots[vbno][1] = fts.Snapshots[vbno][1] repairTs.Seqnos[vbno] = fts.Seqnos[vbno] snapBegin = repairTs.Snapshots[vbno][0] snapEnd = repairTs.Snapshots[vbno][1] } } // If last snap-aligned flushed TS is not avail, then use restartTS if !(repairTs.Seqnos[vbno] >= snapBegin && repairTs.Seqnos[vbno] <= snapEnd) { if rts, ok := ss.streamBucketRestartTsMap[streamId][bucket]; ok && rts != nil { //if no flush has been done yet, use restart TS repairTs.Snapshots[vbno][0] = rts.Snapshots[vbno][0] repairTs.Snapshots[vbno][1] = rts.Snapshots[vbno][1] repairTs.Seqnos[vbno] = rts.Seqnos[vbno] snapBegin = repairTs.Snapshots[vbno][0] snapEnd = repairTs.Snapshots[vbno][1] } } //if seqno is still not with snapshot range, then it is likely a bug in state management if !(repairTs.Seqnos[vbno] >= snapBegin && repairTs.Seqnos[vbno] <= snapEnd) { common.CrashOnError(errors.New("No Valid Restart Seqno Found")) } } } }
// NewProjector creates a news projector instance and // starts a corresponding adminport. func NewProjector(maxvbs int, config c.Config) *Projector { p := &Projector{ topics: make(map[string]*Feed), topicSerialize: make(map[string]*sync.Mutex), maxvbs: maxvbs, pooln: "default", // TODO: should this be configurable ? } // Setup dynamic configuration propagation config, err := c.GetSettingsConfig(config) c.CrashOnError(err) pconfig := config.SectionConfig("projector.", true /*trim*/) p.name = pconfig["name"].String() p.clusterAddr = pconfig["clusterAddr"].String() p.adminport = pconfig["adminport.listenAddr"].String() ef := config["projector.routerEndpointFactory"] config["projector.routerEndpointFactory"] = ef p.config = config p.ResetConfig(config) p.logPrefix = fmt.Sprintf("PROJ[%s]", p.adminport) callb := func(cfg c.Config) { logging.Infof("%v settings notifier from metakv\n", p.logPrefix) cfg.LogConfig(p.logPrefix) p.ResetConfig(cfg) } c.SetupSettingsNotifier(callb, make(chan struct{})) cluster := p.clusterAddr if !strings.HasPrefix(p.clusterAddr, "http://") { cluster = "http://" + cluster } apConfig := config.SectionConfig("projector.adminport.", true) apConfig.SetValue("name", "PRAM") reqch := make(chan ap.Request) p.admind = ap.NewHTTPServer(apConfig, reqch) watchInterval := config["projector.watchInterval"].Int() staleTimeout := config["projector.staleTimeout"].Int() go p.mainAdminPort(reqch) go p.watcherDameon(watchInterval, staleTimeout) logging.Infof("%v started ...\n", p.logPrefix) return p }
//checkFatalDbError checks if the error returned from DB //is fatal and stores it. This error will be returned //to caller on next DB operation func (fdb *fdbSlice) checkFatalDbError(err error) { //panic on all DB errors and recover rather than risk //inconsistent db state common.CrashOnError(err) errStr := err.Error() switch errStr { case "checksum error", "file corruption", "no db instance", "alloc fail", "seek fail", "fsync fail": fdb.fatalDbErr = err } }
//handleWorkerMessage handles messages from workers func (m *mutationMgr) handleWorkerMessage(cmd Message) { switch cmd.GetMsgType() { case STREAM_READER_STREAM_DROP_DATA, STREAM_READER_STREAM_BEGIN, STREAM_READER_STREAM_END, STREAM_READER_ERROR, STREAM_READER_CONN_ERROR, STREAM_READER_HWT: //send message to supervisor to take decision logging.Tracef("MutationMgr::handleWorkerMessage Received %v from worker", cmd) m.supvRespch <- cmd default: logging.Fatalf("MutationMgr::handleWorkerMessage Received unhandled "+ "message from worker %v", cmd) common.CrashOnError(errors.New("Unknown Message On Worker Channel")) } }
func (s *storageMgr) createSnapshotWorker(streamId common.StreamId, bucket string, tsVbuuid *common.TsVbuuid, indexSnapMap IndexSnapMap, numVbuckets int, indexInstMap common.IndexInstMap, indexPartnMap IndexPartnMap, stats *IndexerStats) { defer destroyIndexSnapMap(indexSnapMap) var needsCommit bool snapType := tsVbuuid.GetSnapType() if snapType == common.DISK_SNAP { needsCommit = true } var wg sync.WaitGroup //for every index managed by this indexer for idxInstId, partnMap := range indexPartnMap { // Create snapshots for all indexes in parallel wg.Add(1) go func(idxInstId common.IndexInstId, partnMap PartitionInstMap) { defer wg.Done() idxInst := indexInstMap[idxInstId] idxStats := stats.indexes[idxInst.InstId] lastIndexSnap := indexSnapMap[idxInstId] //if index belongs to the flushed bucket and stream if idxInst.Defn.Bucket == bucket && idxInst.Stream == streamId && idxInst.State != common.INDEX_STATE_DELETED { // List of snapshots for reading current timestamp var isSnapCreated bool = true partnSnaps := make(map[common.PartitionId]PartitionSnapshot) //for all partitions managed by this indexer for partnId, partnInst := range partnMap { var lastPartnSnap PartitionSnapshot if lastIndexSnap != nil { lastPartnSnap = lastIndexSnap.Partitions()[partnId] } sc := partnInst.Sc sliceSnaps := make(map[SliceId]SliceSnapshot) //create snapshot for all the slices for _, slice := range sc.GetAllSlices() { var latestSnapshot Snapshot if lastIndexSnap.Partitions() != nil { lastSliceSnap := lastPartnSnap.Slices()[slice.Id()] latestSnapshot = lastSliceSnap.Snapshot() } //if flush timestamp is greater than last //snapshot timestamp, create a new snapshot snapTs := NewTimestamp(numVbuckets) if latestSnapshot != nil { snapTsVbuuid := latestSnapshot.Timestamp() snapTs = getSeqTsFromTsVbuuid(snapTsVbuuid) } ts := getSeqTsFromTsVbuuid(tsVbuuid) //if the flush TS is greater than the last snapshot TS //and slice has some changes. Skip only in-memory snapshot //in case of unchanged data. if latestSnapshot == nil || (ts.GreaterThan(snapTs) && (slice.IsDirty() || needsCommit)) { newTsVbuuid := tsVbuuid.Copy() var err error var info SnapshotInfo var newSnapshot Snapshot logging.Tracef("StorageMgr::handleCreateSnapshot Creating New Snapshot "+ "Index: %v PartitionId: %v SliceId: %v Commit: %v", idxInstId, partnId, slice.Id(), needsCommit) snapCreateStart := time.Now() if info, err = slice.NewSnapshot(newTsVbuuid, needsCommit); err != nil { logging.Errorf("handleCreateSnapshot::handleCreateSnapshot Error "+ "Creating new snapshot Slice Index: %v Slice: %v. Skipped. Error %v", idxInstId, slice.Id(), err) isSnapCreated = false common.CrashOnError(err) continue } snapCreateDur := time.Since(snapCreateStart) idxStats := stats.indexes[idxInstId] idxStats.numSnapshots.Add(1) if needsCommit { idxStats.numCommits.Add(1) } snapOpenStart := time.Now() if newSnapshot, err = slice.OpenSnapshot(info); err != nil { logging.Errorf("StorageMgr::handleCreateSnapshot Error Creating Snapshot "+ "for Index: %v Slice: %v. Skipped. Error %v", idxInstId, slice.Id(), err) isSnapCreated = false common.CrashOnError(err) continue } snapOpenDur := time.Since(snapOpenStart) logging.Infof("StorageMgr::handleCreateSnapshot Added New Snapshot Index: %v "+ "PartitionId: %v SliceId: %v Crc64: %v (%v) SnapCreateDur %v SnapOpenDur %v", idxInstId, partnId, slice.Id(), tsVbuuid.Crc64, info, snapCreateDur, snapOpenDur) ss := &sliceSnapshot{ id: slice.Id(), snap: newSnapshot, } sliceSnaps[slice.Id()] = ss } else { // Increment reference latestSnapshot.Open() ss := &sliceSnapshot{ id: slice.Id(), snap: latestSnapshot, } sliceSnaps[slice.Id()] = ss logging.Warnf("StorageMgr::handleCreateSnapshot Skipped Creating New Snapshot for Index %v "+ "PartitionId %v SliceId %v. No New Mutations. IsDirty %v", idxInstId, partnId, slice.Id(), slice.IsDirty()) logging.Debugf("StorageMgr::handleCreateSnapshot SnapTs %v FlushTs %v", snapTs, ts) continue } } ps := &partitionSnapshot{ id: partnId, slices: sliceSnaps, } partnSnaps[partnId] = ps } is := &indexSnapshot{ instId: idxInstId, ts: tsVbuuid.Copy(), partns: partnSnaps, } if isSnapCreated { s.updateSnapMapAndNotify(is, idxStats) } else { DestroyIndexSnapshot(is) } s.updateSnapIntervalStat(idxStats) } }(idxInstId, partnMap) } wg.Wait() s.supvRespch <- &MsgMutMgrFlushDone{mType: STORAGE_SNAP_DONE, streamId: streamId, bucket: bucket, ts: tsVbuuid} }