// // Restore semantic: // 1) Each index is associated with the <IndexDefnId, IndexerId>. IndexDefnId is unique for each index defnition, // and IndexerId is unique among the index nodes. Note that IndexDefnId cannot be reused. // 2) Index defn exists for the given <IndexDefnId, IndexerId> in current repository. No action will be applied during restore. // 3) Index defn is deleted or missing in current repository. Index Defn restored from backup if bucket exists. // - Index defn of the same <bucket, name> exists. It will rename the index to <index name>_restore_<seqNo> // - Bucket does not exist. It will restore an index defn with a non-existent bucket. // func (m *requestHandlerContext) handleRestoreIndexMetadataRequest(w http.ResponseWriter, r *http.Request) { if !doAuth(r, w, m.clusterUrl) { return } image := m.convertIndexMetadataRequest(r) if image == nil { send(w, &RestoreResponse{Code: RESP_ERROR, Error: "Unable to process request input"}) return } indexerHostMap := make(map[common.IndexerId]string) current, err := m.getIndexMetadata(m.mgr.getServiceAddrProvider().(*common.ClusterInfoCache), indexerHostMap) if err != nil { send(w, &RestoreResponse{Code: RESP_ERROR, Error: "Unable to get the latest index metadata for restore"}) return } context := &RestoreContext{idxToRestore: make(map[common.IndexerId][]common.IndexDefn), idxToResolve: make(map[common.IndexerId][]common.IndexDefn)} // Figure out what index to restore that has the same IndexDefnId for _, imeta := range image.Metadata { found := false for _, cmeta := range current.Metadata { if imeta.IndexerId == cmeta.IndexerId { m.findIndexToRestoreById(&imeta, &cmeta, context) found = true } } if !found { logging.Debugf("requestHandler.handleRestoreIndexMetadataRequest(): cannot find matching indexer id %s", imeta.IndexerId) for _, idefn := range imeta.IndexDefinitions { logging.Debugf("requestHandler.handleRestoreIndexMetadataRequest(): adding index definition (%v,%v) to to-be-resolve list", idefn.Bucket, idefn.Name) context.idxToResolve[common.IndexerId(imeta.IndexerId)] = append(context.idxToResolve[common.IndexerId(imeta.IndexerId)], idefn) } } } // Figure out what index to restore that has the same bucket and name for indexerId, idxs := range context.idxToResolve { for _, idx := range idxs { m.findIndexToRestoreByName(current, idx, indexerId, context) } } // recreate index success := m.restoreIndex(current, context, indexerHostMap) if success { send(w, &RestoreResponse{Code: RESP_SUCCESS}) return } send(w, &RestoreResponse{Code: RESP_ERROR, Error: "Unable to restore metadata"}) }
func (m *requestHandlerContext) restoreIndex(current *ClusterIndexMetadata, context *RestoreContext, indexerHostMap map[common.IndexerId]string) bool { indexerCountMap := make(map[common.IndexerId]int) for _, meta := range current.Metadata { indexerCountMap[common.IndexerId(meta.IndexerId)] = len(meta.IndexDefinitions) } result := true for indexerId, idxs := range context.idxToRestore { for _, defn := range idxs { host, ok := indexerHostMap[indexerId] if !ok { indexerId = m.findMinIndexer(indexerCountMap) host = indexerHostMap[indexerId] } logging.Debugf("requestHandler.restoreIndex(): restore index definition (%v,%v) on host %v", defn.Bucket, defn.Name, host) result = result && m.makeCreateIndexRequest(defn, host) indexerCountMap[indexerId] = indexerCountMap[indexerId] + 1 } } return result }
func (w *watcher) getIndexerId() c.IndexerId { w.mutex.Lock() defer w.mutex.Unlock() if w.serviceMap == nil { panic("Index node metadata is not initialized") } return c.IndexerId(w.serviceMap.IndexerId) }
func (m *requestHandlerContext) findIndexToRestoreByName(current *ClusterIndexMetadata, defn common.IndexDefn, indexerId common.IndexerId, context *RestoreContext) { logging.Debugf("requestHandler.findIndexToRestoreById(): checking for index definition (%v,%v) in repository by name", defn.Bucket, defn.Name) for _, meta := range current.Metadata { for _, ldefn := range meta.IndexDefinitions { if ldefn.Bucket == defn.Bucket && ldefn.Name == defn.Name { if topology := m.findTopologyByBucket(meta.IndexTopologies, ldefn.Bucket); topology != nil { state, _ := topology.GetStatusByDefn(ldefn.DefnId) if state != common.INDEX_STATE_DELETED && state != common.INDEX_STATE_NIL { logging.Debugf("requestHandler.findIndexToRestoreByName(): find matching index definition (%v,%v) in repository in state %d", ldefn.Bucket, ldefn.Name, state) return } } } } } logging.Debugf("requestHandler.findIndexToRestoreByName(): adding index definition (%v,%v) to restore list", defn.Bucket, defn.Name) context.idxToRestore[common.IndexerId(indexerId)] = append(context.idxToRestore[common.IndexerId(indexerId)], defn) }
func (o *MetadataProvider) WatchMetadata(indexAdminPort string, callback watcherCallback) c.IndexerId { o.mutex.Lock() defer o.mutex.Unlock() logging.Debugf("MetadataProvider.WatchMetadata(): indexer %v", indexAdminPort) for _, watcher := range o.watchers { if watcher.getAdminAddr() == indexAdminPort { return watcher.getIndexerId() } } // start a watcher to the indexer admin watcher, readych := o.startWatcher(indexAdminPort) // wait for indexer to connect success, _ := watcher.waitForReady(readych, 1000, nil) if success { // if successfully connected, retrieve indexerId success, _ = watcher.notifyReady(indexAdminPort, 0, nil) if success { logging.Infof("WatchMetadata(): successfully reach indexer at %v.", indexAdminPort) // watcher succesfully initialized, add it to MetadataProvider o.addWatcherNoLock(watcher, c.INDEXER_ID_NIL) return watcher.getIndexerId() } else { // watcher is ready, but no able to read indexerId readych = nil } } logging.Infof("WatchMetadata(): unable to reach indexer at %v. Retry in background.", indexAdminPort) // watcher is not connected to indexer or fail to get indexer id, // create a temporary index id o.watcherCount = o.watcherCount + 1 tempIndexerId := c.IndexerId(fmt.Sprintf("%v_Indexer_Id_%d", indexAdminPort, o.watcherCount)) killch := make(chan bool, 1) o.pendings[tempIndexerId] = killch // retry it in the background. Return a temporary indexerId for book-keeping. go o.retryHelper(watcher, readych, indexAdminPort, tempIndexerId, killch, callback) return tempIndexerId }
func (r *metadataRepo) updateIndexMetadataNoLock(defnId c.IndexDefnId, inst *IndexInstDistribution) { meta, ok := r.indices[defnId] if ok { idxInst := new(InstanceDefn) idxInst.InstId = c.IndexInstId(inst.InstId) idxInst.State = c.IndexState(inst.State) idxInst.Error = inst.Error idxInst.BuildTime = inst.BuildTime for _, partition := range inst.Partitions { for _, slice := range partition.SinglePartition.Slices { idxInst.IndexerId = c.IndexerId(slice.IndexerId) break } } meta.Instances = []*InstanceDefn{idxInst} } }
func (m *requestHandlerContext) getIndexMetadata(cinfo *common.ClusterInfoCache, indexerHostMap map[common.IndexerId]string) (*ClusterIndexMetadata, error) { if err := cinfo.Fetch(); err != nil { return nil, err } // find all nodes that has a index http service nids := cinfo.GetNodesByServiceType(common.INDEX_HTTP_SERVICE) clusterMeta := &ClusterIndexMetadata{Metadata: make([]LocalIndexMetadata, len(nids))} for i, nid := range nids { addr, err := cinfo.GetServiceAddress(nid, common.INDEX_HTTP_SERVICE) if err == nil { resp, err := getWithAuth(addr + "/getLocalIndexMetadata") if err != nil { return nil, errors.New(fmt.Sprintf("Fail to retrieve index definition from url %s", addr)) } localMeta := new(LocalIndexMetadata) status := convertResponse(resp, localMeta) if status == RESP_ERROR { return nil, errors.New(fmt.Sprintf("Fail to retrieve local metadata from url %s.", addr)) } indexerHostMap[common.IndexerId(localMeta.IndexerId)] = addr clusterMeta.Metadata[i] = *localMeta } else { return nil, errors.New(fmt.Sprintf("Fail to retrieve http endpoint for index node")) } } return clusterMeta, nil }
func (m *requestHandlerContext) findIndexToRestoreById(image *LocalIndexMetadata, current *LocalIndexMetadata, context *RestoreContext) { context.idxToRestore[common.IndexerId(image.IndexerId)] = make([]common.IndexDefn, 0) context.idxToResolve[common.IndexerId(image.IndexerId)] = make([]common.IndexDefn, 0) for _, idefn := range image.IndexDefinitions { match := false for _, cdefn := range current.IndexDefinitions { if idefn.DefnId == cdefn.DefnId { // find index defn in the current repository, check the status logging.Debugf("requestHandler.findIndexToRestoreById(): find matching index definition (%v,%v) in repository", idefn.Bucket, idefn.Name) if topology := m.findTopologyByBucket(current.IndexTopologies, idefn.Bucket); topology != nil { match = true state, _ := topology.GetStatusByDefn(idefn.DefnId) logging.Debugf("requestHandler.findIndexToRestoreById(): index definition (%v,%v) in repository in state %d", idefn.Bucket, idefn.Name, state) if state == common.INDEX_STATE_DELETED || state == common.INDEX_STATE_NIL { // Index Defn exists it the current repository, but it must be // 1) Index Instance does not exist // 2) Index Instance has DELETED state logging.Debugf("requestHandler.findIndexToRestoreById(): adding index definition (%v,%v) to restore list", idefn.Bucket, idefn.Name) context.idxToRestore[common.IndexerId(image.IndexerId)] = append(context.idxToRestore[common.IndexerId(image.IndexerId)], idefn) } } } } if !match { // Index Defn does not exist. Need to find if there is another index with matching <bucket, name> logging.Debugf("requestHandler.findIndexToRestoreById(): adding index definition (%v,%v) to to-be-resolve list", idefn.Bucket, idefn.Name) context.idxToResolve[common.IndexerId(image.IndexerId)] = append(context.idxToResolve[common.IndexerId(image.IndexerId)], idefn) } } }
func (c *MetadataRepo) GetLocalIndexerId() (common.IndexerId, error) { val, err := c.GetLocalValue("IndexerId") return common.IndexerId(val), err }