//
// Restore semantic:
// 1) Each index is associated with the <IndexDefnId, IndexerId>. IndexDefnId is unique for each index defnition,
// and IndexerId is unique among the index nodes. Note that IndexDefnId cannot be reused.
// 2) Index defn exists for the given <IndexDefnId, IndexerId> in current repository. No action will be applied during restore.
// 3) Index defn is deleted or missing in current repository. Index Defn restored from backup if bucket exists.
// - Index defn of the same <bucket, name> exists. It will rename the index to <index name>_restore_<seqNo>
// - Bucket does not exist. It will restore an index defn with a non-existent bucket.
//
func (m *requestHandlerContext) handleRestoreIndexMetadataRequest(w http.ResponseWriter, r *http.Request) {
if !doAuth(r, w, m.clusterUrl) {
return
}
image := m.convertIndexMetadataRequest(r)
if image == nil {
send(w, &RestoreResponse{Code: RESP_ERROR, Error: "Unable to process request input"})
return
}
indexerHostMap := make(map[common.IndexerId]string)
current, err := m.getIndexMetadata(m.mgr.getServiceAddrProvider().(*common.ClusterInfoCache), indexerHostMap)
if err != nil {
send(w, &RestoreResponse{Code: RESP_ERROR, Error: "Unable to get the latest index metadata for restore"})
return
}
context := &RestoreContext{idxToRestore: make(map[common.IndexerId][]common.IndexDefn),
idxToResolve: make(map[common.IndexerId][]common.IndexDefn)}
// Figure out what index to restore that has the same IndexDefnId
for _, imeta := range image.Metadata {
found := false
for _, cmeta := range current.Metadata {
if imeta.IndexerId == cmeta.IndexerId {
m.findIndexToRestoreById(&imeta, &cmeta, context)
found = true
}
}
if !found {
logging.Debugf("requestHandler.handleRestoreIndexMetadataRequest(): cannot find matching indexer id %s", imeta.IndexerId)
for _, idefn := range imeta.IndexDefinitions {
logging.Debugf("requestHandler.handleRestoreIndexMetadataRequest(): adding index definition (%v,%v) to to-be-resolve list", idefn.Bucket, idefn.Name)
context.idxToResolve[common.IndexerId(imeta.IndexerId)] =
append(context.idxToResolve[common.IndexerId(imeta.IndexerId)], idefn)
}
}
}
// Figure out what index to restore that has the same bucket and name
for indexerId, idxs := range context.idxToResolve {
for _, idx := range idxs {
m.findIndexToRestoreByName(current, idx, indexerId, context)
}
}
// recreate index
success := m.restoreIndex(current, context, indexerHostMap)
if success {
send(w, &RestoreResponse{Code: RESP_SUCCESS})
return
}
send(w, &RestoreResponse{Code: RESP_ERROR, Error: "Unable to restore metadata"})
}
func (m *requestHandlerContext) restoreIndex(current *ClusterIndexMetadata,
context *RestoreContext, indexerHostMap map[common.IndexerId]string) bool {
indexerCountMap := make(map[common.IndexerId]int)
for _, meta := range current.Metadata {
indexerCountMap[common.IndexerId(meta.IndexerId)] = len(meta.IndexDefinitions)
}
result := true
for indexerId, idxs := range context.idxToRestore {
for _, defn := range idxs {
host, ok := indexerHostMap[indexerId]
if !ok {
indexerId = m.findMinIndexer(indexerCountMap)
host = indexerHostMap[indexerId]
}
logging.Debugf("requestHandler.restoreIndex(): restore index definition (%v,%v) on host %v", defn.Bucket, defn.Name, host)
result = result && m.makeCreateIndexRequest(defn, host)
indexerCountMap[indexerId] = indexerCountMap[indexerId] + 1
}
}
return result
}
func (w *watcher) getIndexerId() c.IndexerId {
w.mutex.Lock()
defer w.mutex.Unlock()
if w.serviceMap == nil {
panic("Index node metadata is not initialized")
}
return c.IndexerId(w.serviceMap.IndexerId)
}
func (m *requestHandlerContext) findIndexToRestoreByName(current *ClusterIndexMetadata,
defn common.IndexDefn, indexerId common.IndexerId, context *RestoreContext) {
logging.Debugf("requestHandler.findIndexToRestoreById(): checking for index definition (%v,%v) in repository by name", defn.Bucket, defn.Name)
for _, meta := range current.Metadata {
for _, ldefn := range meta.IndexDefinitions {
if ldefn.Bucket == defn.Bucket && ldefn.Name == defn.Name {
if topology := m.findTopologyByBucket(meta.IndexTopologies, ldefn.Bucket); topology != nil {
state, _ := topology.GetStatusByDefn(ldefn.DefnId)
if state != common.INDEX_STATE_DELETED && state != common.INDEX_STATE_NIL {
logging.Debugf("requestHandler.findIndexToRestoreByName(): find matching index definition (%v,%v) in repository in state %d", ldefn.Bucket, ldefn.Name, state)
return
}
}
}
}
}
logging.Debugf("requestHandler.findIndexToRestoreByName(): adding index definition (%v,%v) to restore list", defn.Bucket, defn.Name)
context.idxToRestore[common.IndexerId(indexerId)] = append(context.idxToRestore[common.IndexerId(indexerId)], defn)
}
func (o *MetadataProvider) WatchMetadata(indexAdminPort string, callback watcherCallback) c.IndexerId {
o.mutex.Lock()
defer o.mutex.Unlock()
logging.Debugf("MetadataProvider.WatchMetadata(): indexer %v", indexAdminPort)
for _, watcher := range o.watchers {
if watcher.getAdminAddr() == indexAdminPort {
return watcher.getIndexerId()
}
}
// start a watcher to the indexer admin
watcher, readych := o.startWatcher(indexAdminPort)
// wait for indexer to connect
success, _ := watcher.waitForReady(readych, 1000, nil)
if success {
// if successfully connected, retrieve indexerId
success, _ = watcher.notifyReady(indexAdminPort, 0, nil)
if success {
logging.Infof("WatchMetadata(): successfully reach indexer at %v.", indexAdminPort)
// watcher succesfully initialized, add it to MetadataProvider
o.addWatcherNoLock(watcher, c.INDEXER_ID_NIL)
return watcher.getIndexerId()
} else {
// watcher is ready, but no able to read indexerId
readych = nil
}
}
logging.Infof("WatchMetadata(): unable to reach indexer at %v. Retry in background.", indexAdminPort)
// watcher is not connected to indexer or fail to get indexer id,
// create a temporary index id
o.watcherCount = o.watcherCount + 1
tempIndexerId := c.IndexerId(fmt.Sprintf("%v_Indexer_Id_%d", indexAdminPort, o.watcherCount))
killch := make(chan bool, 1)
o.pendings[tempIndexerId] = killch
// retry it in the background. Return a temporary indexerId for book-keeping.
go o.retryHelper(watcher, readych, indexAdminPort, tempIndexerId, killch, callback)
return tempIndexerId
}
func (r *metadataRepo) updateIndexMetadataNoLock(defnId c.IndexDefnId, inst *IndexInstDistribution) {
meta, ok := r.indices[defnId]
if ok {
idxInst := new(InstanceDefn)
idxInst.InstId = c.IndexInstId(inst.InstId)
idxInst.State = c.IndexState(inst.State)
idxInst.Error = inst.Error
idxInst.BuildTime = inst.BuildTime
for _, partition := range inst.Partitions {
for _, slice := range partition.SinglePartition.Slices {
idxInst.IndexerId = c.IndexerId(slice.IndexerId)
break
}
}
meta.Instances = []*InstanceDefn{idxInst}
}
}
func (m *requestHandlerContext) getIndexMetadata(cinfo *common.ClusterInfoCache,
indexerHostMap map[common.IndexerId]string) (*ClusterIndexMetadata, error) {
if err := cinfo.Fetch(); err != nil {
return nil, err
}
// find all nodes that has a index http service
nids := cinfo.GetNodesByServiceType(common.INDEX_HTTP_SERVICE)
clusterMeta := &ClusterIndexMetadata{Metadata: make([]LocalIndexMetadata, len(nids))}
for i, nid := range nids {
addr, err := cinfo.GetServiceAddress(nid, common.INDEX_HTTP_SERVICE)
if err == nil {
resp, err := getWithAuth(addr + "/getLocalIndexMetadata")
if err != nil {
return nil, errors.New(fmt.Sprintf("Fail to retrieve index definition from url %s", addr))
}
localMeta := new(LocalIndexMetadata)
status := convertResponse(resp, localMeta)
if status == RESP_ERROR {
return nil, errors.New(fmt.Sprintf("Fail to retrieve local metadata from url %s.", addr))
}
indexerHostMap[common.IndexerId(localMeta.IndexerId)] = addr
clusterMeta.Metadata[i] = *localMeta
} else {
return nil, errors.New(fmt.Sprintf("Fail to retrieve http endpoint for index node"))
}
}
return clusterMeta, nil
}
func (m *requestHandlerContext) findIndexToRestoreById(image *LocalIndexMetadata,
current *LocalIndexMetadata, context *RestoreContext) {
context.idxToRestore[common.IndexerId(image.IndexerId)] = make([]common.IndexDefn, 0)
context.idxToResolve[common.IndexerId(image.IndexerId)] = make([]common.IndexDefn, 0)
for _, idefn := range image.IndexDefinitions {
match := false
for _, cdefn := range current.IndexDefinitions {
if idefn.DefnId == cdefn.DefnId {
// find index defn in the current repository, check the status
logging.Debugf("requestHandler.findIndexToRestoreById(): find matching index definition (%v,%v) in repository", idefn.Bucket, idefn.Name)
if topology := m.findTopologyByBucket(current.IndexTopologies, idefn.Bucket); topology != nil {
match = true
state, _ := topology.GetStatusByDefn(idefn.DefnId)
logging.Debugf("requestHandler.findIndexToRestoreById(): index definition (%v,%v) in repository in state %d", idefn.Bucket, idefn.Name, state)
if state == common.INDEX_STATE_DELETED || state == common.INDEX_STATE_NIL {
// Index Defn exists it the current repository, but it must be
// 1) Index Instance does not exist
// 2) Index Instance has DELETED state
logging.Debugf("requestHandler.findIndexToRestoreById(): adding index definition (%v,%v) to restore list", idefn.Bucket, idefn.Name)
context.idxToRestore[common.IndexerId(image.IndexerId)] =
append(context.idxToRestore[common.IndexerId(image.IndexerId)], idefn)
}
}
}
}
if !match {
// Index Defn does not exist. Need to find if there is another index with matching <bucket, name>
logging.Debugf("requestHandler.findIndexToRestoreById(): adding index definition (%v,%v) to to-be-resolve list", idefn.Bucket, idefn.Name)
context.idxToResolve[common.IndexerId(image.IndexerId)] =
append(context.idxToResolve[common.IndexerId(image.IndexerId)], idefn)
}
}
}
func (c *MetadataRepo) GetLocalIndexerId() (common.IndexerId, error) {
val, err := c.GetLocalValue("IndexerId")
return common.IndexerId(val), err
}