Exemple #1
0
//
// Serialize topology into a protobuf message format
//
func convertTopologyToIndexInstProtoMsg(mgr *IndexManager,
	topology *IndexTopology, port string) ([]*protobuf.Instance, error) {

	var result []*protobuf.Instance = nil

	for _, defnRef := range topology.Definitions {

		// look up the index definition from dictionary
		defn, err := mgr.GetIndexDefnById(common.IndexDefnId(defnRef.DefnId))
		if err != nil {
			logging.Debugf("convertTopologyToIndexInstProtoMsg(): Cannot find definition id = %v. Skip", defnRef.DefnId)
			continue
		}

		// Convert definition to protobuf msg
		defn_proto := convertIndexDefnToProtoMsg(defn)

		// iterate through the index inst for this defnition
		// TODO: Remove CREATED state from the if-stmt
		for _, inst := range defnRef.Instances {
			if common.IndexState(inst.State) == common.INDEX_STATE_READY ||
				common.IndexState(inst.State) == common.INDEX_STATE_INITIAL ||
				common.IndexState(inst.State) == common.INDEX_STATE_CREATED ||
				common.IndexState(inst.State) == common.INDEX_STATE_ACTIVE {
				result = append(result, convertIndexInstToProtoMsg(&inst, defn_proto, port))
			}
		}
	}

	return result, nil
}
Exemple #2
0
//
// Get all deleted index instance Id's
//
func GetAllDeletedIndexInstancesId(mgr *IndexManager, buckets []string) ([]uint64, error) {

	var result []uint64 = nil

	// Get the topology from the dictionary
	for _, bucket := range buckets {
		topology, err := mgr.GetTopologyByBucket(bucket)
		if err != nil {
			// TODO: Determine if it is a real error, or just topology does not exist in dictionary
			// If there is an error, return an empty array.  This assume that the topology does not exist.
			logging.Debugf("GetAllDeletedIndexInstances(): Cannot find topology for bucket %s.  Skip.", bucket)
			continue
		}

		for _, defnRef := range topology.Definitions {
			for _, inst := range defnRef.Instances {
				if common.IndexState(inst.State) == common.INDEX_STATE_DELETED {
					result = append(result, inst.InstId)
				}
			}
		}
	}

	return result, nil
}
Exemple #3
0
func (m *LifecycleMgr) UpdateIndexInstance(bucket string, defnId common.IndexDefnId, state common.IndexState,
	streamId common.StreamId, errStr string, buildTime []uint64) error {

	topology, err := m.repo.GetTopologyByBucket(bucket)
	if err != nil {
		logging.Errorf("LifecycleMgr.handleTopologyChange() : index instance update fails. Reason = %v", err)
		return err
	}

	changed := false
	if state != common.INDEX_STATE_NIL {
		changed = topology.UpdateStateForIndexInstByDefn(common.IndexDefnId(defnId), common.IndexState(state)) || changed
	}

	if streamId != common.NIL_STREAM {
		changed = topology.UpdateStreamForIndexInstByDefn(common.IndexDefnId(defnId), common.StreamId(streamId)) || changed
	}

	changed = topology.SetErrorForIndexInstByDefn(common.IndexDefnId(defnId), errStr) || changed

	if changed {
		if err := m.repo.SetTopologyByBucket(bucket, topology); err != nil {
			logging.Errorf("LifecycleMgr.handleTopologyChange() : index instance update fails. Reason = %v", err)
			return err
		}
	}

	return nil
}
Exemple #4
0
//
// Update Index Status on instance
//
func (t *IndexTopology) GetStatusByDefn(defnId common.IndexDefnId) (common.IndexState, string) {

	for i, _ := range t.Definitions {
		if t.Definitions[i].DefnId == uint64(defnId) {
			return common.IndexState(t.Definitions[i].Instances[0].State), t.Definitions[i].Instances[0].Error
		}
	}
	return common.INDEX_STATE_NIL, ""
}
Exemple #5
0
func (m *LifecycleMgr) handleTopologyChange(content []byte) error {

	change := new(topologyChange)
	if err := json.Unmarshal(content, change); err != nil {
		return err
	}

	return m.UpdateIndexInstance(change.Bucket, common.IndexDefnId(change.DefnId), common.IndexState(change.State),
		common.StreamId(change.StreamId), change.Error, change.BuildTime)
}
func (c *clustMgrAgent) handleGetGlobalTopology(cmd Message) {

	logging.Debugf("ClustMgr:handleGetGlobalTopology %v", cmd)

	//get the latest topology from manager
	metaIter, err := c.mgr.NewIndexDefnIterator()
	if err != nil {
		common.CrashOnError(err)
	}
	defer metaIter.Close()

	indexInstMap := make(common.IndexInstMap)

	for _, defn, err := metaIter.Next(); err == nil; _, defn, err = metaIter.Next() {

		var idxDefn common.IndexDefn
		idxDefn = *defn

		t, e := c.mgr.GetTopologyByBucket(idxDefn.Bucket)
		if e != nil {
			common.CrashOnError(e)
		}

		inst := t.GetIndexInstByDefn(idxDefn.DefnId)

		if inst == nil {
			logging.Warnf("ClustMgr:handleGetGlobalTopology Index Instance Not "+
				"Found For Index Definition %v. Ignored.", idxDefn)
			continue
		}

		//for indexer, Ready state doesn't matter. Till index build,
		//the index stays in Created state.
		var state common.IndexState
		instState := common.IndexState(inst.State)
		if instState == common.INDEX_STATE_READY {
			state = common.INDEX_STATE_CREATED
		} else {
			state = instState
		}

		idxInst := common.IndexInst{InstId: common.IndexInstId(inst.InstId),
			Defn:   idxDefn,
			State:  state,
			Stream: common.StreamId(inst.StreamId),
		}

		indexInstMap[idxInst.InstId] = idxInst

	}

	c.supvCmdch <- &MsgClustMgrTopology{indexInstMap: indexInstMap}
}
Exemple #7
0
//
// This function finds the difference between two index definitons (in topology).
// This takes into account the state transition of the index instances within
// the index definitions (e.g. from CREATED to INITIAL).  If there is no matching
// index instance in the old index definition,  then fromState is ignored, and
// index instances (from new index definition) will be added as long as it is in toState.
//
func (s *StreamManager) addInstancesToChangeList(
	oldDefn *IndexDefnDistribution,
	newDefn *IndexDefnDistribution,
	fromStates []common.IndexState,
	toStates []common.IndexState) []*changeRecord {

	var changes []*changeRecord = nil

	logging.Debugf("StreamManager.addInstancesToChangeList(): defn '%v'", newDefn.Name)

	for _, newInst := range newDefn.Instances {
		add := s.inState(common.IndexState(newInst.State), toStates)
		logging.Debugf("StreamManager.addInstancesToChangeList(): found new instance '%v' in state %v",
			newInst.InstId, newInst.State)

		if oldDefn != nil {
			for _, oldInst := range oldDefn.Instances {
				if newInst.InstId == oldInst.InstId {
					if s.inState(common.IndexState(oldInst.State), fromStates) {
						logging.Debugf("StreamManager.addInstancesToChangeList(): found old instance '%v' in state %v",
							oldInst.InstId, oldInst.State)
					}
					add = add && s.inState(common.IndexState(oldInst.State), fromStates) &&
						oldInst.State != newInst.State
				}
			}
		}

		if add {
			logging.Debugf("StreamManager.addInstancesToChangeList(): adding inst '%v' to change list.", newInst.InstId)
			change := &changeRecord{definition: newDefn, instance: &newInst}
			changes = append(changes, change)
		}
	}

	return changes
}
Exemple #8
0
func (r *metadataRepo) updateIndexMetadataNoLock(defnId c.IndexDefnId, inst *IndexInstDistribution) {

	meta, ok := r.indices[defnId]
	if ok {
		idxInst := new(InstanceDefn)
		idxInst.InstId = c.IndexInstId(inst.InstId)
		idxInst.State = c.IndexState(inst.State)
		idxInst.Error = inst.Error
		idxInst.BuildTime = inst.BuildTime

		for _, partition := range inst.Partitions {
			for _, slice := range partition.SinglePartition.Slices {
				idxInst.IndexerId = c.IndexerId(slice.IndexerId)
				break
			}
		}
		meta.Instances = []*InstanceDefn{idxInst}
	}
}
Exemple #9
0
func TestCoordinator(t *testing.T) {

	logging.SetLogLevel(logging.Trace)

	logging.Infof("Start TestCoordinator *********************************************************")

	cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/)
	cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"})
	os.MkdirAll("./data/", os.ModePerm)

	/*
		var requestAddr = "localhost:9885"
		var leaderAddr = "localhost:9884"
	*/
	var config = "./config.json"
	manager.USE_MASTER_REPO = true
	defer func() { manager.USE_MASTER_REPO = false }()

	mgr, err := manager.NewIndexManagerInternal("localhost:9886", "localhost:"+manager.COORD_MAINT_STREAM_PORT, nil, cfg)
	if err != nil {
		t.Fatal(err)
	}
	defer mgr.Close()
	mgr.StartCoordinator(config)
	time.Sleep(time.Duration(1000) * time.Millisecond)

	cleanup(mgr, t)
	time.Sleep(time.Duration(1000) * time.Millisecond)

	// Add a new index definition : 200
	idxDefn := &common.IndexDefn{
		DefnId:          common.IndexDefnId(200),
		Name:            "coordinator_test",
		Using:           common.ForestDB,
		Bucket:          "Default",
		IsPrimary:       false,
		SecExprs:        []string{"Testing"},
		ExprType:        common.N1QL,
		PartitionScheme: common.HASH,
		PartitionKey:    "Testing"}

	err = mgr.HandleCreateIndexDDL(idxDefn)
	if err != nil {
		t.Fatal(err)
	}
	time.Sleep(time.Duration(1000) * time.Millisecond)

	idxDefn, err = mgr.GetIndexDefnById(common.IndexDefnId(200))
	if err != nil {
		t.Fatal(err)
	}

	if idxDefn == nil {
		t.Fatal("Cannot find index definition")
	}

	topology, err := mgr.GetTopologyByBucket("Default")
	if err != nil {
		t.Fatal(err)
	}
	content, err := manager.MarshallIndexTopology(topology)
	if err != nil {
		t.Fatal(err)
	}
	logging.Infof("Topology after index creation : %s", string(content))

	inst := topology.GetIndexInstByDefn(common.IndexDefnId(200))
	if inst == nil || common.IndexState(inst.State) != common.INDEX_STATE_READY {
		t.Fatal("Index Inst not found for index defn 200 or inst state is not in READY")
	}

	cleanup(mgr, t)
	mgr.CleanupTopology()
	mgr.CleanupStabilityTimestamp()
	time.Sleep(time.Duration(1000) * time.Millisecond)

	logging.Infof("Done TestCoordinator. Tearing down *********************************************************")
	mgr.Close()
	time.Sleep(time.Duration(1000) * time.Millisecond)
}