Ejemplo n.º 1
0
func getTopicChan(command string, params []string) (string, string, string, error) {
	if len(params) <= 1 {
		return "", "", "", protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("%s insufficient number of params", command))
	}

	topicName := params[0]
	partitionID := params[1]
	var channelName string
	if len(params) >= 3 {
		channelName = params[2]
	}

	if !protocol.IsValidTopicName(topicName) {
		return "", "", "", protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("%s topic name '%s' is not valid", command, topicName))
	}

	if channelName != "" && !protocol.IsValidChannelName(channelName) {
		return "", "", "", protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL", fmt.Sprintf("%s channel name '%s' is not valid", command, channelName))
	}

	if _, err := GetValidPartitionID(partitionID); err != nil {
		return "", "", "", protocol.NewFatalClientErr(nil, "E_BAD_PARTITIONID", fmt.Sprintf("%s partition id '%s' is not valid", command, partitionID))
	}
	return topicName, channelName, partitionID, nil
}
Ejemplo n.º 2
0
func (s *httpServer) tombstoneNodeForTopicHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
	var messages []string

	node := ps.ByName("node")

	var body struct {
		Topic string `json:"topic"`
	}
	err := json.NewDecoder(req.Body).Decode(&body)
	if err != nil {
		return nil, http_api.Err{400, "INVALID_BODY"}
	}

	if !protocol.IsValidTopicName(body.Topic) {
		return nil, http_api.Err{400, "INVALID_TOPIC"}
	}

	err = s.ci.TombstoneNodeForTopic(body.Topic, node,
		s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
	if err != nil {
		pe, ok := err.(clusterinfo.PartialErr)
		if !ok {
			s.ctx.nsqadmin.logf("ERROR: failed to tombstone node for topic - %s", err)
			return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)}
		}
		s.ctx.nsqadmin.logf("WARNING: %s", err)
		messages = append(messages, pe.Error())
	}

	s.notifyAdminAction("tombstone_topic_producer", body.Topic, "", node, req)

	return struct {
		Message string `json:"message"`
	}{maybeWarnMsg(messages)}, nil
}
Ejemplo n.º 3
0
func (self *NsqLookupCoordinator) CreateTopic(topic string, meta TopicMetaInfo) error {
	if self.leaderNode.GetID() != self.myNode.GetID() {
		coordLog.Infof("not leader while create topic")
		return ErrNotNsqLookupLeader
	}

	if !protocol.IsValidTopicName(topic) {
		return errors.New("invalid topic name")
	}

	// TODO: handle default load factor
	if meta.PartitionNum >= MAX_PARTITION_NUM {
		return errors.New("max partition allowed exceed")
	}

	currentNodes := self.getCurrentNodes()
	if len(currentNodes) < meta.Replica || len(currentNodes) < meta.PartitionNum {
		coordLog.Infof("nodes %v is less than replica or partition %v", len(currentNodes), meta)
		return ErrNodeUnavailable.ToErrorType()
	}
	if len(currentNodes) < meta.Replica*meta.PartitionNum {
		coordLog.Infof("nodes is less than replica*partition")
		return ErrNodeUnavailable.ToErrorType()
	}

	self.joinStateMutex.Lock()
	state, ok := self.joinISRState[topic]
	if !ok {
		state = &JoinISRState{}
		self.joinISRState[topic] = state
	}
	self.joinStateMutex.Unlock()
	state.Lock()
	defer state.Unlock()
	if state.waitingJoin {
		coordLog.Warningf("topic state is not ready:%v, %v ", topic, state)
		return ErrWaitingJoinISR.ToErrorType()
	}
	if meta.SyncEvery > MAX_SYNC_EVERY {
		coordLog.Infof("topic %v sync every with too large %v, set to max", topic, meta)
		meta.SyncEvery = MAX_SYNC_EVERY
	}

	if ok, _ := self.leadership.IsExistTopic(topic); !ok {
		meta.MagicCode = time.Now().UnixNano()
		err := self.leadership.CreateTopic(topic, &meta)
		if err != nil {
			coordLog.Infof("create topic key %v failed :%v", topic, err)
			return err
		}
	} else {
		coordLog.Warningf("topic already exist :%v ", topic)
		return ErrAlreadyExist
	}
	coordLog.Infof("create topic: %v, with meta: %v", topic, meta)

	return self.checkAndUpdateTopicPartitions(currentNodes, topic, meta)
}
Ejemplo n.º 4
0
func (s *httpServer) createTopicChannelHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
	var messages []string

	var body struct {
		Topic         string `json:"topic"`
		PartitionNum  string `json:"partition_num"`
		Replicator    string `json:"replicator"`
		RetentionDays string `json:"retention_days"`
		SyncDisk      string `json:"syncdisk"`
	}
	err := json.NewDecoder(req.Body).Decode(&body)
	if err != nil {
		return nil, http_api.Err{400, err.Error()}
	}

	if !protocol.IsValidTopicName(body.Topic) {
		return nil, http_api.Err{400, "INVALID_TOPIC"}
	}

	if body.PartitionNum == "" {
		return nil, http_api.Err{400, "INVALID_TOPIC_PARTITION_NUM"}
	}
	pnum, err := strconv.Atoi(body.PartitionNum)
	if err != nil {
		return nil, http_api.Err{400, err.Error()}
	}

	if body.Replicator == "" {
		return nil, http_api.Err{400, "INVALID_TOPIC_REPLICATOR"}
	}
	replica, err := strconv.Atoi(body.Replicator)
	if err != nil {
		return nil, http_api.Err{400, err.Error()}
	}

	if body.SyncDisk == "" {
		body.SyncDisk = "2000"
	}
	syncDisk, _ := strconv.Atoi(body.SyncDisk)
	err = s.ci.CreateTopic(body.Topic, pnum, replica,
		syncDisk, body.RetentionDays,
		s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses)
	if err != nil {
		pe, ok := err.(clusterinfo.PartialErr)
		if !ok {
			s.ctx.nsqadmin.logf("ERROR: failed to create topic/channel - %s", err)
			return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)}
		}
		s.ctx.nsqadmin.logf("WARNING: %s", err)
		messages = append(messages, pe.Error())
	}

	s.notifyAdminAction("create_topic", body.Topic, "", "", req)

	return struct {
		Message string `json:"message"`
	}{maybeWarnMsg(messages)}, nil
}
Ejemplo n.º 5
0
func (self *NsqLookupCoordinator) ExpandTopicPartition(topic string, newPartitionNum int) error {
	if self.leaderNode.GetID() != self.myNode.GetID() {
		coordLog.Infof("not leader while create topic")
		return ErrNotNsqLookupLeader
	}

	if !protocol.IsValidTopicName(topic) {
		return errors.New("invalid topic name")
	}

	if newPartitionNum >= MAX_PARTITION_NUM {
		return errors.New("max partition allowed exceed")
	}

	coordLog.Infof("expand topic %v partition number to %v", topic, newPartitionNum)
	if !self.IsClusterStable() {
		return ErrClusterUnstable
	}
	self.joinStateMutex.Lock()
	state, ok := self.joinISRState[topic]
	if !ok {
		state = &JoinISRState{}
		self.joinISRState[topic] = state
	}
	self.joinStateMutex.Unlock()
	state.Lock()
	defer state.Unlock()
	if state.waitingJoin {
		coordLog.Warningf("topic state is not ready:%v, %v ", topic, state)
		return ErrWaitingJoinISR.ToErrorType()
	}
	var meta TopicMetaInfo
	if ok, _ := self.leadership.IsExistTopic(topic); !ok {
		coordLog.Infof("topic not exist %v :%v", topic)
		return ErrTopicNotCreated
	} else {
		oldMeta, oldGen, err := self.leadership.GetTopicMetaInfo(topic)
		if err != nil {
			coordLog.Infof("get topic key %v failed :%v", topic, err)
			return err
		}
		meta = oldMeta
		if newPartitionNum < meta.PartitionNum {
			return errors.New("the partition number can not be reduced")
		}
		currentNodes := self.getCurrentNodes()
		meta.PartitionNum = newPartitionNum
		err = self.updateTopicMeta(currentNodes, topic, meta, oldGen)
		if err != nil {
			coordLog.Infof("update topic %v meta failed :%v", topic, err)
			return err
		}
		return self.checkAndUpdateTopicPartitions(currentNodes, topic, meta)
	}
}
Ejemplo n.º 6
0
func GetTopicArg(rp getter) (string, error) {
	topicName := rp.Get("topic")
	if topicName == "" {
		return "", errors.New("MISSING_ARG_TOPIC")
	}

	if !protocol.IsValidTopicName(topicName) {
		return "", errors.New("INVALID_ARG_TOPIC")
	}

	return topicName, nil
}
Ejemplo n.º 7
0
func (p *protocolV2) preparePub(client *nsqd.ClientV2, params [][]byte, maxBody int64) (int32, *nsqd.Topic, error) {
	var err error

	if len(params) < 2 {
		return 0, nil, protocol.NewFatalClientErr(nil, E_INVALID, "insufficient number of parameters")
	}

	topicName := string(params[1])
	if !protocol.IsValidTopicName(topicName) {
		return 0, nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC",
			fmt.Sprintf("topic name %q is not valid", topicName))
	}
	partition := -1
	if len(params) == 3 {
		partition, err = strconv.Atoi(string(params[2]))
		if err != nil {
			return 0, nil, protocol.NewFatalClientErr(nil, "E_BAD_PARTITION",
				fmt.Sprintf("topic partition is not valid: %v", err))
		}
	}

	if partition == -1 {
		partition = p.ctx.getDefaultPartition(topicName)
	}

	bodyLen, err := readLen(client.Reader, client.LenSlice)
	if err != nil {
		return 0, nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "failed to read body size")
	}

	if bodyLen <= 0 {
		return bodyLen, nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY",
			fmt.Sprintf("invalid body size %d", bodyLen))
	}

	if int64(bodyLen) > maxBody {
		return bodyLen, nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY",
			fmt.Sprintf("body too big %d > %d", bodyLen, maxBody))
	}

	topic, err := p.ctx.getExistingTopic(topicName, partition)
	if err != nil {
		nsqd.NsqLogger().Logf("not existing topic: %v-%v, err:%v", topicName, partition, err.Error())
		return bodyLen, nil, protocol.NewFatalClientErr(nil, E_TOPIC_NOT_EXIST, "")
	}

	if err := p.CheckAuth(client, "PUB", topicName, ""); err != nil {
		return bodyLen, nil, err
	}
	// mpub
	return bodyLen, topic, nil
}
Ejemplo n.º 8
0
func getTopicChanFromOld(command string, params []string) (string, string, error) {
	if len(params) <= 0 {
		return "", "", protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("%s insufficient number of params", command))
	}

	topicName := params[0]
	var channelName string
	if len(params) >= 2 {
		channelName = params[1]
	}

	if !protocol.IsValidTopicName(topicName) {
		return "", "", protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("%s topic name '%s' is not valid", command, topicName))
	}

	if channelName != "" && !protocol.IsValidChannelName(channelName) {
		return "", "", protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL", fmt.Sprintf("%s channel name '%s' is not valid", command, channelName))
	}
	return topicName, channelName, nil
}
Ejemplo n.º 9
0
func (p *protocolV2) internalCreateTopic(client *nsqd.ClientV2, params [][]byte) ([]byte, error) {
	var err error

	if len(params) < 3 {
		return nil, protocol.NewFatalClientErr(nil, E_INVALID, "CREATE_TOPIC insufficient number of parameters")
	}

	topicName := string(params[1])
	if !protocol.IsValidTopicName(topicName) {
		return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC",
			fmt.Sprintf("topic name %q is not valid", topicName))
	}

	partition, err := strconv.Atoi(string(params[2]))
	if err != nil {
		return nil, protocol.NewFatalClientErr(nil, "E_BAD_PARTITION",
			fmt.Sprintf("topic partition is not valid: %v", err))
	}
	if err = p.CheckAuth(client, "CREATE_TOPIC", topicName, ""); err != nil {
		return nil, err
	}

	if partition < 0 {
		return nil, protocol.NewFatalClientErr(nil, "E_BAD_PARTITION", "partition should not less than 0")
	}

	if p.ctx.nsqdCoord != nil {
		return nil, protocol.NewClientErr(err, "E_CREATE_TOPIC_FAILED",
			fmt.Sprintf("CREATE_TOPIC is not allowed here while cluster feature enabled."))
	}

	topic := p.ctx.getTopic(topicName, partition)
	if topic == nil {
		return nil, protocol.NewClientErr(err, "E_CREATE_TOPIC_FAILED",
			fmt.Sprintf("CREATE_TOPIC %v failed", topicName))
	}
	return okBytes, nil
}
Ejemplo n.º 10
0
func (n *NSQD) LoadMetadata(disabled int32) {
	atomic.StoreInt32(&n.isLoading, 1)
	defer atomic.StoreInt32(&n.isLoading, 0)
	fn := fmt.Sprintf(path.Join(n.GetOpts().DataPath, "nsqd.%d.dat"), n.GetOpts().ID)
	data, err := ioutil.ReadFile(fn)
	if err != nil {
		if !os.IsNotExist(err) {
			nsqLog.LogErrorf("failed to read channel metadata from %s - %s", fn, err)
		}
		return
	}

	js, err := simplejson.NewJson(data)
	if err != nil {
		nsqLog.LogErrorf("failed to parse metadata - %s", err)
		return
	}

	topics, err := js.Get("topics").Array()
	if err != nil {
		nsqLog.LogErrorf("failed to parse metadata - %s", err)
		return
	}

	for ti := range topics {
		topicJs := js.Get("topics").GetIndex(ti)

		topicName, err := topicJs.Get("name").String()
		if err != nil {
			nsqLog.LogErrorf("failed to parse metadata - %s", err)
			return
		}
		if !protocol.IsValidTopicName(topicName) {
			nsqLog.LogWarningf("skipping creation of invalid topic %s", topicName)
			continue
		}
		part, err := topicJs.Get("partition").Int()
		if err != nil {
			nsqLog.LogErrorf("failed to parse metadata - %s", err)
			return
		}
		topic := n.internalGetTopic(topicName, part, disabled)

		// old meta should also be loaded
		channels, err := topicJs.Get("channels").Array()
		if err != nil {
			nsqLog.LogErrorf("failed to parse metadata - %s", err)
			return
		}

		for ci := range channels {
			channelJs := topicJs.Get("channels").GetIndex(ci)

			channelName, err := channelJs.Get("name").String()
			if err != nil {
				nsqLog.LogErrorf("failed to parse metadata - %s", err)
				return
			}
			if !protocol.IsValidChannelName(channelName) {
				nsqLog.LogWarningf("skipping creation of invalid channel %s", channelName)
				continue
			}
			channel := topic.GetChannel(channelName)

			paused, _ := channelJs.Get("paused").Bool()
			if paused {
				channel.Pause()
			}
		}
		// we load channels from the new meta file
		topic.LoadChannelMeta()
	}
}
Ejemplo n.º 11
0
func main() {
	var selectedMode int

	cCfg := nsq.NewConfig()
	pCfg := nsq.NewConfig()

	// TODO: remove, deprecated
	flag.Var(&nsq.ConfigFlag{cCfg}, "reader-opt", "(deprecated) use --consumer-opt")
	flag.Var(&nsq.ConfigFlag{cCfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, see http://godoc.org/github.com/absolute8511/go-nsq#Config)")
	flag.Var(&nsq.ConfigFlag{pCfg}, "producer-opt", "option to passthrough to nsq.Producer (may be given multiple times, see http://godoc.org/github.com/absolute8511/go-nsq#Config)")

	flag.Parse()

	if *showVersion {
		fmt.Printf("nsq_to_nsq v%s\n", version.Binary)
		return
	}

	if *topic == "" || *channel == "" {
		log.Fatal("--topic and --channel are required")
	}

	if *destTopic == "" {
		*destTopic = *topic
	}

	if !protocol.IsValidTopicName(*topic) {
		log.Fatal("--topic is invalid")
	}

	if !protocol.IsValidTopicName(*destTopic) {
		log.Fatal("--destination-topic is invalid")
	}

	if !protocol.IsValidChannelName(*channel) {
		log.Fatal("--channel is invalid")
	}

	if len(lookupdHTTPAddrs) == 0 {
		log.Fatal("--lookupd-http-address required")
	}

	if len(destNsqdTCPAddrs) == 0 {
		log.Fatal("--destination-nsqd-tcp-address required")
	}

	switch *mode {
	case "round-robin":
		selectedMode = ModeRoundRobin
	case "hostpool", "epsilon-greedy":
		selectedMode = ModeHostPool
	}

	termChan := make(chan os.Signal, 1)
	signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)

	defaultUA := fmt.Sprintf("nsq_to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION)

	cCfg.UserAgent = defaultUA
	cCfg.MaxInFlight = *maxInFlight

	// TODO: remove, deprecated
	if hasArg("max-backoff-duration") {
		log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --consumer-opt=max_backoff_duration,X")
		cCfg.MaxBackoffDuration = *maxBackoffDuration
	}

	consumer, err := nsq.NewConsumer(*topic, *channel, cCfg)
	if err != nil {
		log.Fatal(err)
	}

	pCfg.UserAgent = defaultUA

	producers := make(map[string]*nsq.Producer)
	for _, addr := range destNsqdTCPAddrs {
		producer, err := nsq.NewProducer(addr, pCfg)
		if err != nil {
			log.Fatalf("failed creating producer %s", err)
		}
		producers[addr] = producer
	}

	perAddressStatus := make(map[string]*timer_metrics.TimerMetrics)
	if len(destNsqdTCPAddrs) == 1 {
		// disable since there is only one address
		perAddressStatus[destNsqdTCPAddrs[0]] = timer_metrics.NewTimerMetrics(0, "")
	} else {
		for _, a := range destNsqdTCPAddrs {
			perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery,
				fmt.Sprintf("[%s]:", a))
		}
	}

	hostPool := hostpool.New(destNsqdTCPAddrs)
	if *mode == "epsilon-greedy" {
		hostPool = hostpool.NewEpsilonGreedy(destNsqdTCPAddrs, 0, &hostpool.LinearEpsilonValueCalculator{})
	}

	handler := &PublishHandler{
		addresses:        destNsqdTCPAddrs,
		producers:        producers,
		mode:             selectedMode,
		hostPool:         hostPool,
		respChan:         make(chan *nsq.ProducerTransaction, len(destNsqdTCPAddrs)),
		perAddressStatus: perAddressStatus,
		timermetrics:     timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"),
	}
	consumer.AddConcurrentHandlers(handler, len(destNsqdTCPAddrs))

	for i := 0; i < len(destNsqdTCPAddrs); i++ {
		go handler.responder()
	}

	err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case <-consumer.StopChan:
			return
		case <-termChan:
			consumer.Stop()
		}
	}
}
Ejemplo n.º 12
0
func (self *NsqLookupCoordinator) ChangeTopicMetaParam(topic string,
	newSyncEvery int, newRetentionDay int, newReplicator int) error {
	if self.leaderNode.GetID() != self.myNode.GetID() {
		coordLog.Infof("not leader while create topic")
		return ErrNotNsqLookupLeader
	}

	if !protocol.IsValidTopicName(topic) {
		return errors.New("invalid topic name")
	}

	if newRetentionDay > MAX_RETENTION_DAYS {
		return errors.New("max retention days allowed exceed")
	}
	if newSyncEvery > MAX_SYNC_EVERY {
		return errors.New("max sync every allowed exceed")
	}
	if newReplicator > 5 {
		return errors.New("max replicator allowed exceed")
	}

	self.joinStateMutex.Lock()
	state, ok := self.joinISRState[topic]
	if !ok {
		state = &JoinISRState{}
		self.joinISRState[topic] = state
	}
	self.joinStateMutex.Unlock()
	state.Lock()
	defer state.Unlock()
	if state.waitingJoin {
		coordLog.Warningf("topic state is not ready:%v, %v ", topic, state)
		return ErrWaitingJoinISR.ToErrorType()
	}
	var meta TopicMetaInfo
	if ok, _ := self.leadership.IsExistTopic(topic); !ok {
		coordLog.Infof("topic not exist %v :%v", topic)
		return ErrTopicNotCreated
	} else {
		oldMeta, oldGen, err := self.leadership.GetTopicMetaInfo(topic)
		if err != nil {
			coordLog.Infof("get topic key %v failed :%v", topic, err)
			return err
		}
		currentNodes := self.getCurrentNodes()
		meta = oldMeta
		if newSyncEvery >= 0 {
			meta.SyncEvery = newSyncEvery
		}
		if newRetentionDay >= 0 {
			meta.RetentionDay = int32(newRetentionDay)
		}
		if newReplicator > 0 {
			meta.Replica = newReplicator
		}
		err = self.updateTopicMeta(currentNodes, topic, meta, oldGen)
		if err != nil {
			return err
		}
		for i := 0; i < meta.PartitionNum; i++ {
			topicInfo, err := self.leadership.GetTopicInfo(topic, i)
			if err != nil {
				coordLog.Infof("failed get info for topic : %v-%v, %v", topic, i, err)
				continue
			}
			topicReplicaInfo := &topicInfo.TopicPartitionReplicaInfo
			err = self.leadership.UpdateTopicNodeInfo(topic, i, topicReplicaInfo, topicReplicaInfo.Epoch)
			if err != nil {
				coordLog.Infof("failed update info for topic : %v-%v, %v", topic, i, err)
				continue
			}
			rpcErr := self.notifyTopicMetaInfo(topicInfo)
			if rpcErr != nil {
				coordLog.Warningf("failed notify topic info : %v", rpcErr)
			} else {
				coordLog.Infof("topic %v update successful.", topicInfo)
			}
		}

		self.triggerCheckTopics("", 0, 0)
	}
	return nil
}
Ejemplo n.º 13
0
func (p *protocolV2) internalSUB(client *nsqd.ClientV2, params [][]byte, enableTrace bool,
	ordered bool, startFrom *ConsumeOffset) ([]byte, error) {

	state := atomic.LoadInt32(&client.State)
	if state != stateInit {
		nsqd.NsqLogger().LogWarningf("[%s] command in wrong state: %v", client, state)
		return nil, protocol.NewFatalClientErr(nil, E_INVALID, "cannot SUB in current state")
	}

	if client.HeartbeatInterval <= 0 {
		return nil, protocol.NewFatalClientErr(nil, E_INVALID, "cannot SUB with heartbeats disabled")
	}

	if len(params) < 3 {
		return nil, protocol.NewFatalClientErr(nil, E_INVALID, "SUB insufficient number of parameters")
	}

	topicName := string(params[1])
	if !protocol.IsValidTopicName(topicName) {
		return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC",
			fmt.Sprintf("SUB topic name %q is not valid", topicName))
	}

	partition := -1
	channelName := ""
	var err error
	channelName = string(params[2])
	if !protocol.IsValidChannelName(channelName) {
		return nil, protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL",
			fmt.Sprintf("SUB channel name %q is not valid", channelName))
	}

	if len(params) == 4 {
		partition, err = strconv.Atoi(string(params[3]))
		if err != nil {
			return nil, protocol.NewFatalClientErr(nil, "E_BAD_PARTITION",
				fmt.Sprintf("topic partition is not valid: %v", err))
		}
	}

	if err = p.CheckAuth(client, "SUB", topicName, channelName); err != nil {
		return nil, err
	}

	if partition == -1 {
		partition = p.ctx.getDefaultPartition(topicName)
	}

	topic, err := p.ctx.getExistingTopic(topicName, partition)
	if err != nil {
		nsqd.NsqLogger().Logf("sub to not existing topic: %v, err:%v", topicName, err.Error())
		return nil, protocol.NewFatalClientErr(nil, E_TOPIC_NOT_EXIST, "")
	}
	if !p.ctx.checkForMasterWrite(topicName, partition) {
		nsqd.NsqLogger().Logf("sub failed on not leader: %v-%v, remote is : %v", topicName, partition, client.RemoteAddr())
		// we need disable topic here to trigger a notify, maybe we failed to notify lookup last time.
		topic.DisableForSlave()
		return nil, protocol.NewFatalClientErr(nil, FailedOnNotLeader, "")
	}
	channel := topic.GetChannel(channelName)
	err = channel.AddClient(client.ID, client)
	if err != nil {
		nsqd.NsqLogger().Logf("sub failed to add client: %v, %v", client, err)
		return nil, protocol.NewFatalClientErr(nil, FailedOnNotWritable, "")
	}

	atomic.StoreInt32(&client.State, stateSubscribed)
	client.Channel = channel
	if enableTrace {
		nsqd.NsqLogger().Logf("sub channel %v with trace enabled, remote is : %v", channelName, client.RemoteAddr())
	}
	if ordered {
		if atomic.LoadInt32(&client.SampleRate) != 0 {
			nsqd.NsqLogger().Errorf("%v", ErrOrderChannelOnSampleRate)
			return nil, protocol.NewFatalClientErr(nil, E_INVALID, ErrOrderChannelOnSampleRate.Error())
		}
		channel.SetOrdered(true)
	}
	if startFrom != nil {
		cnt := channel.GetClientsCount()
		if cnt > 1 {
			nsqd.NsqLogger().LogDebugf("the consume offset: %v can only be set by the first client: %v", startFrom, cnt)
		} else {
			queueOffset, cnt, err := p.ctx.SetChannelOffset(channel, startFrom, false)
			if err != nil {
				return nil, protocol.NewFatalClientErr(nil, E_INVALID, err.Error())
			}
			nsqd.NsqLogger().Logf("set the channel offset: %v (actual set : %v:%v), by client:%v, %v",
				startFrom, queueOffset, cnt, client.String(), client.UserAgent)
		}
	}
	client.EnableTrace = enableTrace
	// update message pump
	client.SubEventChan <- channel

	return okBytes, nil
}
Ejemplo n.º 14
0
func (s *httpServer) doCreateTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {
	reqParams, err := url.ParseQuery(req.URL.RawQuery)
	if err != nil {
		return nil, http_api.Err{400, "INVALID_REQUEST"}
	}

	topicName := reqParams.Get("topic")
	if topicName == "" {
		return nil, http_api.Err{400, "MISSING_ARG_TOPIC"}
	}

	if !protocol.IsValidTopicName(topicName) {
		return nil, http_api.Err{400, "INVALID_ARG_TOPIC"}
	}

	pnumStr := reqParams.Get("partition_num")
	if pnumStr == "" {
		return nil, http_api.Err{400, "MISSING_ARG_TOPIC_PARTITION_NUM"}
	}
	pnum, err := GetValidPartitionNum(pnumStr)
	if err != nil {
		return nil, http_api.Err{400, "INVALID_ARG_TOPIC_PARTITION_NUM"}
	}
	replicatorStr := reqParams.Get("replicator")
	if replicatorStr == "" {
		return nil, http_api.Err{400, "MISSING_ARG_TOPIC_REPLICATOR"}
	}
	replicator, err := GetValidReplicator(replicatorStr)
	if err != nil {
		return nil, http_api.Err{400, "INVALID_ARG_TOPIC_REPLICATOR"}
	}

	suggestLFStr := reqParams.Get("suggestload")
	if suggestLFStr == "" {
		suggestLFStr = "0"
	}
	suggestLF, err := GetValidSuggestLF(suggestLFStr)
	if err != nil {
		return nil, http_api.Err{400, "INVALID_ARG_TOPIC_LOAD_FACTOR"}
	}
	syncEveryStr := reqParams.Get("syncdisk")
	if syncEveryStr == "" {
		syncEveryStr = "0"
	}
	syncEvery, err := strconv.Atoi(syncEveryStr)
	if err != nil {
		nsqlookupLog.Logf("error sync disk param: %v, %v", syncEvery, err)
		return nil, http_api.Err{400, "INVALID_ARG_TOPIC_SYNC_DISK"}
	}
	retentionDaysStr := reqParams.Get("retention")
	if retentionDaysStr == "" {
		retentionDaysStr = "0"
	}
	retentionDays, err := strconv.Atoi(retentionDaysStr)
	if err != nil {
		nsqlookupLog.Logf("error retention param: %v, %v", retentionDaysStr, err)
		return nil, http_api.Err{400, err.Error()}
	}

	if !s.ctx.nsqlookupd.coordinator.IsMineLeader() {
		nsqlookupLog.LogDebugf("create topic (%s) from remote %v should request to leader", topicName, req.RemoteAddr)
		return nil, http_api.Err{400, consistence.ErrFailedOnNotLeader}
	}

	nsqlookupLog.Logf("creating topic(%s) with partition %v replicator: %v load: %v", topicName, pnum, replicator, suggestLF)

	if s.ctx.nsqlookupd.coordinator == nil {
		return nil, http_api.Err{500, "MISSING_COORDINATOR"}
	}
	meta := consistence.TopicMetaInfo{}
	meta.PartitionNum = pnum
	meta.Replica = replicator
	meta.SuggestLF = suggestLF
	meta.SyncEvery = syncEvery
	meta.RetentionDay = int32(retentionDays)
	err = s.ctx.nsqlookupd.coordinator.CreateTopic(topicName, meta)
	if err != nil {
		nsqlookupLog.LogErrorf("DB: adding topic(%s) failed: %v", topicName, err)
		return nil, http_api.Err{400, err.Error()}
	}

	return nil, nil
}