func (t *Topic) LoadChannelMeta() error {
fn := t.getChannelMetaFileName()
data, err := ioutil.ReadFile(fn)
if err != nil {
if !os.IsNotExist(err) {
nsqLog.LogErrorf("failed to read channel metadata from %s - %s", fn, err)
}
return err
}
channels := make([]*ChannelMetaInfo, 0)
err = json.Unmarshal(data, &channels)
if err != nil {
nsqLog.LogErrorf("failed to parse metadata - %s", err)
return err
}
for _, ch := range channels {
channelName := ch.Name
if !protocol.IsValidChannelName(channelName) {
nsqLog.LogWarningf("skipping creation of invalid channel %s", channelName)
continue
}
channel := t.GetChannel(channelName)
if ch.Paused {
channel.Pause()
}
}
return nil
}
func getTopicChan(command string, params []string) (string, string, string, error) {
if len(params) <= 1 {
return "", "", "", protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("%s insufficient number of params", command))
}
topicName := params[0]
partitionID := params[1]
var channelName string
if len(params) >= 3 {
channelName = params[2]
}
if !protocol.IsValidTopicName(topicName) {
return "", "", "", protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("%s topic name '%s' is not valid", command, topicName))
}
if channelName != "" && !protocol.IsValidChannelName(channelName) {
return "", "", "", protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL", fmt.Sprintf("%s channel name '%s' is not valid", command, channelName))
}
if _, err := GetValidPartitionID(partitionID); err != nil {
return "", "", "", protocol.NewFatalClientErr(nil, "E_BAD_PARTITIONID", fmt.Sprintf("%s partition id '%s' is not valid", command, partitionID))
}
return topicName, channelName, partitionID, nil
}
func GetTopicPartitionChannelArgs(rp getter) (string, int, string, error) {
topicName, topicPart, err := GetTopicPartitionArgs(rp)
if err != nil {
return "", -1, "", err
}
channelName := rp.Get("channel")
if channelName == "" {
return "", -1, "", errors.New("MISSING_ARG_CHANNEL")
}
if !protocol.IsValidChannelName(channelName) {
return "", -1, "", errors.New("INVALID_ARG_CHANNEL")
}
return topicName, topicPart, channelName, nil
}
func getTopicChanFromOld(command string, params []string) (string, string, error) {
if len(params) <= 0 {
return "", "", protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("%s insufficient number of params", command))
}
topicName := params[0]
var channelName string
if len(params) >= 2 {
channelName = params[1]
}
if !protocol.IsValidTopicName(topicName) {
return "", "", protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("%s topic name '%s' is not valid", command, topicName))
}
if channelName != "" && !protocol.IsValidChannelName(channelName) {
return "", "", protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL", fmt.Sprintf("%s channel name '%s' is not valid", command, channelName))
}
return topicName, channelName, nil
}
func (n *NSQD) LoadMetadata(disabled int32) {
atomic.StoreInt32(&n.isLoading, 1)
defer atomic.StoreInt32(&n.isLoading, 0)
fn := fmt.Sprintf(path.Join(n.GetOpts().DataPath, "nsqd.%d.dat"), n.GetOpts().ID)
data, err := ioutil.ReadFile(fn)
if err != nil {
if !os.IsNotExist(err) {
nsqLog.LogErrorf("failed to read channel metadata from %s - %s", fn, err)
}
return
}
js, err := simplejson.NewJson(data)
if err != nil {
nsqLog.LogErrorf("failed to parse metadata - %s", err)
return
}
topics, err := js.Get("topics").Array()
if err != nil {
nsqLog.LogErrorf("failed to parse metadata - %s", err)
return
}
for ti := range topics {
topicJs := js.Get("topics").GetIndex(ti)
topicName, err := topicJs.Get("name").String()
if err != nil {
nsqLog.LogErrorf("failed to parse metadata - %s", err)
return
}
if !protocol.IsValidTopicName(topicName) {
nsqLog.LogWarningf("skipping creation of invalid topic %s", topicName)
continue
}
part, err := topicJs.Get("partition").Int()
if err != nil {
nsqLog.LogErrorf("failed to parse metadata - %s", err)
return
}
topic := n.internalGetTopic(topicName, part, disabled)
// old meta should also be loaded
channels, err := topicJs.Get("channels").Array()
if err != nil {
nsqLog.LogErrorf("failed to parse metadata - %s", err)
return
}
for ci := range channels {
channelJs := topicJs.Get("channels").GetIndex(ci)
channelName, err := channelJs.Get("name").String()
if err != nil {
nsqLog.LogErrorf("failed to parse metadata - %s", err)
return
}
if !protocol.IsValidChannelName(channelName) {
nsqLog.LogWarningf("skipping creation of invalid channel %s", channelName)
continue
}
channel := topic.GetChannel(channelName)
paused, _ := channelJs.Get("paused").Bool()
if paused {
channel.Pause()
}
}
// we load channels from the new meta file
topic.LoadChannelMeta()
}
}
func main() {
var selectedMode int
cCfg := nsq.NewConfig()
pCfg := nsq.NewConfig()
// TODO: remove, deprecated
flag.Var(&nsq.ConfigFlag{cCfg}, "reader-opt", "(deprecated) use --consumer-opt")
flag.Var(&nsq.ConfigFlag{cCfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, see http://godoc.org/github.com/absolute8511/go-nsq#Config)")
flag.Var(&nsq.ConfigFlag{pCfg}, "producer-opt", "option to passthrough to nsq.Producer (may be given multiple times, see http://godoc.org/github.com/absolute8511/go-nsq#Config)")
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_nsq v%s\n", version.Binary)
return
}
if *topic == "" || *channel == "" {
log.Fatal("--topic and --channel are required")
}
if *destTopic == "" {
*destTopic = *topic
}
if !protocol.IsValidTopicName(*topic) {
log.Fatal("--topic is invalid")
}
if !protocol.IsValidTopicName(*destTopic) {
log.Fatal("--destination-topic is invalid")
}
if !protocol.IsValidChannelName(*channel) {
log.Fatal("--channel is invalid")
}
if len(lookupdHTTPAddrs) == 0 {
log.Fatal("--lookupd-http-address required")
}
if len(destNsqdTCPAddrs) == 0 {
log.Fatal("--destination-nsqd-tcp-address required")
}
switch *mode {
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool", "epsilon-greedy":
selectedMode = ModeHostPool
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
defaultUA := fmt.Sprintf("nsq_to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION)
cCfg.UserAgent = defaultUA
cCfg.MaxInFlight = *maxInFlight
// TODO: remove, deprecated
if hasArg("max-backoff-duration") {
log.Printf("WARNING: --max-backoff-duration is deprecated in favor of --consumer-opt=max_backoff_duration,X")
cCfg.MaxBackoffDuration = *maxBackoffDuration
}
consumer, err := nsq.NewConsumer(*topic, *channel, cCfg)
if err != nil {
log.Fatal(err)
}
pCfg.UserAgent = defaultUA
producers := make(map[string]*nsq.Producer)
for _, addr := range destNsqdTCPAddrs {
producer, err := nsq.NewProducer(addr, pCfg)
if err != nil {
log.Fatalf("failed creating producer %s", err)
}
producers[addr] = producer
}
perAddressStatus := make(map[string]*timer_metrics.TimerMetrics)
if len(destNsqdTCPAddrs) == 1 {
// disable since there is only one address
perAddressStatus[destNsqdTCPAddrs[0]] = timer_metrics.NewTimerMetrics(0, "")
} else {
for _, a := range destNsqdTCPAddrs {
perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery,
fmt.Sprintf("[%s]:", a))
}
}
hostPool := hostpool.New(destNsqdTCPAddrs)
if *mode == "epsilon-greedy" {
hostPool = hostpool.NewEpsilonGreedy(destNsqdTCPAddrs, 0, &hostpool.LinearEpsilonValueCalculator{})
}
handler := &PublishHandler{
addresses: destNsqdTCPAddrs,
producers: producers,
mode: selectedMode,
hostPool: hostPool,
respChan: make(chan *nsq.ProducerTransaction, len(destNsqdTCPAddrs)),
perAddressStatus: perAddressStatus,
timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"),
}
consumer.AddConcurrentHandlers(handler, len(destNsqdTCPAddrs))
for i := 0; i < len(destNsqdTCPAddrs); i++ {
go handler.responder()
}
err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs)
if err != nil {
log.Fatal(err)
}
for {
select {
case <-consumer.StopChan:
return
case <-termChan:
consumer.Stop()
}
}
}
func (p *protocolV2) internalSUB(client *nsqd.ClientV2, params [][]byte, enableTrace bool,
ordered bool, startFrom *ConsumeOffset) ([]byte, error) {
state := atomic.LoadInt32(&client.State)
if state != stateInit {
nsqd.NsqLogger().LogWarningf("[%s] command in wrong state: %v", client, state)
return nil, protocol.NewFatalClientErr(nil, E_INVALID, "cannot SUB in current state")
}
if client.HeartbeatInterval <= 0 {
return nil, protocol.NewFatalClientErr(nil, E_INVALID, "cannot SUB with heartbeats disabled")
}
if len(params) < 3 {
return nil, protocol.NewFatalClientErr(nil, E_INVALID, "SUB insufficient number of parameters")
}
topicName := string(params[1])
if !protocol.IsValidTopicName(topicName) {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC",
fmt.Sprintf("SUB topic name %q is not valid", topicName))
}
partition := -1
channelName := ""
var err error
channelName = string(params[2])
if !protocol.IsValidChannelName(channelName) {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL",
fmt.Sprintf("SUB channel name %q is not valid", channelName))
}
if len(params) == 4 {
partition, err = strconv.Atoi(string(params[3]))
if err != nil {
return nil, protocol.NewFatalClientErr(nil, "E_BAD_PARTITION",
fmt.Sprintf("topic partition is not valid: %v", err))
}
}
if err = p.CheckAuth(client, "SUB", topicName, channelName); err != nil {
return nil, err
}
if partition == -1 {
partition = p.ctx.getDefaultPartition(topicName)
}
topic, err := p.ctx.getExistingTopic(topicName, partition)
if err != nil {
nsqd.NsqLogger().Logf("sub to not existing topic: %v, err:%v", topicName, err.Error())
return nil, protocol.NewFatalClientErr(nil, E_TOPIC_NOT_EXIST, "")
}
if !p.ctx.checkForMasterWrite(topicName, partition) {
nsqd.NsqLogger().Logf("sub failed on not leader: %v-%v, remote is : %v", topicName, partition, client.RemoteAddr())
// we need disable topic here to trigger a notify, maybe we failed to notify lookup last time.
topic.DisableForSlave()
return nil, protocol.NewFatalClientErr(nil, FailedOnNotLeader, "")
}
channel := topic.GetChannel(channelName)
err = channel.AddClient(client.ID, client)
if err != nil {
nsqd.NsqLogger().Logf("sub failed to add client: %v, %v", client, err)
return nil, protocol.NewFatalClientErr(nil, FailedOnNotWritable, "")
}
atomic.StoreInt32(&client.State, stateSubscribed)
client.Channel = channel
if enableTrace {
nsqd.NsqLogger().Logf("sub channel %v with trace enabled, remote is : %v", channelName, client.RemoteAddr())
}
if ordered {
if atomic.LoadInt32(&client.SampleRate) != 0 {
nsqd.NsqLogger().Errorf("%v", ErrOrderChannelOnSampleRate)
return nil, protocol.NewFatalClientErr(nil, E_INVALID, ErrOrderChannelOnSampleRate.Error())
}
channel.SetOrdered(true)
}
if startFrom != nil {
cnt := channel.GetClientsCount()
if cnt > 1 {
nsqd.NsqLogger().LogDebugf("the consume offset: %v can only be set by the first client: %v", startFrom, cnt)
} else {
queueOffset, cnt, err := p.ctx.SetChannelOffset(channel, startFrom, false)
if err != nil {
return nil, protocol.NewFatalClientErr(nil, E_INVALID, err.Error())
}
nsqd.NsqLogger().Logf("set the channel offset: %v (actual set : %v:%v), by client:%v, %v",
startFrom, queueOffset, cnt, client.String(), client.UserAgent)
}
}
client.EnableTrace = enableTrace
// update message pump
client.SubEventChan <- channel
return okBytes, nil
}