// test channel/topic names
func TestChannelTopicNames(t *testing.T) {
assert.Equal(t, nsq.IsValidChannelName("test"), true)
assert.Equal(t, nsq.IsValidChannelName("test-with_period."), true)
assert.Equal(t, nsq.IsValidChannelName("test#ephemeral"), true)
assert.Equal(t, nsq.IsValidTopicName("test"), true)
assert.Equal(t, nsq.IsValidTopicName("test-with_period."), true)
assert.Equal(t, nsq.IsValidTopicName("test#ephemeral"), false)
assert.Equal(t, nsq.IsValidTopicName("test:ephemeral"), false)
}
func (s *httpServer) emptyTopicHandler(w http.ResponseWriter, req *http.Request) {
reqParams, err := util.NewReqParams(req)
if err != nil {
log.Printf("ERROR: failed to parse request params - %s", err.Error())
util.ApiResponse(w, 500, "INVALID_REQUEST", nil)
return
}
topicName, err := reqParams.Get("topic")
if err != nil {
util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil)
return
}
if !nsq.IsValidTopicName(topicName) {
util.ApiResponse(w, 500, "INVALID_TOPIC", nil)
return
}
topic, err := s.context.nsqd.GetExistingTopic(topicName)
if err != nil {
util.ApiResponse(w, 500, "INVALID_TOPIC", nil)
return
}
err = topic.Empty()
if err != nil {
util.ApiResponse(w, 500, "INTERNAL_ERROR", nil)
return
}
util.ApiResponse(w, 200, "OK", nil)
}
func putHandler(w http.ResponseWriter, req *http.Request) {
reqParams, err := util.NewReqParams(req)
if err != nil {
log.Printf("ERROR: failed to parse request params - %s", err.Error())
util.ApiResponse(w, 500, "INVALID_REQUEST", nil)
return
}
topicName, err := reqParams.Get("topic")
if err != nil {
util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil)
return
}
if !nsq.IsValidTopicName(topicName) {
util.ApiResponse(w, 500, "INVALID_ARG_TOPIC", nil)
return
}
if int64(len(reqParams.Body)) > nsqd.options.maxMessageSize {
util.ApiResponse(w, 500, "MSG_TOO_BIG", nil)
return
}
topic := nsqd.GetTopic(topicName)
msg := nsq.NewMessage(<-nsqd.idChan, reqParams.Body)
err = topic.PutMessage(msg)
if err != nil {
util.ApiResponse(w, 500, "NOK", nil)
return
}
w.Header().Set("Content-Length", "2")
io.WriteString(w, "OK")
}
func (s *httpServer) mputHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
util.ApiResponse(w, 500, "INVALID_REQUEST", nil)
return
}
reqParams, err := util.NewReqParams(req)
if err != nil {
log.Printf("ERROR: failed to parse request params - %s", err.Error())
util.ApiResponse(w, 500, "INVALID_REQUEST", nil)
return
}
topicName, err := reqParams.Get("topic")
if err != nil {
util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil)
return
}
if !nsq.IsValidTopicName(topicName) {
util.ApiResponse(w, 500, "INVALID_ARG_TOPIC", nil)
return
}
topic := s.context.nsqd.GetTopic(topicName)
for _, block := range bytes.Split(reqParams.Body, []byte("\n")) {
if len(block) != 0 {
if int64(len(reqParams.Body)) > s.context.nsqd.options.maxMessageSize {
util.ApiResponse(w, 500, "MSG_TOO_BIG", nil)
return
}
msg := nsq.NewMessage(<-s.context.nsqd.idChan, block)
err := topic.PutMessage(msg)
if err != nil {
util.ApiResponse(w, 500, "NOK", nil)
return
}
}
}
w.Header().Set("Content-Length", "2")
io.WriteString(w, "OK")
}
func getTopicChan(command string, params []string) (string, string, error) {
if len(params) == 0 {
return "", "", nsq.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("%s insufficient number of params", command))
}
topicName := params[0]
var channelName string
if len(params) >= 2 {
channelName = params[1]
}
if !nsq.IsValidTopicName(topicName) {
return "", "", nsq.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("%s topic name '%s' is not valid", command, topicName))
}
if channelName != "" && !nsq.IsValidChannelName(channelName) {
return "", "", nsq.NewFatalClientErr(nil, "E_BAD_CHANNEL", fmt.Sprintf("%s channel name '%s' is not valid", command, channelName))
}
return topicName, channelName, nil
}
func GetTopicChannelArgs(rp Getter) (string, string, error) {
topicName, err := rp.Get("topic")
if err != nil {
return "", "", errors.New("MISSING_ARG_TOPIC")
}
if !nsq.IsValidTopicName(topicName) {
return "", "", errors.New("INVALID_ARG_TOPIC")
}
channelName, err := rp.Get("channel")
if err != nil {
return "", "", errors.New("MISSING_ARG_CHANNEL")
}
if !nsq.IsValidChannelName(channelName) {
return "", "", errors.New("INVALID_ARG_CHANNEL")
}
return topicName, channelName, nil
}
func (p *ProtocolV2) PUB(client *ClientV2, params [][]byte) ([]byte, error) {
var err error
var bodyLen int32
if len(params) < 2 {
return nil, nsq.NewFatalClientErr(nil, "E_INVALID", "PUB insufficient number of parameters")
}
topicName := string(params[1])
if !nsq.IsValidTopicName(topicName) {
return nil, nsq.NewFatalClientErr(nil, "E_BAD_TOPIC",
fmt.Sprintf("PUB topic name '%s' is not valid", topicName))
}
err = binary.Read(client.Reader, binary.BigEndian, &bodyLen)
if err != nil {
return nil, nsq.NewFatalClientErr(err, "E_BAD_MESSAGE", "PUB failed to read message body size")
}
if int64(bodyLen) > nsqd.options.maxMessageSize {
return nil, nsq.NewFatalClientErr(nil, "E_BAD_MESSAGE",
fmt.Sprintf("PUB message too big %d > %d", bodyLen, nsqd.options.maxMessageSize))
}
messageBody := make([]byte, bodyLen)
_, err = io.ReadFull(client.Reader, messageBody)
if err != nil {
return nil, nsq.NewFatalClientErr(err, "E_BAD_MESSAGE", "PUB failed to read message body")
}
topic := nsqd.GetTopic(topicName)
msg := nsq.NewMessage(<-nsqd.idChan, messageBody)
err = topic.PutMessage(msg)
if err != nil {
return nil, nsq.NewFatalClientErr(err, "E_PUB_FAILED", "PUB failed "+err.Error())
}
return []byte("OK"), nil
}
func createTopicHandler(w http.ResponseWriter, req *http.Request) {
reqParams, err := util.NewReqParams(req)
if err != nil {
log.Printf("ERROR: failed to parse request params - %s", err.Error())
util.ApiResponse(w, 500, "INVALID_REQUEST", nil)
return
}
topicName, err := reqParams.Get("topic")
if err != nil {
util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil)
return
}
if !nsq.IsValidTopicName(topicName) {
util.ApiResponse(w, 500, "INVALID_TOPIC", nil)
return
}
nsqd.GetTopic(topicName)
util.ApiResponse(w, 200, "OK", nil)
}
func createTopicHandler(w http.ResponseWriter, req *http.Request) {
reqParams, err := util.NewReqParams(req)
if err != nil {
util.ApiResponse(w, 500, "INVALID_REQUEST", nil)
return
}
topicName, err := reqParams.Get("topic")
if err != nil {
util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil)
return
}
if !nsq.IsValidTopicName(topicName) {
util.ApiResponse(w, 500, "INVALID_TOPIC", nil)
return
}
log.Printf("DB: adding topic(%s)", topicName)
key := Registration{"topic", topicName, ""}
lookupd.DB.AddRegistration(key)
util.ApiResponse(w, 200, "OK", nil)
}
func (p *ProtocolV2) SUB(client *ClientV2, params [][]byte) ([]byte, error) {
if atomic.LoadInt32(&client.State) != nsq.StateInit {
return nil, nsq.NewFatalClientErr(nil, "E_INVALID", "cannot SUB in current state")
}
if client.HeartbeatInterval < 0 {
return nil, nsq.NewFatalClientErr(nil, "E_INVALID", "cannot SUB with heartbeats disabled")
}
if len(params) < 3 {
return nil, nsq.NewFatalClientErr(nil, "E_INVALID", "SUB insufficient number of parameters")
}
topicName := string(params[1])
if !nsq.IsValidTopicName(topicName) {
return nil, nsq.NewFatalClientErr(nil, "E_BAD_TOPIC",
fmt.Sprintf("SUB topic name '%s' is not valid", topicName))
}
channelName := string(params[2])
if !nsq.IsValidChannelName(channelName) {
return nil, nsq.NewFatalClientErr(nil, "E_BAD_CHANNEL",
fmt.Sprintf("SUB channel name '%s' is not valid", channelName))
}
topic := nsqd.GetTopic(topicName)
channel := topic.GetChannel(channelName)
channel.AddClient(client)
atomic.StoreInt32(&client.State, nsq.StateSubscribed)
client.Channel = channel
// update message pump
client.SubEventChan <- channel
return okBytes, nil
}
func createTopicChannelHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
log.Printf("ERROR: invalid %s to POST only method", req.Method)
http.Error(w, "INVALID_REQUEST", 500)
return
}
reqParams := &util.PostParams{req}
topicName, err := reqParams.Get("topic")
if err != nil || !nsq.IsValidTopicName(topicName) {
http.Error(w, "INVALID_TOPIC", 500)
return
}
channelName, err := reqParams.Get("channel")
if err != nil || (len(channelName) > 0 && !nsq.IsValidChannelName(channelName)) {
http.Error(w, "INVALID_CHANNEL", 500)
return
}
for _, addr := range lookupdHTTPAddrs {
endpoint := fmt.Sprintf("http://%s/create_topic?topic=%s", addr, url.QueryEscape(topicName))
log.Printf("LOOKUPD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
continue
}
}
NotifyAdminAction("create_topic", topicName, "", req)
if len(channelName) > 0 {
for _, addr := range lookupdHTTPAddrs {
endpoint := fmt.Sprintf("http://%s/create_channel?topic=%s&channel=%s",
addr, url.QueryEscape(topicName), url.QueryEscape(channelName))
log.Printf("LOOKUPD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: lookupd %s - %s", endpoint, err.Error())
continue
}
}
// TODO: we can remove this when we push new channel information from nsqlookupd -> nsqd
producers, _ := getLookupdTopicProducers(topicName, lookupdHTTPAddrs)
for _, addr := range producers {
endpoint := fmt.Sprintf("http://%s/create_channel?topic=%s&channel=%s",
addr, url.QueryEscape(topicName), url.QueryEscape(channelName))
log.Printf("NSQD: querying %s", endpoint)
_, err := nsq.ApiRequest(endpoint)
if err != nil {
log.Printf("ERROR: nsqd %s - %s", endpoint, err.Error())
continue
}
}
NotifyAdminAction("create_channel", topicName, channelName, req)
}
http.Redirect(w, req, "/lookup", 302)
}
func topicHandler(w http.ResponseWriter, req *http.Request) {
var urlRegex = regexp.MustCompile(`^/topic/(.*)$`)
matches := urlRegex.FindStringSubmatch(req.URL.Path)
if len(matches) == 0 {
http.Error(w, "INVALID_TOPIC", 500)
return
}
parts := strings.Split(matches[1], "/")
topicName := parts[0]
if !nsq.IsValidTopicName(topicName) {
http.Error(w, "INVALID_TOPIC", 500)
return
}
if len(parts) == 2 {
channelName := parts[1]
if !nsq.IsValidChannelName(channelName) {
http.Error(w, "INVALID_CHANNEL", 500)
} else {
channelHandler(w, req, topicName, channelName)
}
return
}
reqParams, err := util.NewReqParams(req)
if err != nil {
log.Printf("ERROR: failed to parse request params - %s", err.Error())
http.Error(w, "INVALID_REQUEST", 500)
return
}
var producers []string
if len(lookupdHTTPAddrs) != 0 {
producers, _ = getLookupdTopicProducers(topicName, lookupdHTTPAddrs)
} else {
producers, _ = getNSQDTopicProducers(topicName, nsqdHTTPAddrs)
}
topicHostStats, channelStats, _ := getNSQDStats(producers, topicName)
globalTopicStats := &TopicHostStats{HostAddress: "Total"}
for _, t := range topicHostStats {
globalTopicStats.AddHostStats(t)
}
p := struct {
Title string
GraphOptions *GraphOptions
Version string
Topic string
TopicProducers []string
TopicHostStats []*TopicHostStats
GlobalTopicStats *TopicHostStats
ChannelStats map[string]*ChannelStats
}{
Title: fmt.Sprintf("NSQ %s", topicName),
GraphOptions: NewGraphOptions(w, req, reqParams),
Version: util.BINARY_VERSION,
Topic: topicName,
TopicProducers: producers,
TopicHostStats: topicHostStats,
GlobalTopicStats: globalTopicStats,
ChannelStats: channelStats,
}
err = templates.ExecuteTemplate(w, "topic.html", p)
if err != nil {
log.Printf("Template Error %s", err.Error())
http.Error(w, "Template Error", 500)
}
}
func (n *NSQd) LoadMetadata() {
fn := fmt.Sprintf(path.Join(n.options.dataPath, "nsqd.%d.dat"), n.workerId)
data, err := ioutil.ReadFile(fn)
if err != nil {
if !os.IsNotExist(err) {
log.Printf("ERROR: failed to read channel metadata from %s - %s", fn, err.Error())
}
return
}
js, err := simplejson.NewJson(data)
if err != nil {
log.Printf("ERROR: failed to parse metadata - %s", err.Error())
return
}
topics, err := js.Get("topics").Array()
if err != nil {
log.Printf("ERROR: failed to parse metadata - %s", err.Error())
return
}
for ti := range topics {
topicJs := js.Get("topics").GetIndex(ti)
topicName, err := topicJs.Get("name").String()
if err != nil {
log.Printf("ERROR: failed to parse metadata - %s", err.Error())
return
}
if !nsq.IsValidTopicName(topicName) {
log.Printf("WARNING: skipping creation of invalid topic %s", topicName)
continue
}
topic := n.GetTopic(topicName)
channels, err := topicJs.Get("channels").Array()
if err != nil {
log.Printf("ERROR: failed to parse metadata - %s", err.Error())
return
}
for ci := range channels {
channelJs := topicJs.Get("channels").GetIndex(ci)
channelName, err := channelJs.Get("name").String()
if err != nil {
log.Printf("ERROR: failed to parse metadata - %s", err.Error())
return
}
if !nsq.IsValidChannelName(channelName) {
log.Printf("WARNING: skipping creation of invalid channel %s", channelName)
continue
}
channel := topic.GetChannel(channelName)
paused, _ := channelJs.Get("paused").Bool()
if paused {
channel.Pause()
}
}
}
}
func (p *ProtocolV2) MPUB(client *ClientV2, params [][]byte) ([]byte, error) {
var err error
if len(params) < 2 {
return nil, nsq.NewFatalClientErr(nil, "E_INVALID", "MPUB insufficient number of parameters")
}
topicName := string(params[1])
if !nsq.IsValidTopicName(topicName) {
return nil, nsq.NewFatalClientErr(nil, "E_BAD_TOPIC",
fmt.Sprintf("E_BAD_TOPIC MPUB topic name '%s' is not valid", topicName))
}
bodyLen, err := p.readLen(client)
if err != nil {
return nil, nsq.NewFatalClientErr(err, "E_BAD_BODY", "MPUB failed to read body size")
}
if int64(bodyLen) > nsqd.options.maxBodySize {
return nil, nsq.NewFatalClientErr(nil, "E_BAD_BODY",
fmt.Sprintf("MPUB body too big %d > %d", bodyLen, nsqd.options.maxBodySize))
}
numMessages, err := p.readLen(client)
if err != nil {
return nil, nsq.NewFatalClientErr(err, "E_BAD_BODY", "MPUB failed to read message count")
}
messages := make([]*nsq.Message, 0, numMessages)
for i := int32(0); i < numMessages; i++ {
messageSize, err := p.readLen(client)
if err != nil {
return nil, nsq.NewFatalClientErr(err, "E_BAD_MESSAGE",
fmt.Sprintf("MPUB failed to read message(%d) body size", i))
}
if int64(messageSize) > nsqd.options.maxMessageSize {
return nil, nsq.NewFatalClientErr(nil, "E_BAD_MESSAGE",
fmt.Sprintf("MPUB message too big %d > %d", messageSize, nsqd.options.maxMessageSize))
}
msgBody := make([]byte, messageSize)
_, err = io.ReadFull(client.Reader, msgBody)
if err != nil {
return nil, nsq.NewFatalClientErr(err, "E_BAD_MESSAGE", "MPUB failed to read message body")
}
messages = append(messages, nsq.NewMessage(<-nsqd.idChan, msgBody))
}
topic := nsqd.GetTopic(topicName)
// if we've made it this far we've validated all the input,
// the only possible error is that the topic is exiting during
// this next call (and no messages will be queued in that case)
err = topic.PutMessages(messages)
if err != nil {
return nil, nsq.NewFatalClientErr(err, "E_MPUB_FAILED", "MPUB failed "+err.Error())
}
return okBytes, nil
}
func (s *httpServer) mputHandler(w http.ResponseWriter, req *http.Request) {
var msgs []*nsq.Message
var exit bool
if req.Method != "POST" {
util.ApiResponse(w, 500, "INVALID_REQUEST", nil)
return
}
reqParams, err := url.ParseQuery(req.URL.RawQuery)
if err != nil {
log.Printf("ERROR: failed to parse request params - %s", err.Error())
util.ApiResponse(w, 500, "INVALID_REQUEST", nil)
}
topicNames, ok := reqParams["topic"]
if !ok {
util.ApiResponse(w, 500, "MISSING_ARG_TOPIC", nil)
return
}
topicName := topicNames[0]
if !nsq.IsValidTopicName(topicName) {
util.ApiResponse(w, 500, "INVALID_ARG_TOPIC", nil)
return
}
topic := s.context.nsqd.GetTopic(topicName)
rdr := bufio.NewReader(req.Body)
for !exit {
block, err := rdr.ReadBytes('\n')
if err != nil {
if err != io.EOF {
util.ApiResponse(w, 500, "INTERNAL_ERROR", nil)
return
}
exit = true
}
if len(block) > 0 && block[len(block)-1] == '\n' {
block = block[:len(block)-1]
}
if int64(len(block)) > s.context.nsqd.options.maxMessageSize {
util.ApiResponse(w, 500, "MSG_TOO_BIG", nil)
return
}
msg := nsq.NewMessage(<-s.context.nsqd.idChan, block)
msgs = append(msgs, msg)
}
err = topic.PutMessages(msgs)
if err != nil {
util.ApiResponse(w, 500, "NOK", nil)
return
}
w.Header().Set("Content-Length", "2")
io.WriteString(w, "OK")
}