func (s *httpServer) tombstoneTopicNode(req *http.Request, topicName string, node string) error { if !protocol.IsValidTopicName(topicName) { return errors.New("INVALID_TOPIC") } if node == "" { return errors.New("INVALID_NODE") } // tombstone the topic on all the lookupds for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses { nsqlookupdVersion, err := s.ci.GetVersion(addr) if err != nil { s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err) } uri := "tombstone_topic_producer" if !nsqlookupdVersion.LT(v1EndpointVersion) { uri = "topic/tombstone" } endpoint := fmt.Sprintf("http://%s/%s?topic=%s&node=%s", addr, uri, url.QueryEscape(topicName), url.QueryEscape(node)) s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint) err = http_api.POSTV1(endpoint) if err != nil { s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err) } } nsqdVersion, err := s.ci.GetVersion(node) if err != nil { s.ctx.nsqadmin.logf("ERROR: failed to get nsqd %s version - %s", node, err) } uri := "delete_topic" if !nsqdVersion.LT(v1EndpointVersion) { uri = "topic/delete" } // delete the topic on the producer endpoint := fmt.Sprintf("http://%s/%s?topic=%s", node, uri, url.QueryEscape(topicName)) s.ctx.nsqadmin.logf("NSQD: querying %s", endpoint) err = http_api.POSTV1(endpoint) if err != nil { s.ctx.nsqadmin.logf("ERROR: nsqd %s - %s", endpoint, err) } s.notifyAdminAction("tombstone_topic_producer", topicName, "", node, req) return nil }
func TestTombstoneRecover(t *testing.T) { opts := NewOptions() opts.Logger = newTestLogger(t) opts.TombstoneLifetime = 50 * time.Millisecond tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts) defer nsqlookupd.Exit() topicName := "tombstone_recover" topicName2 := topicName + "2" conn := mustConnectLookupd(t, tcpAddr) defer conn.Close() identify(t, conn, "ip.address", 5000, 5555, "fake-version") nsq.Register(topicName, "channel1").WriteTo(conn) _, err := nsq.ReadResponse(conn) equal(t, err, nil) nsq.Register(topicName2, "channel2").WriteTo(conn) _, err = nsq.ReadResponse(conn) equal(t, err, nil) endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555") err = http_api.POSTV1(endpoint) equal(t, err, nil) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) data, err := API(endpoint) equal(t, err, nil) producers, _ := data.Get("producers").Array() equal(t, len(producers), 0) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName2) data, err = API(endpoint) equal(t, err, nil) producers, _ = data.Get("producers").Array() equal(t, len(producers), 1) time.Sleep(75 * time.Millisecond) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) data, err = API(endpoint) equal(t, err, nil) producers, _ = data.Get("producers").Array() equal(t, len(producers), 1) }
func (s *httpServer) performVersionNegotiatedRequestsToNSQD( nsqlookupdAddrs []string, nsqdAddrs []string, deprecatedURI string, v1URI string, queryString string) { var err error // get producer structs in one set of up-front requests // so we can negotiate versions // // (this returns an empty list if there are no nsqlookupd configured) producerList, _ := s.ci.GetLookupdProducers(nsqlookupdAddrs) for _, addr := range nsqdAddrs { var nodeVer semver.Version uri := deprecatedURI producer := producerList.Search(addr) if producer != nil { nodeVer = producer.VersionObj } else { // we couldn't find the node in our list // so ask it for a version directly nodeVer, err = s.ci.GetVersion(addr) if err != nil { s.ctx.nsqadmin.logf("ERROR: failed to get nsqd %s version - %s", addr, err) } } if nodeVer.NE(semver.Version{}) && nodeVer.GTE(v1EndpointVersion) { uri = v1URI } endpoint := fmt.Sprintf("http://%s/%s?%s", addr, uri, queryString) s.ctx.nsqadmin.logf("NSQD: querying %s", endpoint) err := http_api.POSTV1(endpoint) if err != nil { s.ctx.nsqadmin.logf("ERROR: nsqd %s - %s", endpoint, err) continue } } }
func (s *httpServer) deleteChannel(req *http.Request, topicName string, channelName string) error { for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses { nsqlookupdVersion, err := s.ci.GetVersion(addr) if err != nil { s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err) } uri := "delete_channel" if !nsqlookupdVersion.LT(v1EndpointVersion) { uri = "channel/delete" } endpoint := fmt.Sprintf("http://%s/%s?topic=%s&channel=%s", addr, uri, url.QueryEscape(topicName), url.QueryEscape(channelName)) s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint) err = http_api.POSTV1(endpoint) if err != nil { s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err) continue } } producerList := s.getTopicProducerList(topicName) s.performVersionNegotiatedRequestsToNSQD( s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses, producerList.HTTPAddrs(), "delete_channel", "channel/delete", fmt.Sprintf("topic=%s&channel=%s", url.QueryEscape(topicName), url.QueryEscape(channelName))) s.notifyAdminAction("delete_channel", topicName, channelName, "", req) return nil }
func (s *httpServer) deleteTopic(req *http.Request, topicName string) error { // for topic removal, you need to get all the producers *first* producerList := s.getTopicProducerList(topicName) // remove the topic from all the lookupds for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses { nsqlookupdVersion, err := s.ci.GetVersion(addr) if err != nil { s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err) } uri := "delete_topic" if !nsqlookupdVersion.LT(v1EndpointVersion) { uri = "topic/delete" } endpoint := fmt.Sprintf("http://%s/%s?topic=%s", addr, uri, topicName) s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint) err = http_api.POSTV1(endpoint) if err != nil { s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err) continue } } s.performVersionNegotiatedRequestsToNSQD( s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses, producerList.HTTPAddrs(), "delete_topic", "topic/delete", fmt.Sprintf("topic=%s", url.QueryEscape(topicName))) s.notifyAdminAction("delete_topic", topicName, "", "", req) return nil }
func TestTombstonedNodes(t *testing.T) { opts := NewOptions() opts.Logger = newTestLogger(t) tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts) defer nsqlookupd.Exit() lookupdHTTPAddrs := []string{fmt.Sprintf("%s", httpAddr)} topicName := "inactive_nodes" conn := mustConnectLookupd(t, tcpAddr) defer conn.Close() identify(t, conn, "ip.address", 5000, 5555, "fake-version") nsq.Register(topicName, "channel1").WriteTo(conn) _, err := nsq.ReadResponse(conn) equal(t, err, nil) producers, _ := clusterinfo.New(nil).GetLookupdProducers(lookupdHTTPAddrs) equal(t, len(producers), 1) equal(t, len(producers[0].Topics), 1) equal(t, producers[0].Topics[0].Topic, topicName) equal(t, producers[0].Topics[0].Tombstoned, false) endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s", httpAddr, topicName, "ip.address:5555") err = http_api.POSTV1(endpoint) equal(t, err, nil) producers, _ = clusterinfo.New(nil).GetLookupdProducers(lookupdHTTPAddrs) equal(t, len(producers), 1) equal(t, len(producers[0].Topics), 1) equal(t, producers[0].Topics[0].Topic, topicName) equal(t, producers[0].Topics[0].Tombstoned, true) }
func (s *httpServer) createTopicChannel(req *http.Request, topicName string, channelName string) error { if !protocol.IsValidTopicName(topicName) { return errors.New("INVALID_TOPIC") } if len(channelName) > 0 && !protocol.IsValidChannelName(channelName) { return errors.New("INVALID_CHANNEL") } for _, addr := range s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses { nsqlookupdVersion, err := s.ci.GetVersion(addr) if err != nil { s.ctx.nsqadmin.logf("ERROR: failed to get nsqlookupd %s version - %s", addr, err) } uri := "create_topic" if !nsqlookupdVersion.LT(v1EndpointVersion) { uri = "topic/create" } endpoint := fmt.Sprintf("http://%s/%s?topic=%s", addr, uri, url.QueryEscape(topicName)) s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint) err = http_api.POSTV1(endpoint) if err != nil { s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err) continue } if len(channelName) > 0 { uri := "create_channel" if !nsqlookupdVersion.LT(v1EndpointVersion) { uri = "channel/create" } endpoint := fmt.Sprintf("http://%s/%s?topic=%s&channel=%s", addr, uri, url.QueryEscape(topicName), url.QueryEscape(channelName)) s.ctx.nsqadmin.logf("LOOKUPD: querying %s", endpoint) err := http_api.POSTV1(endpoint) if err != nil { s.ctx.nsqadmin.logf("ERROR: lookupd %s - %s", endpoint, err) continue } } } s.notifyAdminAction("create_topic", topicName, "", "", req) if len(channelName) > 0 { // TODO: we can remove this when we push new channel information from nsqlookupd -> nsqd producerList, _ := s.ci.GetLookupdTopicProducers(topicName, s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses) s.performVersionNegotiatedRequestsToNSQD( s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses, producerList.HTTPAddrs(), "create_channel", "channel/create", fmt.Sprintf("topic=%s&channel=%s", url.QueryEscape(topicName), url.QueryEscape(channelName))) s.notifyAdminAction("create_channel", topicName, channelName, "", req) } return nil }
func TestCluster(t *testing.T) { lopts := nsqlookupd.NewOptions() lopts.Logger = newTestLogger(t) lopts.BroadcastAddress = "127.0.0.1" _, _, lookupd := mustStartNSQLookupd(lopts) opts := NewOptions() opts.Logger = newTestLogger(t) opts.NSQLookupdTCPAddresses = []string{lookupd.RealTCPAddr().String()} opts.BroadcastAddress = "127.0.0.1" _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "cluster_test" + strconv.Itoa(int(time.Now().Unix())) hostname, err := os.Hostname() equal(t, err, nil) url := fmt.Sprintf("http://%s/topic/create?topic=%s", nsqd.RealHTTPAddr(), topicName) err = http_api.POSTV1(url) equal(t, err, nil) url = fmt.Sprintf("http://%s/channel/create?topic=%s&channel=ch", nsqd.RealHTTPAddr(), topicName) err = http_api.POSTV1(url) equal(t, err, nil) // allow some time for nsqd to push info to nsqlookupd time.Sleep(350 * time.Millisecond) endpoint := fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr()) data, err := API(endpoint) equal(t, err, nil) topicData := data.Get("topic:" + topicName + ":") producers, _ := topicData.Array() equal(t, len(producers), 1) producer := topicData.GetIndex(0) equal(t, producer.Get("hostname").MustString(), hostname) equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1") equal(t, producer.Get("tcp_port").MustInt(), nsqd.RealTCPAddr().Port) equal(t, producer.Get("tombstoned").MustBool(), false) channelData := data.Get("channel:" + topicName + ":ch") producers, _ = channelData.Array() equal(t, len(producers), 1) producer = topicData.GetIndex(0) equal(t, producer.Get("hostname").MustString(), hostname) equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1") equal(t, producer.Get("tcp_port").MustInt(), nsqd.RealTCPAddr().Port) equal(t, producer.Get("tombstoned").MustBool(), false) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName) data, err = API(endpoint) producers, _ = data.Get("producers").Array() equal(t, len(producers), 1) producer = data.Get("producers").GetIndex(0) equal(t, producer.Get("hostname").MustString(), hostname) equal(t, producer.Get("broadcast_address").MustString(), "127.0.0.1") equal(t, producer.Get("tcp_port").MustInt(), nsqd.RealTCPAddr().Port) channels, _ := data.Get("channels").Array() equal(t, len(channels), 1) channel := channels[0].(string) equal(t, channel, "ch") url = fmt.Sprintf("http://%s/topic/delete?topic=%s", nsqd.RealHTTPAddr(), topicName) err = http_api.POSTV1(url) equal(t, err, nil) // allow some time for nsqd to push info to nsqlookupd time.Sleep(350 * time.Millisecond) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName) data, err = API(endpoint) equal(t, err, nil) producers, _ = data.Get("producers").Array() equal(t, len(producers), 0) endpoint = fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr()) data, err = API(endpoint) equal(t, err, nil) producers, _ = data.Get("topic:" + topicName + ":").Array() equal(t, len(producers), 0) producers, _ = data.Get("channel:" + topicName + ":ch").Array() equal(t, len(producers), 0) }