コード例 #1
0
func TestForwardFailOver(t *testing.T) {
	log.Println("---- TestForwardFailOver ----")
	counter := int64(0)

	primaryAddr, primaryCloser := runMockServer(t, "", &counter)
	close(primaryCloser) // shutdown primary server immediately
	sleep(1)
	primaryConfigServer := newConfigServer(primaryAddr)
	secondaryAddr, secondaryCloser := runMockServer(t, "", &counter)
	secondaryConfigServer := newConfigServer(secondaryAddr)
	configServers := []*hydra.ConfigServer{
		primaryConfigServer,
		secondaryConfigServer,
	}
	c := hydra.NewContext()
	outForward, err := hydra.NewOutForward(configServers)
	if err != nil {
		t.Error(err)
		return
	}
	c.RunProcess(outForward)

	sleep(1)

	recordSet := prepareRecordSet()
	c.MessageCh <- recordSet
	sleep(1)

	if n := atomic.LoadInt64(&counter); n != int64(len(TestMessageLines)) {
		t.Error("insufficient recieved messages. sent", len(TestMessageLines), "recieved", n)
	}
	c.Shutdown()
	close(secondaryCloser)
	sleep(1)
}
コード例 #2
0
func TestMonitorServer(t *testing.T) {
	config := &hydra.Config{
		Monitor: &hydra.ConfigMonitor{
			Host: "localhost",
			Port: 0,
		},
	}
	c := hydra.NewContext()
	monitor, _ := hydra.NewMonitor(config)
	c.RunProcess(monitor)

	expectedMessages := make(map[string]int64)
	expectedBytes := make(map[string]int64)
	tags := []string{"foo", "bar", "dummy.test"}
	for _, tag := range tags {
		for i := 1; i <= 100; i++ {
			m := rand.Int63n(10)
			b := rand.Int63n(2560)
			c.MonitorCh <- &hydra.SentStat{
				Tag:      tag,
				Messages: m,
				Bytes:    b,
			}
			expectedMessages[tag] += m
			expectedBytes[tag] += b
		}
	}
	sleep(1)

	resp, err := http.Get(fmt.Sprintf("http://%s/", monitor.Addr))
	if err != nil {
		t.Error(err)
	}
	defer resp.Body.Close()
	if ct := resp.Header.Get("Content-Type"); ct != "application/json" {
		t.Error("invalid content-type", ct)
	}
	body, _ := ioutil.ReadAll(resp.Body)
	js := bytes.NewReader(body)
	for tag, n := range expectedMessages {
		js.Seek(int64(0), os.SEEK_SET)
		var got int64
		scan.ScanJSON(js, "/sent/"+tag+"/messages", &got)
		if got != n {
			t.Errorf("/sent/%s/messages got %d expected %d", tag, got, n)
		}
	}
	for tag, n := range expectedBytes {
		js.Seek(int64(0), os.SEEK_SET)
		var got int64
		scan.ScanJSON(js, "/sent/"+tag+"/bytes", &got)
		if got != n {
			t.Errorf("/sent/%s/bytes got %d expected %d", tag, got, n)
		}
	}

	log.Println(string(body))
}
コード例 #3
0
func TestTrailRegexp(t *testing.T) {
	tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
	file, _ := ioutil.TempFile(tmpdir, "logfile.")
	defer os.RemoveAll(tmpdir)

	reg := hydra.RegexpApache
	configLogfile := &hydra.ConfigLogfile{
		Tag:        "test",
		File:       file.Name(),
		Format:     hydra.FormatRegexp,
		Regexp:     &hydra.Regexp{Regexp: reg},
		FieldName:  "message",
		ConvertMap: hydra.NewConvertMap("size:integer,code:integer"),
		TimeParse:  true,
		TimeFormat: hydra.TimeFormatApache,
		TimeKey:    "time",
	}
	c := hydra.NewContext()
	watcher, err := hydra.NewWatcher()
	if err != nil {
		t.Error(err)
	}
	inTail, err := hydra.NewInTail(configLogfile, watcher)
	if err != nil {
		t.Error(err)
	}
	c.RunProcess(inTail)
	c.RunProcess(watcher)
	go func() {
		time.Sleep(1 * time.Second)
		fileWriter(t, file, RegexpLogs)
	}()

	i := 0
	for i < len(RegexpLogs) {
		recordSet := <-c.MessageCh
		if recordSet.Tag != "test" {
			t.Errorf("got %v\nwant %v", recordSet.Tag, "test")
		}
		for _, _record := range recordSet.Records {
			record := _record.(*fluent.TinyFluentRecord)
			d := record.GetAllData()
			e := RegexpParsed[i]
			if ts, ok := e["_time"]; ok {
				if ts.(time.Time).Unix() != record.Timestamp {
					t.Errorf("expected record[%d] timestamp %s got %s", i, ts, record.Timestamp)
				}
				delete(e, "_time")
			}
			if !reflect.DeepEqual(e, d) {
				t.Errorf("expected %#v got %#v", d, e)
			}
			i++
		}
	}
}
コード例 #4
0
func TestTrailLTSV(t *testing.T) {
	tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
	file, _ := ioutil.TempFile(tmpdir, "logfile.")
	defer os.RemoveAll(tmpdir)

	configLogfile := &hydra.ConfigLogfile{
		Tag:        "test",
		File:       file.Name(),
		Format:     hydra.FormatLTSV,
		ConvertMap: hydra.NewConvertMap("foo:integer"),
		FieldName:  "message",
		TimeParse:  true,
		TimeFormat: hydra.DefaultTimeFormat,
		TimeKey:    hydra.DefaultTimeKey,
	}
	c := hydra.NewContext()
	watcher, err := hydra.NewWatcher()
	if err != nil {
		t.Error(err)
	}
	inTail, err := hydra.NewInTail(configLogfile, watcher)
	if err != nil {
		t.Error(err)
	}
	c.RunProcess(inTail)
	c.RunProcess(watcher)
	go func() {
		time.Sleep(1 * time.Second)
		fileWriter(t, file, LTSVLogs)
	}()

	i := 0
	for i < len(LTSVLogs) {
		recordSet := <-c.MessageCh
		if recordSet.Tag != "test" {
			t.Errorf("got %v\nwant %v", recordSet.Tag, "test")
		}
		for _, _record := range recordSet.Records {
			record := _record.(*fluent.TinyFluentRecord)
			if foo, _ := record.GetData("foo"); foo != LTSVParsed[i]["foo"] {
				t.Errorf("unexpected record %v", record)
			}
			if bar, _ := record.GetData("bar"); bar != LTSVParsed[i]["bar"] {
				t.Errorf("unexpected record %v", record)
			}
			if ts, ok := LTSVParsed[i]["_time"]; ok {
				if ts.(time.Time).Unix() != record.Timestamp {
					t.Errorf("expected timestamp %s got %s", ts, record.Timestamp)
				}
			}
			i++
		}
	}
}
コード例 #5
0
func TestInForward(t *testing.T) {
	config := &hydra.ConfigReceiver{
		Host:              "127.0.0.1",
		Port:              0,
		MaxBufferMessages: 1000,
	}
	c := hydra.NewContext()
	inForward, err := hydra.NewInForward(config)
	if err != nil {
		t.Error(err)
	}
	c.RunProcess(inForward)

	host, _port, _ := net.SplitHostPort(inForward.Addr.String())
	port, _ := strconv.Atoi(_port)
	logger, err := client.New(client.Config{
		FluentHost: host,
		FluentPort: port,
	})
	if err != nil {
		t.Error(err)
	}
	log.Println("logger", logger)
	defer logger.Close()

	tag := "myapp.access"
	for i := 0; i < 10; i++ {
		var data = map[string]interface{}{
			"foo":  "bar",
			"hoge": "hoge",
		}
		logger.Post(tag, data)
	}
	n := 0
RECEIVE:
	for {
		select {
		case <-c.MessageCh:
			n++
			continue
		case <-time.After(1 * time.Second):
			break RECEIVE
		}
	}
	if n != 10 {
		t.Errorf("arrived messages %d expected %d", n, 10)
	}
}
コード例 #6
0
func TestForwardReconnect(t *testing.T) {
	log.Println("---- TestForwardReconnect ----")
	counter := int64(0)

	addr, mockCloser := runMockServer(t, "", &counter)
	configServer := newConfigServer(addr)
	c := hydra.NewContext()
	outForward, err := hydra.NewOutForward([]*hydra.ConfigServer{configServer})
	if err != nil {
		t.Error(err)
	}
	c.RunProcess(outForward)

	recordSet := prepareRecordSet()
	c.MessageCh <- recordSet
	sleep(1)

	t.Log("notify shutdown mockServer")
	close(mockCloser)
	t.Log("waiting for shutdown complated 3 sec")
	sleep(3)

	t.Log("restarting mock server on same addr", addr)
	_, mockCloser = runMockServer(t, addr, &counter)
	sleep(1)
	c.MessageCh <- recordSet // Afeter unexpected server closing, first Write() will be succeeded and lost...
	sleep(1)
	c.MessageCh <- recordSet
	t.Log("waiting for reconnect & resend completed 5 sec")
	sleep(3)

	if n := atomic.LoadInt64(&counter); n != int64(len(TestMessageLines)*2) {
		t.Error("insufficient recieved messages. sent", len(TestMessageLines)*2, "recieved", n)
	}
	close(mockCloser)
	c.Shutdown()
	sleep(1)
}
コード例 #7
0
func TestTrail(t *testing.T) {
	hydra.ReadBufferSize = ReadBufferSizeForTest

	tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
	file, _ := ioutil.TempFile(tmpdir, "logfile.")
	defer os.RemoveAll(tmpdir)

	go fileWriter(t, file, Logs)

	configLogfile := &hydra.ConfigLogfile{
		Tag:       "test",
		File:      file.Name(),
		FieldName: "message",
	}
	c := hydra.NewContext()
	watcher, err := hydra.NewWatcher()
	if err != nil {
		t.Error(err)
	}
	inTail, err := hydra.NewInTail(configLogfile, watcher)
	if err != nil {
		t.Error(err)
	}
	c.RunProcess(inTail)
	c.RunProcess(watcher)

	resultCh := make(chan string)
	go reciever(t, c.MessageCh, "test", resultCh)

	recieved := <-resultCh
	sent := strings.Join(Logs, "")
	if recieved != sent {
		t.Errorf("sent logs and recieved logs is different. sent %d bytes, recieved %d bytes", len(sent), len(recieved))
		fmt.Print(sent)
		fmt.Print(recieved)
	}
}
コード例 #8
0
func TestForwardSingle(t *testing.T) {
	log.Println("---- TestForwardSingle ----")
	counter := int64(0)

	addr, mockCloser := runMockServer(t, "", &counter)
	configServer := newConfigServer(addr)
	c := hydra.NewContext()
	outForward, err := hydra.NewOutForward([]*hydra.ConfigServer{configServer})
	if err != nil {
		t.Error(err)
	}
	c.RunProcess(outForward)

	recordSet := prepareRecordSet()
	c.MessageCh <- recordSet
	sleep(3)

	if n := atomic.LoadInt64(&counter); n != int64(len(TestMessageLines)) {
		t.Error("insufficient recieved messages. sent", len(TestMessageLines), "recieved", n)
	}
	c.Shutdown()
	close(mockCloser)
	sleep(1)
}