func TestForwardFailOver(t *testing.T) {
log.Println("---- TestForwardFailOver ----")
counter := int64(0)
primaryAddr, primaryCloser := runMockServer(t, "", &counter)
close(primaryCloser) // shutdown primary server immediately
sleep(1)
primaryConfigServer := newConfigServer(primaryAddr)
secondaryAddr, secondaryCloser := runMockServer(t, "", &counter)
secondaryConfigServer := newConfigServer(secondaryAddr)
configServers := []*hydra.ConfigServer{
primaryConfigServer,
secondaryConfigServer,
}
c := hydra.NewContext()
outForward, err := hydra.NewOutForward(configServers)
if err != nil {
t.Error(err)
return
}
c.RunProcess(outForward)
sleep(1)
recordSet := prepareRecordSet()
c.MessageCh <- recordSet
sleep(1)
if n := atomic.LoadInt64(&counter); n != int64(len(TestMessageLines)) {
t.Error("insufficient recieved messages. sent", len(TestMessageLines), "recieved", n)
}
c.Shutdown()
close(secondaryCloser)
sleep(1)
}
func TestMonitorServer(t *testing.T) {
config := &hydra.Config{
Monitor: &hydra.ConfigMonitor{
Host: "localhost",
Port: 0,
},
}
c := hydra.NewContext()
monitor, _ := hydra.NewMonitor(config)
c.RunProcess(monitor)
expectedMessages := make(map[string]int64)
expectedBytes := make(map[string]int64)
tags := []string{"foo", "bar", "dummy.test"}
for _, tag := range tags {
for i := 1; i <= 100; i++ {
m := rand.Int63n(10)
b := rand.Int63n(2560)
c.MonitorCh <- &hydra.SentStat{
Tag: tag,
Messages: m,
Bytes: b,
}
expectedMessages[tag] += m
expectedBytes[tag] += b
}
}
sleep(1)
resp, err := http.Get(fmt.Sprintf("http://%s/", monitor.Addr))
if err != nil {
t.Error(err)
}
defer resp.Body.Close()
if ct := resp.Header.Get("Content-Type"); ct != "application/json" {
t.Error("invalid content-type", ct)
}
body, _ := ioutil.ReadAll(resp.Body)
js := bytes.NewReader(body)
for tag, n := range expectedMessages {
js.Seek(int64(0), os.SEEK_SET)
var got int64
scan.ScanJSON(js, "/sent/"+tag+"/messages", &got)
if got != n {
t.Errorf("/sent/%s/messages got %d expected %d", tag, got, n)
}
}
for tag, n := range expectedBytes {
js.Seek(int64(0), os.SEEK_SET)
var got int64
scan.ScanJSON(js, "/sent/"+tag+"/bytes", &got)
if got != n {
t.Errorf("/sent/%s/bytes got %d expected %d", tag, got, n)
}
}
log.Println(string(body))
}
func TestTrailRegexp(t *testing.T) {
tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
file, _ := ioutil.TempFile(tmpdir, "logfile.")
defer os.RemoveAll(tmpdir)
reg := hydra.RegexpApache
configLogfile := &hydra.ConfigLogfile{
Tag: "test",
File: file.Name(),
Format: hydra.FormatRegexp,
Regexp: &hydra.Regexp{Regexp: reg},
FieldName: "message",
ConvertMap: hydra.NewConvertMap("size:integer,code:integer"),
TimeParse: true,
TimeFormat: hydra.TimeFormatApache,
TimeKey: "time",
}
c := hydra.NewContext()
watcher, err := hydra.NewWatcher()
if err != nil {
t.Error(err)
}
inTail, err := hydra.NewInTail(configLogfile, watcher)
if err != nil {
t.Error(err)
}
c.RunProcess(inTail)
c.RunProcess(watcher)
go func() {
time.Sleep(1 * time.Second)
fileWriter(t, file, RegexpLogs)
}()
i := 0
for i < len(RegexpLogs) {
recordSet := <-c.MessageCh
if recordSet.Tag != "test" {
t.Errorf("got %v\nwant %v", recordSet.Tag, "test")
}
for _, _record := range recordSet.Records {
record := _record.(*fluent.TinyFluentRecord)
d := record.GetAllData()
e := RegexpParsed[i]
if ts, ok := e["_time"]; ok {
if ts.(time.Time).Unix() != record.Timestamp {
t.Errorf("expected record[%d] timestamp %s got %s", i, ts, record.Timestamp)
}
delete(e, "_time")
}
if !reflect.DeepEqual(e, d) {
t.Errorf("expected %#v got %#v", d, e)
}
i++
}
}
}
func TestTrailLTSV(t *testing.T) {
tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
file, _ := ioutil.TempFile(tmpdir, "logfile.")
defer os.RemoveAll(tmpdir)
configLogfile := &hydra.ConfigLogfile{
Tag: "test",
File: file.Name(),
Format: hydra.FormatLTSV,
ConvertMap: hydra.NewConvertMap("foo:integer"),
FieldName: "message",
TimeParse: true,
TimeFormat: hydra.DefaultTimeFormat,
TimeKey: hydra.DefaultTimeKey,
}
c := hydra.NewContext()
watcher, err := hydra.NewWatcher()
if err != nil {
t.Error(err)
}
inTail, err := hydra.NewInTail(configLogfile, watcher)
if err != nil {
t.Error(err)
}
c.RunProcess(inTail)
c.RunProcess(watcher)
go func() {
time.Sleep(1 * time.Second)
fileWriter(t, file, LTSVLogs)
}()
i := 0
for i < len(LTSVLogs) {
recordSet := <-c.MessageCh
if recordSet.Tag != "test" {
t.Errorf("got %v\nwant %v", recordSet.Tag, "test")
}
for _, _record := range recordSet.Records {
record := _record.(*fluent.TinyFluentRecord)
if foo, _ := record.GetData("foo"); foo != LTSVParsed[i]["foo"] {
t.Errorf("unexpected record %v", record)
}
if bar, _ := record.GetData("bar"); bar != LTSVParsed[i]["bar"] {
t.Errorf("unexpected record %v", record)
}
if ts, ok := LTSVParsed[i]["_time"]; ok {
if ts.(time.Time).Unix() != record.Timestamp {
t.Errorf("expected timestamp %s got %s", ts, record.Timestamp)
}
}
i++
}
}
}
func TestInForward(t *testing.T) {
config := &hydra.ConfigReceiver{
Host: "127.0.0.1",
Port: 0,
MaxBufferMessages: 1000,
}
c := hydra.NewContext()
inForward, err := hydra.NewInForward(config)
if err != nil {
t.Error(err)
}
c.RunProcess(inForward)
host, _port, _ := net.SplitHostPort(inForward.Addr.String())
port, _ := strconv.Atoi(_port)
logger, err := client.New(client.Config{
FluentHost: host,
FluentPort: port,
})
if err != nil {
t.Error(err)
}
log.Println("logger", logger)
defer logger.Close()
tag := "myapp.access"
for i := 0; i < 10; i++ {
var data = map[string]interface{}{
"foo": "bar",
"hoge": "hoge",
}
logger.Post(tag, data)
}
n := 0
RECEIVE:
for {
select {
case <-c.MessageCh:
n++
continue
case <-time.After(1 * time.Second):
break RECEIVE
}
}
if n != 10 {
t.Errorf("arrived messages %d expected %d", n, 10)
}
}
func TestForwardReconnect(t *testing.T) {
log.Println("---- TestForwardReconnect ----")
counter := int64(0)
addr, mockCloser := runMockServer(t, "", &counter)
configServer := newConfigServer(addr)
c := hydra.NewContext()
outForward, err := hydra.NewOutForward([]*hydra.ConfigServer{configServer})
if err != nil {
t.Error(err)
}
c.RunProcess(outForward)
recordSet := prepareRecordSet()
c.MessageCh <- recordSet
sleep(1)
t.Log("notify shutdown mockServer")
close(mockCloser)
t.Log("waiting for shutdown complated 3 sec")
sleep(3)
t.Log("restarting mock server on same addr", addr)
_, mockCloser = runMockServer(t, addr, &counter)
sleep(1)
c.MessageCh <- recordSet // Afeter unexpected server closing, first Write() will be succeeded and lost...
sleep(1)
c.MessageCh <- recordSet
t.Log("waiting for reconnect & resend completed 5 sec")
sleep(3)
if n := atomic.LoadInt64(&counter); n != int64(len(TestMessageLines)*2) {
t.Error("insufficient recieved messages. sent", len(TestMessageLines)*2, "recieved", n)
}
close(mockCloser)
c.Shutdown()
sleep(1)
}
func TestTrail(t *testing.T) {
hydra.ReadBufferSize = ReadBufferSizeForTest
tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
file, _ := ioutil.TempFile(tmpdir, "logfile.")
defer os.RemoveAll(tmpdir)
go fileWriter(t, file, Logs)
configLogfile := &hydra.ConfigLogfile{
Tag: "test",
File: file.Name(),
FieldName: "message",
}
c := hydra.NewContext()
watcher, err := hydra.NewWatcher()
if err != nil {
t.Error(err)
}
inTail, err := hydra.NewInTail(configLogfile, watcher)
if err != nil {
t.Error(err)
}
c.RunProcess(inTail)
c.RunProcess(watcher)
resultCh := make(chan string)
go reciever(t, c.MessageCh, "test", resultCh)
recieved := <-resultCh
sent := strings.Join(Logs, "")
if recieved != sent {
t.Errorf("sent logs and recieved logs is different. sent %d bytes, recieved %d bytes", len(sent), len(recieved))
fmt.Print(sent)
fmt.Print(recieved)
}
}
func TestForwardSingle(t *testing.T) {
log.Println("---- TestForwardSingle ----")
counter := int64(0)
addr, mockCloser := runMockServer(t, "", &counter)
configServer := newConfigServer(addr)
c := hydra.NewContext()
outForward, err := hydra.NewOutForward([]*hydra.ConfigServer{configServer})
if err != nil {
t.Error(err)
}
c.RunProcess(outForward)
recordSet := prepareRecordSet()
c.MessageCh <- recordSet
sleep(3)
if n := atomic.LoadInt64(&counter); n != int64(len(TestMessageLines)) {
t.Error("insufficient recieved messages. sent", len(TestMessageLines), "recieved", n)
}
c.Shutdown()
close(mockCloser)
sleep(1)
}