Esempio n. 1
0
func main() {
	if len(os.Args) != 2 {
		fmt.Println("Usage\ntail-f filename\n")
		os.Exit(1)
	}
	filename, err := hydra.Rel2Abs(os.Args[1])
	if err != nil {
		panic(err)
	}
	watcher, err := hydra.NewWatcher()
	if err != nil {
		panic(err)
	}
	messageCh, monitorCh := hydra.NewChannel()
	config := &hydra.ConfigLogfile{
		Tag:       "dummy",
		File:      filename,
		FieldName: "message",
	}
	inTail, err := hydra.NewInTail(config, watcher, messageCh, monitorCh)
	go watcher.Run()
	go inTail.Run()
	done := make(chan interface{})
	for {
		recordSet := <-messageCh
		for _, record := range recordSet.Records {
			b, ok := record.GetData("message")
			if ok {
				fmt.Println(string(b.([]byte)))
			}
		}
	}
	<-done
}
Esempio n. 2
0
func run(config *hydra.Config) {
	messageCh, monitorCh := hydra.NewChannel()

	if config.ReadBufferSize > 0 {
		hydra.ReadBufferSize = config.ReadBufferSize
		log.Println("[info] set ReadBufferSize", hydra.ReadBufferSize)
	}

	// start monitor server
	monitor, err := hydra.NewMonitor(config, monitorCh)
	if err != nil {
		log.Println("[error] Couldn't start monitor server.", err)
	} else {
		go monitor.Run()
	}

	// start out_forward
	outForward, err := hydra.NewOutForward(config.Servers, messageCh, monitorCh)
	if err != nil {
		log.Println("[error]", err)
	} else {
		outForward.RoundRobin = config.ServerRoundRobin
		if outForward.RoundRobin {
			log.Println("[info] ServerRoundRobin enabled")
		}
		go outForward.Run()
	}

	// start watcher && in_tail
	if len(config.Logs) > 0 {
		watcher, err := hydra.NewWatcher()
		if err != nil {
			log.Println("[error]", err)
		}
		for _, configLogfile := range config.Logs {
			tail, err := hydra.NewInTail(configLogfile, watcher, messageCh, monitorCh)
			if err != nil {
				log.Println("[error]", err)
			} else {
				go tail.Run()
			}
		}
		go watcher.Run()
	}

	// start in_forward
	if config.Receiver != nil {
		if runtime.GOMAXPROCS(0) < 2 {
			log.Println("[warning] When using Receiver, recommend to set GOMAXPROCS >= 2.")
		}
		inForward, err := hydra.NewInForward(config.Receiver, messageCh, monitorCh)
		if err != nil {
			log.Println("[error]", err)
		} else {
			go inForward.Run()
		}
	}
}
func TestTrailRegexp(t *testing.T) {
	tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
	file, _ := ioutil.TempFile(tmpdir, "logfile.")
	defer os.RemoveAll(tmpdir)

	reg := hydra.RegexpApache
	configLogfile := &hydra.ConfigLogfile{
		Tag:        "test",
		File:       file.Name(),
		Format:     hydra.FormatRegexp,
		Regexp:     &hydra.Regexp{Regexp: reg},
		FieldName:  "message",
		ConvertMap: hydra.NewConvertMap("size:integer,code:integer"),
		TimeParse:  true,
		TimeFormat: hydra.TimeFormatApache,
		TimeKey:    "time",
	}
	c := hydra.NewContext()
	watcher, err := hydra.NewWatcher()
	if err != nil {
		t.Error(err)
	}
	inTail, err := hydra.NewInTail(configLogfile, watcher)
	if err != nil {
		t.Error(err)
	}
	c.RunProcess(inTail)
	c.RunProcess(watcher)
	go func() {
		time.Sleep(1 * time.Second)
		fileWriter(t, file, RegexpLogs)
	}()

	i := 0
	for i < len(RegexpLogs) {
		recordSet := <-c.MessageCh
		if recordSet.Tag != "test" {
			t.Errorf("got %v\nwant %v", recordSet.Tag, "test")
		}
		for _, _record := range recordSet.Records {
			record := _record.(*fluent.TinyFluentRecord)
			d := record.GetAllData()
			e := RegexpParsed[i]
			if ts, ok := e["_time"]; ok {
				if ts.(time.Time).Unix() != record.Timestamp {
					t.Errorf("expected record[%d] timestamp %s got %s", i, ts, record.Timestamp)
				}
				delete(e, "_time")
			}
			if !reflect.DeepEqual(e, d) {
				t.Errorf("expected %#v got %#v", d, e)
			}
			i++
		}
	}
}
func TestTrailLTSV(t *testing.T) {
	tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
	file, _ := ioutil.TempFile(tmpdir, "logfile.")
	defer os.RemoveAll(tmpdir)

	configLogfile := &hydra.ConfigLogfile{
		Tag:        "test",
		File:       file.Name(),
		Format:     hydra.FormatLTSV,
		ConvertMap: hydra.NewConvertMap("foo:integer"),
		FieldName:  "message",
		TimeParse:  true,
		TimeFormat: hydra.DefaultTimeFormat,
		TimeKey:    hydra.DefaultTimeKey,
	}
	c := hydra.NewContext()
	watcher, err := hydra.NewWatcher()
	if err != nil {
		t.Error(err)
	}
	inTail, err := hydra.NewInTail(configLogfile, watcher)
	if err != nil {
		t.Error(err)
	}
	c.RunProcess(inTail)
	c.RunProcess(watcher)
	go func() {
		time.Sleep(1 * time.Second)
		fileWriter(t, file, LTSVLogs)
	}()

	i := 0
	for i < len(LTSVLogs) {
		recordSet := <-c.MessageCh
		if recordSet.Tag != "test" {
			t.Errorf("got %v\nwant %v", recordSet.Tag, "test")
		}
		for _, _record := range recordSet.Records {
			record := _record.(*fluent.TinyFluentRecord)
			if foo, _ := record.GetData("foo"); foo != LTSVParsed[i]["foo"] {
				t.Errorf("unexpected record %v", record)
			}
			if bar, _ := record.GetData("bar"); bar != LTSVParsed[i]["bar"] {
				t.Errorf("unexpected record %v", record)
			}
			if ts, ok := LTSVParsed[i]["_time"]; ok {
				if ts.(time.Time).Unix() != record.Timestamp {
					t.Errorf("expected timestamp %s got %s", ts, record.Timestamp)
				}
			}
			i++
		}
	}
}
func TestTrailJSON(t *testing.T) {
	tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
	file, _ := ioutil.TempFile(tmpdir, "logfile.")
	defer os.RemoveAll(tmpdir)

	configLogfile := &hydra.ConfigLogfile{
		Tag:        "test",
		File:       file.Name(),
		Format:     hydra.JSON,
		FieldName:  "message",
		ConvertMap: hydra.NewConvertMap("bar:integer"),
		TimeParse:  true,
		TimeFormat: hydra.DefaultTimeFormat,
		TimeKey:    hydra.DefaultTimeKey,
	}
	msgCh, monCh := hydra.NewChannel()
	watcher, err := hydra.NewWatcher()
	if err != nil {
		t.Error(err)
	}
	inTail, err := hydra.NewInTail(configLogfile, watcher, msgCh, monCh)
	if err != nil {
		t.Error(err)
	}
	go inTail.Run()
	go watcher.Run()
	go func() {
		time.Sleep(1 * time.Second)
		fileWriter(t, file, JSONLogs)
	}()

	i := 0
	for i < len(JSONLogs) {
		recordSet := <-msgCh
		if recordSet.Tag != "test" {
			t.Errorf("got %v\nwant %v", recordSet.Tag, "test")
		}
		for _, _record := range recordSet.Records {
			record := _record.(*fluent.TinyFluentRecord)
			if foo, _ := record.GetData("foo"); foo != JSONParsed[i]["foo"] {
				t.Errorf("unexpected record[%d] got:foo=%#v expected:%#v", i, foo, JSONParsed[i]["foo"])
			}
			if bar, _ := record.GetData("bar"); bar != JSONParsed[i]["bar"] {
				t.Errorf("unexpected record[%d] got:bar=%#v expected:%#v %#v", i, bar, JSONParsed[i]["bar"], record)
			}
			if ts, ok := JSONParsed[i]["_time"]; ok {
				if ts.(time.Time).Unix() != record.Timestamp {
					t.Errorf("expected record[%d] timestamp %s got %s", i, ts, record.Timestamp)
				}
			}
			i++
		}
	}
}
func TestTrail(t *testing.T) {
	hydra.ReadBufferSize = ReadBufferSizeForTest

	tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
	file, _ := ioutil.TempFile(tmpdir, "logfile.")
	defer os.RemoveAll(tmpdir)

	go fileWriter(t, file, Logs)

	configLogfile := &hydra.ConfigLogfile{
		Tag:       "test",
		File:      file.Name(),
		FieldName: "message",
	}
	c := hydra.NewContext()
	watcher, err := hydra.NewWatcher()
	if err != nil {
		t.Error(err)
	}
	inTail, err := hydra.NewInTail(configLogfile, watcher)
	if err != nil {
		t.Error(err)
	}
	c.RunProcess(inTail)
	c.RunProcess(watcher)

	resultCh := make(chan string)
	go reciever(t, c.MessageCh, "test", resultCh)

	recieved := <-resultCh
	sent := strings.Join(Logs, "")
	if recieved != sent {
		t.Errorf("sent logs and recieved logs is different. sent %d bytes, recieved %d bytes", len(sent), len(recieved))
		fmt.Print(sent)
		fmt.Print(recieved)
	}
}