func TestConvertMap(t *testing.T) {
convertMap := hydra.NewConvertMap("user_id:integer,paid:bool,paid_user_amount:float,bar:integer,baz:integer")
data := map[string]interface{}{
"user_id": "12345",
"paid": "true",
"paid_user_amount": "1.234",
"foo": "45678",
"bar": float64(67890),
"baz": 98765,
}
convertMap.ConvertTypes(data)
if data["user_id"] != int64(12345) {
t.Errorf("convert integer failed")
}
if data["paid"] != true {
t.Errorf("convert bool failed")
}
if data["paid_user_amount"] != float64(1.234) {
t.Errorf("convert float failed %#v", data["paid_user_amount"])
}
if data["foo"] != "45678" {
t.Errorf("foo must be not converted %#v", data["foo"])
}
if data["bar"] != int64(67890) {
t.Errorf("bar must be not converted %#v", data["bar"])
}
if data["baz"] != int64(98765) {
t.Errorf("baz must be not converted %#v", data["baz"])
}
}
func TestTrailRegexp(t *testing.T) {
tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
file, _ := ioutil.TempFile(tmpdir, "logfile.")
defer os.RemoveAll(tmpdir)
reg := hydra.RegexpApache
configLogfile := &hydra.ConfigLogfile{
Tag: "test",
File: file.Name(),
Format: hydra.FormatRegexp,
Regexp: &hydra.Regexp{Regexp: reg},
FieldName: "message",
ConvertMap: hydra.NewConvertMap("size:integer,code:integer"),
TimeParse: true,
TimeFormat: hydra.TimeFormatApache,
TimeKey: "time",
}
c := hydra.NewContext()
watcher, err := hydra.NewWatcher()
if err != nil {
t.Error(err)
}
inTail, err := hydra.NewInTail(configLogfile, watcher)
if err != nil {
t.Error(err)
}
c.RunProcess(inTail)
c.RunProcess(watcher)
go func() {
time.Sleep(1 * time.Second)
fileWriter(t, file, RegexpLogs)
}()
i := 0
for i < len(RegexpLogs) {
recordSet := <-c.MessageCh
if recordSet.Tag != "test" {
t.Errorf("got %v\nwant %v", recordSet.Tag, "test")
}
for _, _record := range recordSet.Records {
record := _record.(*fluent.TinyFluentRecord)
d := record.GetAllData()
e := RegexpParsed[i]
if ts, ok := e["_time"]; ok {
if ts.(time.Time).Unix() != record.Timestamp {
t.Errorf("expected record[%d] timestamp %s got %s", i, ts, record.Timestamp)
}
delete(e, "_time")
}
if !reflect.DeepEqual(e, d) {
t.Errorf("expected %#v got %#v", d, e)
}
i++
}
}
}
func TestTrailLTSV(t *testing.T) {
tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
file, _ := ioutil.TempFile(tmpdir, "logfile.")
defer os.RemoveAll(tmpdir)
configLogfile := &hydra.ConfigLogfile{
Tag: "test",
File: file.Name(),
Format: hydra.FormatLTSV,
ConvertMap: hydra.NewConvertMap("foo:integer"),
FieldName: "message",
TimeParse: true,
TimeFormat: hydra.DefaultTimeFormat,
TimeKey: hydra.DefaultTimeKey,
}
c := hydra.NewContext()
watcher, err := hydra.NewWatcher()
if err != nil {
t.Error(err)
}
inTail, err := hydra.NewInTail(configLogfile, watcher)
if err != nil {
t.Error(err)
}
c.RunProcess(inTail)
c.RunProcess(watcher)
go func() {
time.Sleep(1 * time.Second)
fileWriter(t, file, LTSVLogs)
}()
i := 0
for i < len(LTSVLogs) {
recordSet := <-c.MessageCh
if recordSet.Tag != "test" {
t.Errorf("got %v\nwant %v", recordSet.Tag, "test")
}
for _, _record := range recordSet.Records {
record := _record.(*fluent.TinyFluentRecord)
if foo, _ := record.GetData("foo"); foo != LTSVParsed[i]["foo"] {
t.Errorf("unexpected record %v", record)
}
if bar, _ := record.GetData("bar"); bar != LTSVParsed[i]["bar"] {
t.Errorf("unexpected record %v", record)
}
if ts, ok := LTSVParsed[i]["_time"]; ok {
if ts.(time.Time).Unix() != record.Timestamp {
t.Errorf("expected timestamp %s got %s", ts, record.Timestamp)
}
}
i++
}
}
}
func TestTrailJSON(t *testing.T) {
tmpdir, _ := ioutil.TempDir(os.TempDir(), "hydra-test")
file, _ := ioutil.TempFile(tmpdir, "logfile.")
defer os.RemoveAll(tmpdir)
configLogfile := &hydra.ConfigLogfile{
Tag: "test",
File: file.Name(),
Format: hydra.JSON,
FieldName: "message",
ConvertMap: hydra.NewConvertMap("bar:integer"),
TimeParse: true,
TimeFormat: hydra.DefaultTimeFormat,
TimeKey: hydra.DefaultTimeKey,
}
msgCh, monCh := hydra.NewChannel()
watcher, err := hydra.NewWatcher()
if err != nil {
t.Error(err)
}
inTail, err := hydra.NewInTail(configLogfile, watcher, msgCh, monCh)
if err != nil {
t.Error(err)
}
go inTail.Run()
go watcher.Run()
go func() {
time.Sleep(1 * time.Second)
fileWriter(t, file, JSONLogs)
}()
i := 0
for i < len(JSONLogs) {
recordSet := <-msgCh
if recordSet.Tag != "test" {
t.Errorf("got %v\nwant %v", recordSet.Tag, "test")
}
for _, _record := range recordSet.Records {
record := _record.(*fluent.TinyFluentRecord)
if foo, _ := record.GetData("foo"); foo != JSONParsed[i]["foo"] {
t.Errorf("unexpected record[%d] got:foo=%#v expected:%#v", i, foo, JSONParsed[i]["foo"])
}
if bar, _ := record.GetData("bar"); bar != JSONParsed[i]["bar"] {
t.Errorf("unexpected record[%d] got:bar=%#v expected:%#v %#v", i, bar, JSONParsed[i]["bar"], record)
}
if ts, ok := JSONParsed[i]["_time"]; ok {
if ts.(time.Time).Unix() != record.Timestamp {
t.Errorf("expected record[%d] timestamp %s got %s", i, ts, record.Timestamp)
}
}
i++
}
}
}
func BenchmarkConvertMap(b *testing.B) {
convertMap := hydra.NewConvertMap("user_id:integer,paid:bool,paid_user_amount:float")
b.ResetTimer()
for i := 0; i < b.N; i++ {
data := map[string]interface{}{
"user_id": "12345",
"paid": "true",
"paid_user_amount": "1.234",
"foo": "45678",
}
convertMap.ConvertTypes(data)
}
}