func (s *FilterSuite) TestFilter(c *C) {
log.Println("testing Filter")
b, ch := test_utils.NewBlock("testingFilter", "filter")
go blocks.BlockRoutine(b)
ruleMsg := map[string]interface{}{"Filter": ".device == 'iPhone'"}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan}
queryOutChan := make(blocks.MsgChan)
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case messageI := <-queryOutChan:
c.Assert(messageI, DeepEquals, ruleMsg)
case message := <-outChan:
log.Println(message)
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
}
}
}
func (s *MovingAverageSuite) TestMovingAverage(c *C) {
loghub.Start()
log.Println("testing moving average")
b, ch := test_utils.NewBlock("testing movingaverave", "movingaverage")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
ruleMsg := map[string]interface{}{"Path": ".a", "Window": "4s"}
rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- rule
/*
m := map[string]string{"b": "test"}
arr := []interface{}{m}
inMsg := map[string]interface{}{"a": arr}
ch.InChan <- &blocks.Msg{Msg: inMsg, Route: "in"}
*/
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case <-outChan:
}
}
}
func (s *ParseXMLSuite) TestParseXML(c *C) {
log.Println("testing ParseXML")
b, ch := test_utils.NewBlock("testingParseXML", "parsexml")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
// where to find the xml in input
ruleMsg := map[string]interface{}{"Path": ".data"}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
queryOutChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
})
var xmldata = string(`
<?xml version="1.0" encoding="utf-8"?>
<OdfBody DocumentType="DT_GM" Date="20130131" Time="140807885" LogicalDate="20130131" Venue="ACV" Language="ENG" FeedFlag="P" DocumentCode="AS0ACV000" Version="3" Serial="1">
<Competition Code="OG2014">
<Config SDelay="60" />
</Competition>
</OdfBody>
`)
time.AfterFunc(time.Duration(2)*time.Second, func() {
xmlMsg := map[string]interface{}{"data": xmldata}
postData := &blocks.Msg{Msg: xmlMsg, Route: "in"}
ch.InChan <- postData
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("Rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
odfbody := message["OdfBody"].(map[string]interface{})
competition := odfbody["Competition"].(map[string]interface{})
c.Assert(odfbody["DocumentType"], Equals, "DT_GM")
c.Assert(competition["Code"], Equals, "OG2014")
}
}
}
func (s *FromPostSuite) TestFromPost(c *C) {
log.Println("testing FromPost")
b, ch := test_utils.NewBlock("testingPost", "frompost")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
inputMsg := map[string]interface{}{"Foo": "BAR"}
inputBlock := &blocks.Msg{Msg: inputMsg, Route: "in"}
ch.InChan <- inputBlock
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case <-outChan:
}
}
}
func (s *SetSuite) TestSet(c *C) {
loghub.Start()
log.Println("testing set")
b, ch := test_utils.NewBlock("testing set", "set")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
ruleMsg := map[string]interface{}{"Path": ".a"}
rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- rule
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case <-outChan:
}
}
}
func (s *PoissonSuite) TestPoisson(c *C) {
loghub.Start()
log.Println("testing Poisson")
b, ch := test_utils.NewBlock("testingPoisson", "poisson")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case <-outChan:
}
}
}
func (s *DeDupeSuite) TestDeDupe(c *C) {
loghub.Start()
log.Println("testing dedupe")
b, ch := test_utils.NewBlock("testing dedupe", "dedupe")
emittedValues := make(map[string]bool)
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
ruleMsg := map[string]interface{}{"Path": ".a"}
rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- rule
var sampleInput = map[string]interface{}{
"a": "foobar",
}
time.AfterFunc(time.Duration(2)*time.Second, func() {
postData := &blocks.Msg{Msg: sampleInput, Route: "in"}
ch.InChan <- postData
})
time.AfterFunc(time.Duration(1)*time.Second, func() {
postData := &blocks.Msg{Msg: sampleInput, Route: "in"}
ch.InChan <- postData
})
time.AfterFunc(time.Duration(1)*time.Second, func() {
postData := &blocks.Msg{Msg: map[string]interface{}{"a": "baz"}, Route: "in"}
ch.InChan <- postData
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
value := message["a"].(string)
_, ok := emittedValues[value]
if ok {
c.Errorf("block emitted a dupe message", value)
} else {
emittedValues[value] = true
}
}
}
}
func (s *GetHTTPSuite) TestGetHTTPXML(c *C) {
log.Println("testing GetHTTP with XML")
b, ch := test_utils.NewBlock("testingGetHTTPXML", "gethttp")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
ruleMsg := map[string]interface{}{"Path": ".url"}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
queryOutChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
})
time.AfterFunc(time.Duration(2)*time.Second, func() {
xmlMsg := map[string]interface{}{"url": "https://raw.github.com/nytlabs/streamtools/master/examples/odf.xml"}
postData := &blocks.Msg{Msg: xmlMsg, Route: "in"}
ch.InChan <- postData
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("Rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
messageData := message["data"].(string)
var xmldata = string(`<?xml version="1.0" encoding="utf-8"?>
<OdfBody DocumentType="DT_GM" Date="20130131" Time="140807885" LogicalDate="20130131" Venue="ACV" Language="ENG" FeedFlag="P" DocumentCode="AS0ACV000" Version="3" Serial="1">
<Competition Code="OG2014">
<Config SDelay="60" />
</Competition>
</OdfBody>
`)
c.Assert(messageData, Equals, xmldata)
}
}
}
func (s *SyncSuite) TestSync(c *C) {
loghub.Start()
log.Println("testing Sync")
b, ch := test_utils.NewBlock("testingSync", "sync")
go blocks.BlockRoutine(b)
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
err := <-ch.ErrChan
if err != nil {
c.Errorf(err.Error())
}
}
func (s *RedisSuite) TestRedisSMEMBERS(c *C) {
loghub.Start()
log.Println("testing Redis: SMEMBERS")
b, ch := test_utils.NewBlock("testingRedisSMEM", "redis")
go blocks.BlockRoutine(b)
m := []string{"'foobar'"}
ruleMsg := map[string]interface{}{"Server": "localhost:6379", "Password": "", "Command": "SMEMBERS", "Arguments": m}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan}
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{"tick": "123"}, Route: "in"}
queryOutChan := make(blocks.MsgChan)
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("Rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
expectedValue := []interface{}{"baz"}
c.Assert(message["response"], DeepEquals, expectedValue)
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
}
}
}
func (s *FromPostSuite) TestFromPostXML(c *C) {
log.Println("testing fromPost with XML")
b, ch := test_utils.NewBlock("testingFromPostXML", "frompost")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
var xmlstring = string(`
<?xml version="1.0" encoding="utf-8"?>
<OdfBody DocumentType="DT_GM" Date="20130131" Time="140807885" LogicalDate="20130131" Venue="ACV" Language="ENG" FeedFlag="P" DocumentCode="AS0ACV000" Version="3" Serial="1">
<Competition Code="OG2014">
<Config SDelay="60" />
</Competition>
</OdfBody>
`)
var xmldata = map[string]interface{}{
"data": xmlstring,
}
time.AfterFunc(time.Duration(2)*time.Second, func() {
postData := &blocks.Msg{Msg: xmldata, Route: "in"}
ch.InChan <- postData
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
messageXML := message["data"].(string)
c.Assert(messageXML, Equals, xmlstring)
}
}
}
func (s *GetHTTPSuite) TestGetHTTP(c *C) {
log.Println("testing GetHTTP")
b, ch := test_utils.NewBlock("testingGetHTTP", "gethttp")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
ruleMsg := map[string]interface{}{"Path": ".url"}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
queryOutChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
})
time.AfterFunc(time.Duration(2)*time.Second, func() {
nsqMsg := map[string]interface{}{"url": "https://raw.github.com/nytlabs/streamtools/master/examples/citibike.json"}
postData := &blocks.Msg{Msg: nsqMsg, Route: "in"}
ch.InChan <- postData
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("Rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case msg := <-outChan:
log.Println(msg)
}
}
}
func (s *NSQSuite) TestToNSQ(c *C) {
log.Println("testing toNSQ")
toB, toC := test_utils.NewBlock("testingToNSQ", "tonsq")
go blocks.BlockRoutine(toB)
ruleMsg := map[string]interface{}{"Topic": "librarytest", "NsqdTCPAddrs": "127.0.0.1:4150"}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
toC.InChan <- toRule
toQueryChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(1)*time.Second, func() {
toC.QueryChan <- &blocks.QueryMsg{MsgChan: toQueryChan, Route: "rule"}
})
outChan := make(chan *blocks.Msg)
toC.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan}
time.AfterFunc(time.Duration(2)*time.Second, func() {
nsqMsg := map[string]interface{}{"Foo": "Bar"}
postData := &blocks.Msg{Msg: nsqMsg, Route: "in"}
toC.InChan <- postData
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
toC.QuitChan <- true
})
for {
select {
case messageI := <-toQueryChan:
c.Assert(messageI, DeepEquals, ruleMsg)
case message := <-outChan:
log.Println("printing message from outChan:", message)
case err := <-toC.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
}
}
}
func (s *MaskSuite) TestMask(c *C) {
log.Println("testing Mask")
b, ch := test_utils.NewBlock("testingMask", "mask")
go blocks.BlockRoutine(b)
ruleMsg := map[string]interface{}{
"Mask": map[string]interface{}{
".foo": "{}",
},
}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan}
queryOutChan := make(blocks.MsgChan)
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("Rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case message := <-outChan:
log.Println(message)
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
}
}
}
func (s *ToFileSuite) TestToFile(c *C) {
loghub.Start()
log.Println("testing toFile")
b, ch := test_utils.NewBlock("testingToFile", "tofile")
go blocks.BlockRoutine(b)
ruleMsg := map[string]interface{}{"Filename": "foobar.log"}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan}
queryOutChan := make(blocks.MsgChan)
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
time.AfterFunc(time.Duration(5)*time.Second, func() {
err := os.Remove("foobar.log")
if err != nil {
c.Errorf(err.Error())
}
ch.QuitChan <- true
})
for {
select {
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
c.Fail()
}
case message := <-outChan:
log.Println(message)
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
}
}
}
func (s *NSQSuite) TestFromNSQ(c *C) {
log.Println("testing fromNSQ")
fromB, fromC := test_utils.NewBlock("testingfromNSQ", "fromnsq")
go blocks.BlockRoutine(fromB)
outChan := make(chan *blocks.Msg)
fromC.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan}
var maxInFlight float64 = 100
nsqSetup := map[string]interface{}{"ReadTopic": "librarytest", "LookupdAddr": "127.0.0.1:4161", "ReadChannel": "libtestchannel", "MaxInFlight": maxInFlight}
fromRule := &blocks.Msg{Msg: nsqSetup, Route: "rule"}
fromC.InChan <- fromRule
fromQueryChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(2)*time.Second, func() {
fromC.QueryChan <- &blocks.QueryMsg{MsgChan: fromQueryChan, Route: "rule"}
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
fromC.QuitChan <- true
})
for {
select {
case messageI := <-fromQueryChan:
c.Assert(messageI, DeepEquals, nsqSetup)
case message := <-outChan:
log.Println("printing message from outChan:", message)
case err := <-fromC.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
}
}
}
func (s *ToElasticsearchSuite) TestToElasticsearch(c *C) {
log.Println("testing ToElasticsearch")
b, ch := test_utils.NewBlock("testingToElasticsearch", "toelasticsearch")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
ruleMsg := map[string]interface{}{"Host": "localhost", "Port": "9200", "Index": "librarytest", "IndexType": "foobar"}
rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- rule
queryOutChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case <-outChan:
}
}
}
func (s *TickerSuite) TestTicker(c *C) {
loghub.Start()
log.Println("testing Ticker")
b, ch := test_utils.NewBlock("testingTicker", "ticker")
go blocks.BlockRoutine(b)
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan}
queryOutChan := make(blocks.MsgChan)
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
ruleMsg := map[string]interface{}{"Interval": "1s"}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("Rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
c.Assert(message["tick"], NotNil)
}
}
}
func (s *HistogramSuite) TestHistogram(c *C) {
log.Println("testing Histogram")
b, ch := test_utils.NewBlock("testingHistogram", "histogram")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
ruleMsg := map[string]interface{}{"Window": "10s", "Path": ".data"}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
queryOutChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
c.Fail()
}
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case <-outChan:
}
}
}
func (s *TimeseriesSuite) TestTimeseries(c *C) {
log.Println("testing Timeseries")
b, ch := test_utils.NewBlock("testingTimeseries", "timeseries")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
log.Println("out")
}
}
}
func (s *FromFileSuite) TestFromFile(c *C) {
log.Println("testing FromFile")
b, ch := test_utils.NewBlock("testingFile", "fromfile")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
f, err := ioutil.TempFile("", "streamtools_test_from_file.log")
if err != nil {
c.Errorf(err.Error())
}
defer syscall.Unlink(f.Name())
var fromfilestring = string(`{"Name": "Jacqui Maher", "Location": "Brooklyn", "Dog": "Conor S. Dogberst" }
{"Name": "Nik Hanselmann", "Location": "New York", "Dog": "None:(" }
{"Name": "Mike Dewar", "Location": "The Moon", "Dog": "Percy ? Dewar" }`)
ioutil.WriteFile(f.Name(), []byte(fromfilestring), 0644)
ruleMsg := map[string]interface{}{"Filename": f.Name()}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"}
})
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"}
})
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"}
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
var expectedNames = []string{"Jacqui Maher", "Nik Hanselmann", "Mike Dewar"}
var expectedLocations = []string{"Brooklyn", "New York", "The Moon"}
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
nameReceived, ok := message["Name"].(string)
if !ok {
log.Println("failed asserting message['Name'] to a string")
}
locationReceived, ok := message["Location"].(string)
if !ok {
log.Println("failed asserting message['Location'] to a string")
}
if !test_utils.StringInSlice(expectedNames, nameReceived) {
log.Println("failed finding", nameReceived, "in expected names list")
c.Fail()
}
if !test_utils.StringInSlice(expectedLocations, locationReceived) {
log.Println("failed finding", locationReceived, "in expected locations list")
c.Fail()
}
}
}
}
func (s *CacheSuite) TestCache(c *C) {
loghub.Start()
log.Println("testing cache")
b, ch := test_utils.NewBlock("testing cache", "cache")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
ruleMsg := map[string]interface{}{"KeyPath": ".name", "ValuePath": ".count", "TimeToLive": "1m"}
rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- rule
// Add some data to the cache
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{"count": "100", "name": "The New York Times"}, Route: "in"}
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{"count": "4", "name": "The New York Times"}, Route: "in"}
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{"count": "50", "name": "Hacks/Hackers"}, Route: "in"}
})
// Query for keys
keysChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(2)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: keysChan, Route: "keys"}
})
// And values
valuesChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(2)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: valuesChan, Route: "values"}
})
// And the entire cache contents
dumpChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(2)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: dumpChan, Route: "dump"}
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-keysChan:
message := messageI.(map[string]interface{})
keys := message["keys"].([]string)
c.Assert(keys, DeepEquals, []string{"The New York Times", "Hacks/Hackers"})
case messageI := <-valuesChan:
message := messageI.(map[string]interface{})
values := message["values"].([]interface{})
c.Assert(values, HasLen, 2)
case messageI := <-dumpChan:
message := messageI.(map[string]interface{})
c.Assert(message["dump"], HasLen, 2)
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
log.Println(message)
}
}
}
func (s *CountSuite) TestCount(c *C) {
log.Println("testing Count")
b, ch := test_utils.NewBlock("testingCount", "count")
go blocks.BlockRoutine(b)
ruleMsg := map[string]interface{}{"Window": "1s"}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
inMsgMsg := map[string]interface{}{}
inMsg := &blocks.Msg{Msg: inMsgMsg, Route: "in"}
ch.InChan <- inMsg
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan}
queryOutChan := make(blocks.MsgChan)
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
countChan := make(blocks.MsgChan)
ch.QueryChan <- &blocks.QueryMsg{MsgChan: countChan, Route: "count"}
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
pollMsg := map[string]interface{}{}
toPoll := &blocks.Msg{Msg: pollMsg, Route: "poll"}
ch.InChan <- toPoll
testOutput := map[string]interface{}{
"Count": 1.0,
}
for {
select {
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
c.Fail()
}
case messageI := <-countChan:
if !reflect.DeepEqual(messageI, testOutput) {
log.Println("count mismatch", messageI, testOutput)
c.Fail()
}
case messageI := <-outChan:
if !reflect.DeepEqual(messageI.Msg, testOutput) {
log.Println("poll mismatch", messageI.Msg, testOutput)
c.Fail()
}
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
}
}
}
func (s *FromSQSSuite) TestFromSQS(c *C) {
loghub.Start()
log.Println("testing FromSQS")
sampleResponse := string(`
<CreateQueueResponse
xmlns="http://sqs.us-east-1.amazonaws.com/doc/2012-11-05/"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:type="CreateQueueResponse">
<CreateQueueResult>
<QueueUrl>
http://sqs.us-east-1.amazonaws.com/770098461991/queue2
</QueueUrl>
</CreateQueueResult>
<ResponseMetadata>
<RequestId>cb919c0a-9bce-4afe-9b48-9bdf2412bb67</RequestId>
</ResponseMetadata>
</CreateQueueResponse>
`)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, sampleResponse)
}))
defer ts.Close()
b, ch := test_utils.NewBlock("testingFromSQS", "fromsqs")
go blocks.BlockRoutine(b)
ruleMsg := map[string]interface{}{"QueueName": "queue2", "AccessKey": "123access", "AccessSecret": "123secret", "MaxNumberOfMessages": "10"}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan}
queryOutChan := make(blocks.MsgChan)
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("Rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case message := <-outChan:
log.Println(message)
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
}
}
}
func (s *ParseCSVSuite) TestParseCSV(c *C) {
log.Println("testing ParseCSV")
b, ch := test_utils.NewBlock("testingParseCSV", "parsecsv")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
// where to find the xml in input
headers := []string{"name", "email", "phone"}
ruleMsg := map[string]interface{}{"Path": ".data", "Headers": headers}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
queryOutChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
})
var csvInput = `
Jacqui Maher, [email protected], 555-5550
Mike Dewar, [email protected], 555-5551
Nik Hanselmann, [email protected], 555-5552
Jane Friedoff, [email protected], 555-5553, Extra
`
time.AfterFunc(time.Duration(1)*time.Second, func() {
csvMsg := map[string]interface{}{"data": csvInput}
postData := &blocks.Msg{Msg: csvMsg, Route: "in"}
ch.InChan <- postData
})
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"}
})
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"}
})
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"}
})
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"}
})
time.AfterFunc(time.Duration(6)*time.Second, func() {
ch.QuitChan <- true
})
var expectedNames = []string{"Jacqui Maher", "Nik Hanselmann", "Mike Dewar", "Jane Friedoff"}
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("Rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
log.Println(message)
nameReceived := message["name"].(string)
if !test_utils.StringInSlice(expectedNames, nameReceived) {
log.Println("failed finding", nameReceived, "in expected names list")
c.Fail()
}
}
}
}
func (s *MapSuite) TestMap(c *C) {
log.Println("testing Map")
b, ch := test_utils.NewBlock("testingMap", "map")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan}
//outChan := make(chan *blocks.Msg)
//ch.AddChan <- &blocks.AddChanMsg{
// Route: "out",
// Channel: outChan,
//}
mapMsg := map[string]interface{}{"MegaBar": ".bar"}
ruleMsg := map[string]interface{}{"Map": mapMsg, "Additive": false}
// send rule
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
// send rule query in a sec
queryOutChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
})
// quit after 5
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
// send test input
inputMsg := map[string]interface{}{"bar": "something", "foo": "another thing"}
inputBlock := &blocks.Msg{Msg: inputMsg, Route: "in"}
time.AfterFunc(time.Duration(2)*time.Second, func() {
ch.InChan <- inputBlock
})
for {
select {
case messageI := <-queryOutChan:
message := messageI.(map[string]interface{})
if !reflect.DeepEqual(message["Map"], ruleMsg["Map"]) {
log.Println(messageI)
log.Println("was expecting", ruleMsg["Map"], "but got", message["Map"])
c.Fail()
}
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
log.Println(message)
c.Assert(message["MegaBar"], Equals, "something")
c.Assert(message["foo"], IsNil)
}
}
}
func (s *WebRequestSuite) TestWebRequestGet(c *C) {
log.Println("testing WebRequest: GET")
b, ch := test_utils.NewBlock("testingWebRequestGet", "webRequest")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
var okResponse interface{}
statusOk := bytes.NewBufferString(`{"Status": "OK"}`)
err := json.Unmarshal(statusOk.Bytes(), &okResponse)
if err != nil {
log.Println("unable to unmarshal json")
c.Fail()
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
fmt.Fprint(w, okResponse)
}))
defer ts.Close()
headers := map[string]interface{}{"Content-Type": "application/json"}
ruleMsg := map[string]interface{}{"Url": ts.URL, "UrlPath": "", "BodyPath": ".", "Method": "GET", "Headers": headers}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
queryOutChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
})
time.AfterFunc(time.Duration(2)*time.Second, func() {
foobarMsg := map[string]interface{}{"foo": "bar"}
postData := &blocks.Msg{Msg: foobarMsg, Route: "in"}
ch.InChan <- postData
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("Rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
messageHeaders := message["headers"].(http.Header)
c.Assert(message["body"], NotNil)
c.Assert(message["headers"], NotNil)
c.Assert(messageHeaders.Get("Content-Type"), Equals, "application/json; charset=utf-8")
c.Assert(message["status"], Equals, "200 OK")
}
}
}
func (s *WebRequestSuite) TestWebRequestGetXML(c *C) {
log.Println("testing WebRequest: GET with XML")
b, ch := test_utils.NewBlock("testingWebRequestGetXML", "webRequest")
go blocks.BlockRoutine(b)
outChan := make(chan *blocks.Msg)
ch.AddChan <- &blocks.AddChanMsg{
Route: "out",
Channel: outChan,
}
var xmldata = string(`
<?xml version="1.0" encoding="utf-8"?>
<OdfBody DocumentType="DT_GM" Date="20130131" Time="140807885" LogicalDate="20130131" Venue="ACV" Language="ENG" FeedFlag="P" DocumentCode="AS0ACV000" Version="3" Serial="1">
<Competition Code="OG2014">
<Config SDelay="60" />
</Competition>
</OdfBody>
`)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/xml; charset=utf-8")
fmt.Fprint(w, xmldata)
}))
defer ts.Close()
headers := map[string]interface{}{"Content-Type": "application/xml"}
ruleMsg := map[string]interface{}{"Url": ts.URL, "UrlPath": "", "BodyPath": ".", "Method": "GET", "Headers": headers}
toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"}
ch.InChan <- toRule
queryOutChan := make(blocks.MsgChan)
time.AfterFunc(time.Duration(1)*time.Second, func() {
ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"}
})
time.AfterFunc(time.Duration(2)*time.Second, func() {
emptyMsg := make(map[string]interface{})
postData := &blocks.Msg{Msg: emptyMsg, Route: "in"}
ch.InChan <- postData
})
time.AfterFunc(time.Duration(5)*time.Second, func() {
ch.QuitChan <- true
})
for {
select {
case err := <-ch.ErrChan:
if err != nil {
c.Errorf(err.Error())
} else {
return
}
case messageI := <-queryOutChan:
if !reflect.DeepEqual(messageI, ruleMsg) {
log.Println("Rule mismatch:", messageI, ruleMsg)
c.Fail()
}
case messageI := <-outChan:
message := messageI.Msg.(map[string]interface{})
messageHeaders := message["headers"].(http.Header)
c.Assert(message["body"], NotNil)
c.Assert(message["headers"], NotNil)
c.Assert(messageHeaders.Get("Content-Type"), Equals, "text/xml; charset=utf-8")
c.Assert(message["status"], Equals, "200 OK")
}
}
}
