func ScheduledSpec(c gospec.Context) {
scheduled := newScheduled(RETRY_KEY)
was := Config.namespace
Config.namespace = "prod:"
c.Specify("empties retry queues up to the current time", func() {
conn := Config.Pool.Get()
defer conn.Close()
now := time.Now().Unix()
message1, _ := NewMsg("{\"queue\":\"default\",\"foo\":\"bar1\"}")
message2, _ := NewMsg("{\"queue\":\"myqueue\",\"foo\":\"bar2\"}")
message3, _ := NewMsg("{\"queue\":\"default\",\"foo\":\"bar3\"}")
conn.Do("zadd", "prod:"+RETRY_KEY, now-60, message1.ToJson())
conn.Do("zadd", "prod:"+RETRY_KEY, now-10, message2.ToJson())
conn.Do("zadd", "prod:"+RETRY_KEY, now+60, message3.ToJson())
scheduled.poll(false)
defaultCount, _ := redis.Int(conn.Do("llen", "prod:queue:default"))
myqueueCount, _ := redis.Int(conn.Do("llen", "prod:queue:myqueue"))
pending, _ := redis.Int(conn.Do("zcard", "prod:"+RETRY_KEY))
c.Expect(defaultCount, Equals, 1)
c.Expect(myqueueCount, Equals, 1)
c.Expect(pending, Equals, 1)
})
Config.namespace = was
}
func ConfigSpec(c gospec.Context) {
var recoverOnPanic = func(f func()) (err interface{}) {
defer func() {
if cause := recover(); cause != nil {
err = cause
}
}()
f()
return
}
c.Specify("sets redis pool size which defaults to 1", func() {
c.Expect(Config.Pool.MaxIdle, Equals, 1)
Configure(map[string]string{
"server": "localhost:6379",
"process": "1",
"pool": "20",
})
c.Expect(Config.Pool.MaxIdle, Equals, 20)
})
c.Specify("can specify custom process", func() {
c.Expect(Config.processId, Equals, "1")
Configure(map[string]string{
"server": "localhost:6379",
"process": "2",
})
c.Expect(Config.processId, Equals, "2")
})
c.Specify("requires a server parameter", func() {
err := recoverOnPanic(func() {
Configure(map[string]string{"process": "2"})
})
c.Expect(err, Equals, "Configure requires a 'server' option, which identifies a Redis instance")
})
c.Specify("requires a process parameter", func() {
err := recoverOnPanic(func() {
Configure(map[string]string{"server": "localhost:6379"})
})
c.Expect(err, Equals, "Configure requires a 'process' option, which uniquely identifies this instance")
})
c.Specify("adds ':' to the end of the namespace", func() {
c.Expect(Config.namespace, Equals, "")
Configure(map[string]string{
"server": "localhost:6379",
"process": "1",
"namespace": "prod",
})
c.Expect(Config.namespace, Equals, "prod:")
})
}
func MiddlewareStatsSpec(c gospec.Context) {
var job = (func(args *Args) {
// noop
})
layout := "2006-01-02"
manager := newManager("myqueue", job, 1)
worker := newWorker(manager)
message, _ := NewMsg("{\"jid\":\"2\",\"retry\":true}")
c.Specify("increments processed stats", func() {
conn := Config.Pool.Get()
defer conn.Close()
count, _ := redis.Int(conn.Do("get", "stat:processed"))
dayCount, _ := redis.Int(conn.Do("get", "stat:processed:"+time.Now().UTC().Format(layout)))
c.Expect(count, Equals, 0)
c.Expect(dayCount, Equals, 0)
worker.process(message)
count, _ = redis.Int(conn.Do("get", "stat:processed"))
dayCount, _ = redis.Int(conn.Do("get", "stat:processed:"+time.Now().UTC().Format(layout)))
c.Expect(count, Equals, 1)
c.Expect(dayCount, Equals, 1)
})
c.Specify("failed job", func() {
var job = (func(args *Args) {
panic("AHHHH")
})
manager := newManager("myqueue", job, 1)
worker := newWorker(manager)
c.Specify("increments failed stats", func() {
conn := Config.Pool.Get()
defer conn.Close()
count, _ := redis.Int(conn.Do("get", "stat:failed"))
dayCount, _ := redis.Int(conn.Do("get", "stat:failed:"+time.Now().UTC().Format(layout)))
c.Expect(count, Equals, 0)
c.Expect(dayCount, Equals, 0)
worker.process(message)
count, _ = redis.Int(conn.Do("get", "stat:failed"))
dayCount, _ = redis.Int(conn.Do("get", "stat:failed:"+time.Now().UTC().Format(layout)))
c.Expect(count, Equals, 1)
c.Expect(dayCount, Equals, 1)
})
})
}
func ManagerSpec(c gospec.Context) {
processed := make(chan *Args)
testJob := (func(args *Args) {
processed <- args
})
c.Specify("newManager", func() {
c.Specify("sets queue with namespace", func() {
manager := newManager("myqueue", testJob, 10)
c.Expect(manager.queue, Equals, "queue:myqueue")
})
c.Specify("sets job function", func() {
manager := newManager("myqueue", testJob, 10)
c.Expect(fmt.Sprint(manager.job), Equals, fmt.Sprint(testJob))
})
c.Specify("sets worker concurrency", func() {
manager := newManager("myqueue", testJob, 10)
c.Expect(manager.concurrency, Equals, 10)
})
})
c.Specify("manage", func() {
conn := Config.pool.Get()
defer conn.Close()
message, _ := NewMsg("{\"foo\":\"bar\",\"args\":[\"foo\",\"bar\"]}")
message2, _ := NewMsg("{\"foo\":\"bar2\",\"args\":[\"foo\",\"bar2\"]}")
c.Specify("coordinates processing of queue messages", func() {
manager := newManager("manager1", testJob, 10)
conn.Do("lpush", "queue:manager1", message.ToJson())
conn.Do("lpush", "queue:manager1", message2.ToJson())
manager.start()
c.Expect(<-processed, Equals, message.Args())
c.Expect(<-processed, Equals, message2.Args())
manager.quit()
len, _ := redis.Int(conn.Do("llen", "queue:manager1"))
c.Expect(len, Equals, 0)
})
c.Specify("prepare stops fetching new messages from queue", func() {
manager := newManager("manager2", testJob, 10)
manager.start()
manager.prepare()
conn.Do("lpush", "queue:manager2", message)
conn.Do("lpush", "queue:manager2", message2)
manager.quit()
len, _ := redis.Int(conn.Do("llen", "queue:manager2"))
c.Expect(len, Equals, 2)
})
})
}
func ConnectionSpec(c gospec.Context) {
config := NewConfig("localhost:6379", "15", 2)
config.AddQueue("myqueue", ".*")
conn := NewConnection(config)
c.Specify("NewConnection", func() {
c.Specify("registers any queues defined in configuration", func() {
c.Expect(len(conn.Queues()), Equals, 1)
config.AddQueue("myqueue2", ".*")
conn.RegisterQueues()
c.Expect(len(conn.Queues()), Equals, 2)
})
c.Specify("stores registered queues in redis", func() {
r := config.Pool.Get()
defer r.Close()
values, _ := redis.Strings(r.Do("hgetall", "fairway:registered_queues"))
expected := []string{"myqueue", ".*"}
for i, str := range values {
c.Expect(str, Equals, expected[i])
}
})
})
c.Specify("Queues", func() {
c.Specify("returns a Queue for every currently registered queue", func() {
c.Expect(*conn.Queues()[0], Equals, *NewQueue(conn, "myqueue"))
})
})
c.Specify("Deliver", func() {
c.Specify("adds message to the facet for the queue", func() {
r := config.Pool.Get()
defer r.Close()
count, _ := redis.Int(r.Do("llen", "fairway:myqueue:default"))
c.Expect(count, Equals, 0)
msg, _ := NewMsg(map[string]string{"name": "mymessage"})
conn.Deliver(msg)
count, _ = redis.Int(r.Do("llen", "fairway:myqueue:default"))
c.Expect(count, Equals, 1)
value, _ := redis.String(r.Do("lindex", "fairway:myqueue:default", 0))
c.Expect(value, Equals, msg.json())
})
c.Specify("adds facets to the list of active facets", func() {
r := config.Pool.Get()
defer r.Close()
facets, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(facets), Equals, 0)
msg, _ := NewMsg(map[string]string{})
conn.Deliver(msg)
facets, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(facets), Equals, 1)
c.Expect(facets[0], Equals, "default")
})
c.Specify("pushes facet onto the facet queue", func() {
r := config.Pool.Get()
defer r.Close()
count, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(count, Equals, 0)
msg, _ := NewMsg(map[string]string{})
conn.Deliver(msg)
count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(count, Equals, 1)
value, _ := redis.String(r.Do("lindex", "fairway:myqueue:facet_queue", 0))
c.Expect(value, Equals, "default")
})
c.Specify("doesn't push facet if already active", func() {
r := config.Pool.Get()
defer r.Close()
r.Do("sadd", "fairway:myqueue:active_facets", "default")
r.Do("hset", "fairway:myqueue:facet_pool", "default", "1")
msg, _ := NewMsg(map[string]string{})
conn.Deliver(msg)
count, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(count, Equals, 0)
})
c.Specify("returns nil if delivery succeeds", func() {
msg, _ := NewMsg(map[string]string{})
err := conn.Deliver(msg)
c.Expect(err, IsNil)
})
c.Specify("returns error if delivery fails", func() {
config := NewConfig("localhost:9999", "15", 2)
conn := NewConnection(config)
msg, _ := NewMsg(map[string]string{})
err := conn.Deliver(msg)
c.Expect(err.Error(), Equals, "dial tcp 127.0.0.1:9999: connection refused")
})
})
}
func WorkerSpec(c gospec.Context) {
var processed = make(chan *Args)
middlewareCalled = false
var testJob = (func(args *Args) {
processed <- args
})
manager := newManager("myqueue", testJob, 1)
c.Specify("newWorker", func() {
c.Specify("it returns an instance of worker with connection to manager", func() {
worker := newWorker(manager)
c.Expect(worker.manager, Equals, manager)
})
})
c.Specify("work", func() {
worker := newWorker(manager)
messages := make(chan *Msg)
message, _ := NewMsg("{\"jid\":\"2309823\",\"args\":[\"foo\",\"bar\"]}")
c.Specify("calls job with message args", func() {
go worker.work(messages)
messages <- message
args, _ := (<-processed).Array()
<-manager.confirm
c.Expect(len(args), Equals, 2)
c.Expect(args[0], Equals, "foo")
c.Expect(args[1], Equals, "bar")
worker.quit()
})
c.Specify("confirms job completed", func() {
go worker.work(messages)
messages <- message
<-processed
c.Expect(<-manager.confirm, Equals, message)
worker.quit()
})
c.Specify("runs defined middleware", func() {
Middleware.Append(&testMiddleware{})
go worker.work(messages)
messages <- message
<-processed
<-manager.confirm
c.Expect(middlewareCalled, IsTrue)
worker.quit()
})
c.Specify("recovers and confirms if job panics", func() {
var panicJob = (func(args *Args) {
panic("AHHHHHHHHH")
})
manager := newManager("myqueue", panicJob, 1)
worker := newWorker(manager)
go worker.work(messages)
messages <- message
c.Expect(<-manager.confirm, Equals, message)
worker.quit()
})
})
}
func ChanneledConnectionSpec(c gospec.Context) {
config := NewConfig("localhost:6379", "15", 2)
config.AddQueue("myqueue", "typea")
config.AddQueue("myqueue2", "typeb")
conn := NewChanneledConnection(config, func(message *Msg) string {
channel, _ := message.Get("type").String()
return fmt.Sprint("channel:type", channel, ":channel")
})
c.Specify("Deliver", func() {
c.Specify("only queues up message for matching queues", func() {
r := config.Pool.Get()
defer r.Close()
count, _ := redis.Int(r.Do("llen", "fairway:myqueue:default"))
c.Expect(count, Equals, 0)
count, _ = redis.Int(r.Do("llen", "fairway:myqueue2:default"))
c.Expect(count, Equals, 0)
msg, _ := NewMsg(map[string]string{"type": "a"})
conn.Deliver(msg)
count, _ = redis.Int(r.Do("llen", "fairway:myqueue:default"))
c.Expect(count, Equals, 1)
count, _ = redis.Int(r.Do("llen", "fairway:myqueue2:default"))
c.Expect(count, Equals, 0)
})
})
c.Specify("DeliverBytes", func() {
c.Specify("only queues up message for matching queues", func() {
r := config.Pool.Get()
defer r.Close()
count, _ := redis.Int(r.Do("llen", "fairway:myqueue:default"))
c.Expect(count, Equals, 0)
count, _ = redis.Int(r.Do("llen", "fairway:myqueue2:default"))
c.Expect(count, Equals, 0)
msg, _ := NewMsg(map[string]string{"type": "a"})
conn.DeliverBytes("channel:typea:channel", "default", []byte(msg.json()))
count, _ = redis.Int(r.Do("llen", "fairway:myqueue:default"))
c.Expect(count, Equals, 1)
count, _ = redis.Int(r.Do("llen", "fairway:myqueue2:default"))
c.Expect(count, Equals, 0)
})
})
}
func WorkerSpec(c gospec.Context) {
var processed = make(chan *Args)
var testJob = (func(message *Msg) {
processed <- message.Args()
})
manager := newManager("myqueue", testJob, 1)
c.Specify("newWorker", func() {
c.Specify("it returns an instance of worker with connection to manager", func() {
worker := newWorker(manager)
c.Expect(worker.manager, Equals, manager)
})
})
c.Specify("work", func() {
worker := newWorker(manager)
messages := make(chan *Msg)
message, _ := NewMsg("{\"jid\":\"2309823\",\"args\":[\"foo\",\"bar\"]}")
c.Specify("calls job with message args", func() {
go worker.work(messages)
messages <- message
args, _ := (<-processed).Array()
<-manager.confirm
c.Expect(len(args), Equals, 2)
c.Expect(args[0], Equals, "foo")
c.Expect(args[1], Equals, "bar")
worker.quit()
})
c.Specify("confirms job completed", func() {
go worker.work(messages)
messages <- message
<-processed
c.Expect(confirm(manager), Equals, message)
worker.quit()
})
c.Specify("runs defined middleware and confirms", func() {
Middleware.Append(&testMiddleware{})
go worker.work(messages)
messages <- message
<-processed
c.Expect(confirm(manager), Equals, message)
c.Expect(testMiddlewareCalled, IsTrue)
worker.quit()
Middleware = NewMiddleware(
&MiddlewareLogging{},
&MiddlewareRetry{},
&MiddlewareStats{},
)
})
c.Specify("doesn't confirm if middleware cancels acknowledgement", func() {
Middleware.Append(&failMiddleware{})
go worker.work(messages)
messages <- message
<-processed
c.Expect(confirm(manager), IsNil)
c.Expect(failMiddlewareCalled, IsTrue)
worker.quit()
Middleware = NewMiddleware(
&MiddlewareLogging{},
&MiddlewareRetry{},
&MiddlewareStats{},
)
})
c.Specify("recovers and confirms if job panics", func() {
var panicJob = (func(message *Msg) {
panic("AHHHHHHHHH")
})
manager := newManager("myqueue", panicJob, 1)
worker := newWorker(manager)
go worker.work(messages)
messages <- message
c.Expect(confirm(manager), Equals, message)
worker.quit()
})
})
}
func EnqueueSpec(c gospec.Context) {
was := Config.Namespace
Config.Namespace = "prod:"
c.Specify("Enqueue", func() {
conn := Config.Pool.Get()
defer conn.Close()
c.Specify("makes the queue available", func() {
Enqueue("enqueue1", "Add", []int{1, 2})
found, _ := redis.Bool(conn.Do("sismember", "prod:queues", "enqueue1"))
c.Expect(found, IsTrue)
})
c.Specify("adds a job to the queue", func() {
nb, _ := redis.Int(conn.Do("llen", "prod:queue:enqueue2"))
c.Expect(nb, Equals, 0)
Enqueue("enqueue2", "Add", []int{1, 2})
nb, _ = redis.Int(conn.Do("llen", "prod:queue:enqueue2"))
c.Expect(nb, Equals, 1)
})
c.Specify("saves the arguments", func() {
Enqueue("enqueue3", "Compare", []string{"foo", "bar"})
bytes, _ := redis.Bytes(conn.Do("lpop", "prod:queue:enqueue3"))
var result map[string]interface{}
json.Unmarshal(bytes, &result)
c.Expect(result["class"], Equals, "Compare")
args := result["args"].([]interface{})
c.Expect(len(args), Equals, 2)
c.Expect(args[0], Equals, "foo")
c.Expect(args[1], Equals, "bar")
})
c.Specify("has a jid", func() {
Enqueue("enqueue4", "Compare", []string{"foo", "bar"})
bytes, _ := redis.Bytes(conn.Do("lpop", "prod:queue:enqueue4"))
var result map[string]interface{}
json.Unmarshal(bytes, &result)
c.Expect(result["class"], Equals, "Compare")
jid := result["jid"].(string)
c.Expect(len(jid), Equals, 24)
})
c.Specify("has enqueued_at that is close to now", func() {
Enqueue("enqueue5", "Compare", []string{"foo", "bar"})
bytes, _ := redis.Bytes(conn.Do("lpop", "prod:queue:enqueue5"))
var result map[string]interface{}
json.Unmarshal(bytes, &result)
c.Expect(result["class"], Equals, "Compare")
ea := result["enqueued_at"].(float64)
c.Expect(ea, Not(Equals), 0)
c.Expect(ea, IsWithin(0.1), nowToSecondsWithNanoPrecision())
})
c.Specify("has retry and retry_count when set", func() {
EnqueueWithOptions("enqueue6", "Compare", []string{"foo", "bar"}, EnqueueOptions{RetryCount: 1, Retry: 3})
bytes, _ := redis.Bytes(conn.Do("lpop", "prod:queue:enqueue6"))
var result map[string]interface{}
json.Unmarshal(bytes, &result)
c.Expect(result["class"], Equals, "Compare")
retry := result["retry"].(int)
c.Expect(retry, Equals, true)
retryCount := int(result["retry_count"].(float64))
c.Expect(retryCount, Equals, 13)
})
})
c.Specify("EnqueueIn", func() {
scheduleQueue := "prod:" + SCHEDULED_JOBS_KEY
conn := Config.Pool.Get()
defer conn.Close()
c.Specify("has added a job in the scheduled queue", func() {
_, err := EnqueueIn("enqueuein1", "Compare", 10, map[string]interface{}{"foo": "bar"})
c.Expect(err, Equals, nil)
scheduledCount, _ := redis.Int(conn.Do("zcard", scheduleQueue))
c.Expect(scheduledCount, Equals, 1)
conn.Do("del", scheduleQueue)
})
c.Specify("has the correct 'queue'", func() {
_, err := EnqueueIn("enqueuein2", "Compare", 10, map[string]interface{}{"foo": "bar"})
c.Expect(err, Equals, nil)
var data EnqueueData
elem, err := conn.Do("zrange", scheduleQueue, 0, -1)
bytes, err := redis.Bytes(elem.([]interface{})[0], err)
json.Unmarshal(bytes, &data)
c.Expect(data.Queue, Equals, "enqueuein2")
conn.Do("del", scheduleQueue)
})
})
Config.Namespace = was
}
func EnqueueSpec(c gospec.Context) {
was := Config.namespace
Config.namespace = "prod:"
c.Specify("Enqueue", func() {
conn := Config.Pool.Get()
defer conn.Close()
c.Specify("makes the queue available", func() {
Enqueue("enqueue1", "Add", []int{1, 2})
found, _ := redis.Bool(conn.Do("sismember", "prod:queues", "enqueue1"))
c.Expect(found, IsTrue)
})
c.Specify("adds a job to the queue", func() {
nb, _ := redis.Int(conn.Do("llen", "prod:queue:enqueue2"))
c.Expect(nb, Equals, 0)
Enqueue("enqueue2", "Add", []int{1, 2})
nb, _ = redis.Int(conn.Do("llen", "prod:queue:enqueue2"))
c.Expect(nb, Equals, 1)
})
c.Specify("saves the arguments", func() {
Enqueue("enqueue3", "Compare", []string{"foo", "bar"})
bytes, _ := redis.Bytes(conn.Do("lpop", "prod:queue:enqueue3"))
var result map[string]interface{}
json.Unmarshal(bytes, &result)
c.Expect(result["class"], Equals, "Compare")
args := result["args"].([]interface{})
c.Expect(len(args), Equals, 2)
c.Expect(args[0], Equals, "foo")
c.Expect(args[1], Equals, "bar")
})
})
Config.namespace = was
}
func QueueSpec(c gospec.Context) {
config := NewConfig("localhost:6379", "15", 2)
config.AddQueue("myqueue", ".*")
conn := NewConnection(config)
queue := NewQueue(conn, "myqueue")
c.Specify("NewQueue", func() {
})
c.Specify("Pull", func() {
c.Specify("pulls a message off the queue using FIFO", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
conn.Deliver(msg1)
conn.Deliver(msg2)
queueName, message := queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
queueName, message = queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg2.json())
})
c.Specify("places pulled message on inflight sorted set until acknowledged", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
conn.Deliver(msg1)
c.Expect(len(queue.Inflight()), Equals, 0)
queueName, message := queue.Pull(100)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
c.Expect(len(queue.Inflight()), Equals, 1)
c.Expect(queue.Inflight()[0], Equals, msg1.json())
queue.Ack(msg1)
c.Expect(len(queue.Inflight()), Equals, 0)
})
c.Specify("pulls from inflight message set if messages are unacknowledged", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
conn.Deliver(msg1)
conn.Deliver(msg2)
queueName, message := queue.Pull(0)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
queueName, message = queue.Pull(10)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
queueName, message = queue.Pull(10)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg2.json())
})
c.Specify("allows puller to ping to keep message inflight", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
conn.Deliver(msg1)
conn.Deliver(msg2)
queueName, message := queue.Pull(0)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
// Extends time before message is resent
queue.Ping(msg1, 10)
queueName, message = queue.Pull(10)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg2.json())
// Sets time for message to resend to now
queue.Ping(msg1, 0)
queueName, message = queue.Pull(10)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
})
c.Specify("set limits messages inflight", func() {
limit, err := queue.InflightLimit()
c.Expect(limit, Equals, 0)
c.Expect(err, IsNil)
queue.SetInflightLimit(1)
limit, err = queue.InflightLimit()
c.Expect(limit, Equals, 1)
c.Expect(err, IsNil)
})
c.Specify("limits messages inflight", func() {
r := config.Pool.Get()
defer r.Close()
config.Facet = func(msg *Msg) string {
str, _ := msg.Get("facet").String()
return str
}
msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
msg3, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage3"})
conn.Deliver(msg1)
conn.Deliver(msg2)
conn.Deliver(msg3)
queue.SetInflightLimit(1)
_, message := queue.Pull(2)
c.Expect(message.json(), Equals, msg1.json())
count, _ := redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
c.Expect(count, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg3.json())
count, _ = redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
c.Expect(count, Equals, 1)
count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
c.Expect(count, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
count, _ = redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
c.Expect(count, Equals, 1)
count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
c.Expect(count, Equals, 1)
queue.Ack(msg1)
queue.Ack(msg1)
queue.Ack(msg1)
queue.Ack(msg1)
queue.Ack(msg1)
count, err := redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
c.Expect(count, Equals, 0)
c.Expect(err.Error(), Equals, "redigo: nil returned")
count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
c.Expect(count, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg2.json())
count, _ = redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
c.Expect(count, Equals, 1)
count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
c.Expect(count, Equals, 1)
})
c.Specify("prevents overlimit messages when all messages are inflight", func() {
r := config.Pool.Get()
defer r.Close()
config.Facet = func(msg *Msg) string {
str, _ := msg.Get("facet").String()
return str
}
msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
msg3, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage3"})
queue.SetInflightLimit(1)
active, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
conn.Deliver(msg1)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message := queue.Pull(2)
c.Expect(message.json(), Equals, msg1.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
conn.Deliver(msg2)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
queue.Ack(msg1)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg2.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
conn.Deliver(msg3)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
queue.Ack(msg2)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg3.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
queue.Ack(msg3)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
msg4, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage4"})
conn.Deliver(msg4)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg4.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
queue.Ack(msg4)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
})
c.Specify("if inflight limit is 0, no limit", func() {
r := config.Pool.Get()
defer r.Close()
config.Facet = func(msg *Msg) string {
str, _ := msg.Get("facet").String()
return str
}
msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
msg3, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage3"})
queue.SetInflightLimit(0)
conn.Deliver(msg1)
conn.Deliver(msg2)
conn.Deliver(msg3)
active, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 2)
fqueue, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 2)
_, message := queue.Pull(2)
c.Expect(message.json(), Equals, msg1.json())
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg3.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg2.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
msg4, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage4"})
conn.Deliver(msg4)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
queue.Ack(msg1)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg4.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
queue.Ack(msg2)
queue.Ack(msg3)
queue.Ack(msg4)
})
c.Specify("doesn't place pulled message on inflight sorted set if inflight is disabled", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
conn.Deliver(msg1)
c.Expect(len(queue.Inflight()), Equals, 0)
queueName, message := queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
c.Expect(len(queue.Inflight()), Equals, 0)
})
c.Specify("doesn't pull from inflight message set if inflight is disabled", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
conn.Deliver(msg1)
conn.Deliver(msg2)
queueName, message := queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
queueName, message = queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg2.json())
})
c.Specify("pulls from facets of the queue in round-robin", func() {
r := config.Pool.Get()
defer r.Close()
config.Facet = func(msg *Msg) string {
str, _ := msg.Get("facet").String()
return str
}
msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
msg3, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage3"})
active, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
conn.Deliver(msg1)
conn.Deliver(msg2)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
conn.Deliver(msg3)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 2)
c.Expect(active[0], Equals, "1")
c.Expect(active[1], Equals, "2")
_, message := queue.Pull(-1)
c.Expect(message.json(), Equals, msg1.json())
_, message = queue.Pull(-1)
c.Expect(message.json(), Equals, msg3.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
_, message = queue.Pull(-1)
c.Expect(message.json(), Equals, msg2.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
})
c.Specify("removes facet from active list if it becomes empty", func() {
r := config.Pool.Get()
defer r.Close()
msg, _ := NewMsg(map[string]string{})
conn.Deliver(msg)
count, _ := redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
c.Expect(count, Equals, 1)
queue.Pull(-1)
count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
c.Expect(count, Equals, 0)
})
c.Specify("returns nil if there are no messages to receive", func() {
msg, _ := NewMsg(map[string]string{})
conn.Deliver(msg)
queueName, message := queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
queueName, message = queue.Pull(-1)
c.Expect(queueName, Equals, "")
c.Expect(message, IsNil)
})
})
}
func MiddlewareSpec(c gospec.Context) {
middleware := NewMiddleware()
message, _ := NewMsg("{\"foo\":\"bar\"}")
order = make([]string, 0)
first := &m1{}
second := &m2{}
c.Specify("newMiddleware", func() {
c.Specify("doesn't contain any middleware", func() {
c.Expect(len(middleware.actions), Equals, 0)
})
c.Specify("can specify middleware when initializing", func() {
middleware = NewMiddleware(first, second)
c.Expect(middleware.actions[0], Equals, first)
c.Expect(middleware.actions[1], Equals, second)
})
})
c.Specify("Append", func() {
c.Specify("adds function at the end of the list", func() {
middleware.Append(first)
middleware.Append(second)
middleware.call("myqueue", message, func() {
order = append(order, "job")
})
c.Expect(
arrayCompare(order, []string{
"m1 enter",
"m2 enter",
"job",
"m2 leave",
"m1 leave",
}),
IsTrue,
)
})
})
c.Specify("{repend", func() {
c.Specify("adds function at the beginning of the list", func() {
middleware.Prepend(first)
middleware.Prepend(second)
middleware.call("myqueue", message, func() {
order = append(order, "job")
})
c.Expect(
arrayCompare(order, []string{
"m2 enter",
"m1 enter",
"job",
"m1 leave",
"m2 leave",
}),
IsTrue,
)
})
})
}
func ConfigSpec(c gospec.Context) {
config := NewConfig("localhost:6379", "15", 10)
c.Specify("NewConfig", func() {
c.Specify("namespace is fairway", func() {
c.Expect(config.Namespace, Equals, "fairway")
})
c.Specify("sets the facet to always return 'default'", func() {
msg, _ := NewMsg(make([]string, 0))
c.Expect(config.Facet(msg), Equals, "default")
})
c.Specify("doesn't have any defined queues", func() {
c.Expect(len(config.queues), Equals, 0)
})
})
c.Specify("sets redis pool size", func() {
c.Expect(config.Pool.MaxIdle, Equals, 10)
c.Expect(config.Pool.MaxActive, Equals, 0)
config = NewConfig("localhost:6379", "15", 20)
c.Expect(config.Pool.MaxIdle, Equals, 20)
c.Expect(config.Pool.MaxActive, Equals, 0)
})
c.Specify("can specify custom namespace", func() {
config.Namespace = "mynamespace"
c.Expect(config.Namespace, Equals, "mynamespace")
})
c.Specify("can specify custom facet", func() {
config.Facet = func(message *Msg) string {
return "myfacet"
}
msg, _ := NewMsg(make([]string, 0))
c.Expect(config.Facet(msg), Equals, "myfacet")
})
c.Specify("can define a queue", func() {
config.AddQueue("myqueue", "default")
c.Expect(len(config.queues), Equals, 1)
c.Expect(*config.queues[0], Equals, QueueDefinition{"myqueue", "default"})
})
}
func MsgSpec(c gospec.Context) {
c.Specify("NewMsg", func() {
c.Specify("returns a new message with body as the content", func() {
msg, _ := NewMsg(map[string]string{"hello": "world"})
c.Expect(msg.json(), Equals, "{\"hello\":\"world\"}")
})
c.Specify("returns err if couldn't convert object", func() {
msg, err := NewMsg(func() {})
c.Expect(msg, IsNil)
c.Expect(err, Not(IsNil))
})
})
c.Specify("NewMsgFromString", func() {
c.Specify("returns a new message with string as the content", func() {
msg, _ := NewMsgFromString("{\"hello\":\"world\"}")
c.Expect(msg.json(), Equals, "{\"hello\":\"world\"}")
})
c.Specify("returns err if couldn't convert string", func() {
msg, err := NewMsgFromString("not json")
c.Expect(msg, IsNil)
c.Expect(err, Not(IsNil))
})
})
}
func MiddlewareRetrySpec(c gospec.Context) {
var panicingJob = (func(args *Args) {
panic("AHHHH")
})
var wares = newMiddleware(
&MiddlewareRetry{},
)
layout := "2006-01-02 15:04:05 MST"
manager := newManager("myqueue", panicingJob, 1)
worker := newWorker(manager)
c.Specify("puts messages in retry queue when they fail", func() {
message, _ := NewMsg("{\"jid\":\"2\",\"retry\":true}")
wares.call("myqueue", message, func() {
worker.process(message)
})
conn := Config.Pool.Get()
defer conn.Close()
retries, _ := redis.Strings(conn.Do("zrange", RETRY_KEY, 0, 1))
c.Expect(retries[0], Equals, message.ToJson())
})
c.Specify("allows disabling retries", func() {
message, _ := NewMsg("{\"jid\":\"2\",\"retry\":false}")
wares.call("myqueue", message, func() {
worker.process(message)
})
conn := Config.Pool.Get()
defer conn.Close()
count, _ := redis.Int(conn.Do("zcard", RETRY_KEY))
c.Expect(count, Equals, 0)
})
c.Specify("doesn't retry by default", func() {
message, _ := NewMsg("{\"jid\":\"2\"}")
wares.call("myqueue", message, func() {
worker.process(message)
})
conn := Config.Pool.Get()
defer conn.Close()
count, _ := redis.Int(conn.Do("zcard", RETRY_KEY))
c.Expect(count, Equals, 0)
})
c.Specify("allows numeric retries", func() {
message, _ := NewMsg("{\"jid\":\"2\",\"retry\":5}")
wares.call("myqueue", message, func() {
worker.process(message)
})
conn := Config.Pool.Get()
defer conn.Close()
retries, _ := redis.Strings(conn.Do("zrange", RETRY_KEY, 0, 1))
c.Expect(retries[0], Equals, message.ToJson())
})
c.Specify("handles new failed message", func() {
message, _ := NewMsg("{\"jid\":\"2\",\"retry\":true}")
wares.call("myqueue", message, func() {
worker.process(message)
})
conn := Config.Pool.Get()
defer conn.Close()
retries, _ := redis.Strings(conn.Do("zrange", RETRY_KEY, 0, 1))
message, _ = NewMsg(retries[0])
queue, _ := message.Get("queue").String()
error_message, _ := message.Get("error_message").String()
error_class, _ := message.Get("error_class").String()
retry_count, _ := message.Get("retry_count").Int()
error_backtrace, _ := message.Get("error_backtrace").String()
failed_at, _ := message.Get("failed_at").String()
c.Expect(queue, Equals, "myqueue")
c.Expect(error_message, Equals, "AHHHH")
c.Expect(error_class, Equals, "")
c.Expect(retry_count, Equals, 0)
c.Expect(error_backtrace, Equals, "")
c.Expect(failed_at, Equals, time.Now().UTC().Format(layout))
})
c.Specify("handles recurring failed message", func() {
message, _ := NewMsg("{\"jid\":\"2\",\"retry\":true,\"queue\":\"default\",\"error_message\":\"bam\",\"failed_at\":\"2013-07-20 14:03:42 UTC\",\"retry_count\":10}")
wares.call("myqueue", message, func() {
worker.process(message)
})
conn := Config.Pool.Get()
defer conn.Close()
retries, _ := redis.Strings(conn.Do("zrange", RETRY_KEY, 0, 1))
message, _ = NewMsg(retries[0])
queue, _ := message.Get("queue").String()
error_message, _ := message.Get("error_message").String()
retry_count, _ := message.Get("retry_count").Int()
failed_at, _ := message.Get("failed_at").String()
retried_at, _ := message.Get("retried_at").String()
c.Expect(queue, Equals, "myqueue")
c.Expect(error_message, Equals, "AHHHH")
c.Expect(retry_count, Equals, 11)
c.Expect(failed_at, Equals, "2013-07-20 14:03:42 UTC")
c.Expect(retried_at, Equals, time.Now().UTC().Format(layout))
})
c.Specify("handles recurring failed message with customized max", func() {
message, _ := NewMsg("{\"jid\":\"2\",\"retry\":10,\"queue\":\"default\",\"error_message\":\"bam\",\"failed_at\":\"2013-07-20 14:03:42 UTC\",\"retry_count\":8}")
wares.call("myqueue", message, func() {
worker.process(message)
})
conn := Config.Pool.Get()
defer conn.Close()
retries, _ := redis.Strings(conn.Do("zrange", RETRY_KEY, 0, 1))
message, _ = NewMsg(retries[0])
queue, _ := message.Get("queue").String()
error_message, _ := message.Get("error_message").String()
retry_count, _ := message.Get("retry_count").Int()
failed_at, _ := message.Get("failed_at").String()
retried_at, _ := message.Get("retried_at").String()
c.Expect(queue, Equals, "myqueue")
c.Expect(error_message, Equals, "AHHHH")
c.Expect(retry_count, Equals, 9)
c.Expect(failed_at, Equals, "2013-07-20 14:03:42 UTC")
c.Expect(retried_at, Equals, time.Now().UTC().Format(layout))
})
c.Specify("doesn't retry after default number of retries", func() {
message, _ := NewMsg("{\"jid\":\"2\",\"retry\":true,\"retry_count\":25}")
wares.call("myqueue", message, func() {
worker.process(message)
})
conn := Config.Pool.Get()
defer conn.Close()
count, _ := redis.Int(conn.Do("zcard", RETRY_KEY))
c.Expect(count, Equals, 0)
})
c.Specify("doesn't retry after customized number of retries", func() {
message, _ := NewMsg("{\"jid\":\"2\",\"retry\":3,\"retry_count\":3}")
wares.call("myqueue", message, func() {
worker.process(message)
})
conn := Config.Pool.Get()
defer conn.Close()
count, _ := redis.Int(conn.Do("zcard", RETRY_KEY))
c.Expect(count, Equals, 0)
})
}
func FetchSpec(c gospec.Context) {
c.Specify("Config.Fetch", func() {
c.Specify("it returns an instance of fetch with queue", func() {
fetch := buildFetch("fetchQueue1")
c.Expect(fetch.Queue(), Equals, "queue:fetchQueue1")
fetch.Close()
})
})
c.Specify("Fetch", func() {
message, _ := NewMsg("{\"foo\":\"bar\"}")
c.Specify("it puts messages from the queues on the messages channel", func() {
fetch := buildFetch("fetchQueue2")
conn := Config.Pool.Get()
defer conn.Close()
conn.Do("lpush", "queue:fetchQueue2", message.ToJson())
message := <-fetch.Messages()
c.Expect(message, Equals, message)
len, _ := redis.Int(conn.Do("llen", "queue:fetchQueue2"))
c.Expect(len, Equals, 0)
fetch.Close()
})
c.Specify("places in progress messages on private queue", func() {
fetch := buildFetch("fetchQueue3")
conn := Config.Pool.Get()
defer conn.Close()
conn.Do("lpush", "queue:fetchQueue3", message.ToJson())
<-fetch.Messages()
len, _ := redis.Int(conn.Do("llen", "queue:fetchQueue3:1:inprogress"))
c.Expect(len, Equals, 1)
messages, _ := redis.Strings(conn.Do("lrange", "queue:fetchQueue3:1:inprogress", 0, -1))
c.Expect(messages[0], Equals, message.ToJson())
fetch.Close()
})
c.Specify("removes in progress message when acknowledged", func() {
fetch := buildFetch("fetchQueue4")
conn := Config.Pool.Get()
defer conn.Close()
conn.Do("lpush", "queue:fetchQueue4", message.ToJson())
<-fetch.Messages()
fetch.Acknowledge(message)
len, _ := redis.Int(conn.Do("llen", "queue:fetchQueue4:1:inprogress"))
c.Expect(len, Equals, 0)
fetch.Close()
})
c.Specify("removes in progress message when serialized differently", func() {
json := "{\"foo\":\"bar\",\"args\":[]}"
message, _ := NewMsg(json)
c.Expect(json, Not(Equals), message.ToJson())
fetch := buildFetch("fetchQueue5")
conn := Config.Pool.Get()
defer conn.Close()
conn.Do("lpush", "queue:fetchQueue5", json)
<-fetch.Messages()
fetch.Acknowledge(message)
len, _ := redis.Int(conn.Do("llen", "queue:fetchQueue5:1:inprogress"))
c.Expect(len, Equals, 0)
fetch.Close()
})
c.Specify("refires any messages left in progress from prior instance", func() {
message2, _ := NewMsg("{\"foo\":\"bar2\"}")
message3, _ := NewMsg("{\"foo\":\"bar3\"}")
conn := Config.Pool.Get()
defer conn.Close()
conn.Do("lpush", "queue:fetchQueue6:1:inprogress", message.ToJson())
conn.Do("lpush", "queue:fetchQueue6:1:inprogress", message2.ToJson())
conn.Do("lpush", "queue:fetchQueue6", message3.ToJson())
fetch := buildFetch("fetchQueue6")
c.Expect(<-fetch.Messages(), Equals, message2)
c.Expect(<-fetch.Messages(), Equals, message)
c.Expect(<-fetch.Messages(), Equals, message3)
fetch.Acknowledge(message)
fetch.Acknowledge(message2)
fetch.Acknowledge(message3)
len, _ := redis.Int(conn.Do("llen", "queue:fetchQueue6:1:inprogress"))
c.Expect(len, Equals, 0)
fetch.Close()
})
})
}
func WorkersSpec(c gospec.Context) {
c.Specify("Workers", func() {
c.Specify("allows running in tests", func() {
called = make(chan bool)
Process("myqueue", myJob, 10)
Start()
Enqueue("myqueue", "Add", []int{1, 2})
<-called
Quit()
})
// TODO make this test more deterministic, randomly locks up in travis.
//c.Specify("allows starting and stopping multiple times", func() {
// called = make(chan bool)
// Process("myqueue", myJob, 10)
// Start()
// Quit()
// Start()
// Enqueue("myqueue", "Add", []int{1, 2})
// <-called
// Quit()
//})
c.Specify("runs beforeStart hooks", func() {
hooks := []string{}
BeforeStart(func() {
hooks = append(hooks, "1")
})
BeforeStart(func() {
hooks = append(hooks, "2")
})
BeforeStart(func() {
hooks = append(hooks, "3")
})
Start()
c.Expect(reflect.DeepEqual(hooks, []string{"1", "2", "3"}), IsTrue)
Quit()
// Clear out global hooks variable
beforeStart = nil
})
c.Specify("runs beforeStart hooks", func() {
hooks := []string{}
DuringDrain(func() {
hooks = append(hooks, "1")
})
DuringDrain(func() {
hooks = append(hooks, "2")
})
DuringDrain(func() {
hooks = append(hooks, "3")
})
Start()
c.Expect(reflect.DeepEqual(hooks, []string{}), IsTrue)
Quit()
c.Expect(reflect.DeepEqual(hooks, []string{"1", "2", "3"}), IsTrue)
// Clear out global hooks variable
duringDrain = nil
})
})
}
func MsgSpec(c gospec.Context) {
c.Specify("NewMsg", func() {
c.Specify("unmarshals json", func() {
msg, _ := NewMsg("{\"hello\":\"world\",\"foo\":3}")
hello, _ := msg.Get("hello").String()
foo, _ := msg.Get("foo").Int()
c.Expect(hello, Equals, "world")
c.Expect(foo, Equals, 3)
})
c.Specify("returns an error if invalid json", func() {
msg, err := NewMsg("{\"hello:\"world\",\"foo\":3}")
c.Expect(msg, IsNil)
c.Expect(err, Not(IsNil))
})
})
c.Specify("Args", func() {
c.Specify("returns args key", func() {
msg, _ := NewMsg("{\"hello\":\"world\",\"args\":[\"foo\",\"bar\"]}")
c.Expect(msg.Args().ToJson(), Equals, "[\"foo\",\"bar\"]")
})
c.Specify("returns empty array if args key doesn't exist", func() {
msg, _ := NewMsg("{\"hello\":\"world\"}")
c.Expect(msg.Args().ToJson(), Equals, "[]")
})
})
}
func ManagerSpec(c gospec.Context) {
processed := make(chan *Args)
testJob := (func(message *Msg) {
processed <- message.Args()
})
was := Config.Namespace
Config.Namespace = "prod:"
c.Specify("newManager", func() {
c.Specify("sets queue with namespace", func() {
manager := newManager("myqueue", testJob, 10)
c.Expect(manager.queue, Equals, "prod:queue:myqueue")
})
c.Specify("sets job function", func() {
manager := newManager("myqueue", testJob, 10)
c.Expect(fmt.Sprint(manager.job), Equals, fmt.Sprint(testJob))
})
c.Specify("sets worker concurrency", func() {
manager := newManager("myqueue", testJob, 10)
c.Expect(manager.concurrency, Equals, 10)
})
c.Specify("no per-manager middleware means 'use global Middleware object'", func() {
manager := newManager("myqueue", testJob, 10)
c.Expect(manager.mids, Equals, Middleware)
})
c.Specify("per-manager middlewares create separate middleware chains", func() {
mid1 := customMid{[]string{}, "0"}
manager := newManager("myqueue", testJob, 10, &mid1)
c.Expect(manager.mids, Not(Equals), Middleware)
c.Expect(len(manager.mids.actions), Equals, len(Middleware.actions)+1)
})
})
c.Specify("manage", func() {
conn := Config.Pool.Get()
defer conn.Close()
message, _ := NewMsg("{\"foo\":\"bar\",\"args\":[\"foo\",\"bar\"]}")
message2, _ := NewMsg("{\"foo\":\"bar2\",\"args\":[\"foo\",\"bar2\"]}")
c.Specify("coordinates processing of queue messages", func() {
manager := newManager("manager1", testJob, 10)
conn.Do("lpush", "prod:queue:manager1", message.ToJson())
conn.Do("lpush", "prod:queue:manager1", message2.ToJson())
manager.start()
c.Expect(<-processed, Equals, message.Args())
c.Expect(<-processed, Equals, message2.Args())
manager.quit()
len, _ := redis.Int(conn.Do("llen", "prod:queue:manager1"))
c.Expect(len, Equals, 0)
})
c.Specify("per-manager middlwares are called separately, global middleware is called in each manager", func() {
mid1 := customMid{[]string{}, "1"}
mid2 := customMid{[]string{}, "2"}
mid3 := customMid{[]string{}, "3"}
oldMiddleware := Middleware
Middleware = NewMiddleware()
Middleware.Append(&mid1)
manager1 := newManager("manager1", testJob, 10)
manager2 := newManager("manager2", testJob, 10, &mid2)
manager3 := newManager("manager3", testJob, 10, &mid3)
conn.Do("lpush", "prod:queue:manager1", message.ToJson())
conn.Do("lpush", "prod:queue:manager2", message.ToJson())
conn.Do("lpush", "prod:queue:manager3", message.ToJson())
manager1.start()
manager2.start()
manager3.start()
<-processed
<-processed
<-processed
Middleware = oldMiddleware
c.Expect(
arrayCompare(mid1.Trace, []string{"11", "12", "11", "12", "11", "12"}),
IsTrue,
)
c.Expect(
arrayCompare(mid2.Trace, []string{"21", "22"}),
IsTrue,
)
c.Expect(
arrayCompare(mid3.Trace, []string{"31", "32"}),
IsTrue,
)
manager1.quit()
manager2.quit()
manager3.quit()
})
c.Specify("prepare stops fetching new messages from queue", func() {
manager := newManager("manager2", testJob, 10)
manager.start()
manager.prepare()
conn.Do("lpush", "prod:queue:manager2", message)
conn.Do("lpush", "prod:queue:manager2", message2)
manager.quit()
len, _ := redis.Int(conn.Do("llen", "prod:queue:manager2"))
c.Expect(len, Equals, 2)
})
})
Config.Namespace = was
}
func EnqueueSpec(c gospec.Context) {
was := Config.Namespace
Config.Namespace = "prod:"
c.Specify("Enqueue", func() {
conn := Config.Pool.Get()
defer conn.Close()
c.Specify("makes the queue available", func() {
Enqueue("enqueue1", "Add", []int{1, 2})
found, _ := redis.Bool(conn.Do("sismember", "prod:queues", "enqueue1"))
c.Expect(found, IsTrue)
})
c.Specify("adds a job to the queue", func() {
nb, _ := redis.Int(conn.Do("llen", "prod:queue:enqueue2"))
c.Expect(nb, Equals, 0)
Enqueue("enqueue2", "Add", []int{1, 2})
nb, _ = redis.Int(conn.Do("llen", "prod:queue:enqueue2"))
c.Expect(nb, Equals, 1)
})
c.Specify("saves the arguments", func() {
Enqueue("enqueue3", "Compare", []string{"foo", "bar"})
bytes, _ := redis.Bytes(conn.Do("lpop", "prod:queue:enqueue3"))
var result map[string]interface{}
json.Unmarshal(bytes, &result)
c.Expect(result["class"], Equals, "Compare")
args := result["args"].([]interface{})
c.Expect(len(args), Equals, 2)
c.Expect(args[0], Equals, "foo")
c.Expect(args[1], Equals, "bar")
})
c.Specify("has a jid", func() {
Enqueue("enqueue4", "Compare", []string{"foo", "bar"})
bytes, _ := redis.Bytes(conn.Do("lpop", "prod:queue:enqueue4"))
var result map[string]interface{}
json.Unmarshal(bytes, &result)
c.Expect(result["class"], Equals, "Compare")
jid := result["jid"].(string)
c.Expect(len(jid), Equals, 24)
})
c.Specify("has enqueued_at that is close to now", func() {
Enqueue("enqueue5", "Compare", []string{"foo", "bar"})
bytes, _ := redis.Bytes(conn.Do("lpop", "prod:queue:enqueue5"))
var result map[string]interface{}
json.Unmarshal(bytes, &result)
c.Expect(result["class"], Equals, "Compare")
ea := result["enqueued_at"].(float64)
c.Expect(ea, Not(Equals), 0)
c.Expect(ea, IsWithin(0.1), float64(time.Now().UnixNano())/1000000000)
})
c.Specify("has retry and retry_count when set", func() {
EnqueueWithOptions("enqueue6", "Compare", []string{"foo", "bar"}, EnqueueOptions{RetryCount: 13, Retry: true})
bytes, _ := redis.Bytes(conn.Do("lpop", "prod:queue:enqueue6"))
var result map[string]interface{}
json.Unmarshal(bytes, &result)
c.Expect(result["class"], Equals, "Compare")
retry := result["retry"].(bool)
c.Expect(retry, Equals, true)
retryCount := int(result["retry_count"].(float64))
c.Expect(retryCount, Equals, 13)
})
})
Config.Namespace = was
}