func (s *scripts) ack(queueName string, facet string, message *Msg) error {
conn := s.config.Pool.Get()
defer conn.Close()
_, err := redis.Strings(s.ackScript.Do(conn, s.namespace(), queueName, facet, message.Original))
return err
}
func (s *scripts) ping(queueName string, message *Msg, wait int) error {
conn := s.config.Pool.Get()
defer conn.Close()
_, err := redis.Strings(s.pingScript.Do(conn, s.namespace(), int(time.Now().Unix()),
wait, queueName, message.Original))
return err
}
func (s *scripts) inflight(queueName string) []string {
conn := s.config.Pool.Get()
defer conn.Close()
result, err := redis.Strings(s.inflightScript.Do(conn, s.namespace(), queueName))
if err != nil {
return []string{}
}
return result
}
func (s *scripts) pull(queueName string, wait int) (string, *Msg) {
conn := s.config.Pool.Get()
defer conn.Close()
script := s.findScript(FairwayPull, 3)
result, err := redis.Strings(script.Do(conn, s.namespace(), int(time.Now().Unix()), wait, queueName))
if err != nil {
return "", nil
}
queue := result[0]
message, _ := NewMsgFromString(result[1])
return queue, message
}
func (s *scripts) registeredQueues() ([]string, error) {
conn := s.config.Pool.Get()
defer conn.Close()
return redis.Strings(conn.Do("hkeys", s.registeredQueuesKey()))
}
func ConnectionSpec(c gospec.Context) {
config := NewConfig("localhost:6379", "15", 2)
config.AddQueue("myqueue", ".*")
conn := NewConnection(config)
c.Specify("NewConnection", func() {
c.Specify("registers any queues defined in configuration", func() {
c.Expect(len(conn.Queues()), Equals, 1)
config.AddQueue("myqueue2", ".*")
conn.RegisterQueues()
c.Expect(len(conn.Queues()), Equals, 2)
})
c.Specify("stores registered queues in redis", func() {
r := config.Pool.Get()
defer r.Close()
values, _ := redis.Strings(r.Do("hgetall", "fairway:registered_queues"))
expected := []string{"myqueue", ".*"}
for i, str := range values {
c.Expect(str, Equals, expected[i])
}
})
})
c.Specify("Queues", func() {
c.Specify("returns a Queue for every currently registered queue", func() {
c.Expect(*conn.Queues()[0], Equals, *NewQueue(conn, "myqueue"))
})
})
c.Specify("Deliver", func() {
c.Specify("adds message to the facet for the queue", func() {
r := config.Pool.Get()
defer r.Close()
count, _ := redis.Int(r.Do("llen", "fairway:myqueue:default"))
c.Expect(count, Equals, 0)
msg, _ := NewMsg(map[string]string{"name": "mymessage"})
conn.Deliver(msg)
count, _ = redis.Int(r.Do("llen", "fairway:myqueue:default"))
c.Expect(count, Equals, 1)
value, _ := redis.String(r.Do("lindex", "fairway:myqueue:default", 0))
c.Expect(value, Equals, msg.json())
})
c.Specify("adds facets to the list of active facets", func() {
r := config.Pool.Get()
defer r.Close()
facets, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(facets), Equals, 0)
msg, _ := NewMsg(map[string]string{})
conn.Deliver(msg)
facets, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(facets), Equals, 1)
c.Expect(facets[0], Equals, "default")
})
c.Specify("pushes facet onto the facet queue", func() {
r := config.Pool.Get()
defer r.Close()
count, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(count, Equals, 0)
msg, _ := NewMsg(map[string]string{})
conn.Deliver(msg)
count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(count, Equals, 1)
value, _ := redis.String(r.Do("lindex", "fairway:myqueue:facet_queue", 0))
c.Expect(value, Equals, "default")
})
c.Specify("doesn't push facet if already active", func() {
r := config.Pool.Get()
defer r.Close()
r.Do("sadd", "fairway:myqueue:active_facets", "default")
r.Do("hset", "fairway:myqueue:facet_pool", "default", "1")
msg, _ := NewMsg(map[string]string{})
conn.Deliver(msg)
count, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(count, Equals, 0)
})
c.Specify("returns nil if delivery succeeds", func() {
msg, _ := NewMsg(map[string]string{})
err := conn.Deliver(msg)
c.Expect(err, IsNil)
})
c.Specify("returns error if delivery fails", func() {
config := NewConfig("localhost:9999", "15", 2)
conn := NewConnection(config)
msg, _ := NewMsg(map[string]string{})
err := conn.Deliver(msg)
c.Expect(err.Error(), Equals, "dial tcp 127.0.0.1:9999: connection refused")
})
})
}
func QueueSpec(c gospec.Context) {
config := NewConfig("localhost:6379", "15", 2)
config.AddQueue("myqueue", ".*")
conn := NewConnection(config)
queue := NewQueue(conn, "myqueue")
c.Specify("NewQueue", func() {
})
c.Specify("Pull", func() {
c.Specify("pulls a message off the queue using FIFO", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
conn.Deliver(msg1)
conn.Deliver(msg2)
queueName, message := queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
queueName, message = queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg2.json())
})
c.Specify("places pulled message on inflight sorted set until acknowledged", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
conn.Deliver(msg1)
c.Expect(len(queue.Inflight()), Equals, 0)
queueName, message := queue.Pull(100)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
c.Expect(len(queue.Inflight()), Equals, 1)
c.Expect(queue.Inflight()[0], Equals, msg1.json())
queue.Ack(msg1)
c.Expect(len(queue.Inflight()), Equals, 0)
})
c.Specify("pulls from inflight message set if messages are unacknowledged", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
conn.Deliver(msg1)
conn.Deliver(msg2)
queueName, message := queue.Pull(0)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
queueName, message = queue.Pull(10)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
queueName, message = queue.Pull(10)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg2.json())
})
c.Specify("allows puller to ping to keep message inflight", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
conn.Deliver(msg1)
conn.Deliver(msg2)
queueName, message := queue.Pull(0)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
// Extends time before message is resent
queue.Ping(msg1, 10)
queueName, message = queue.Pull(10)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg2.json())
// Sets time for message to resend to now
queue.Ping(msg1, 0)
queueName, message = queue.Pull(10)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
})
c.Specify("set limits messages inflight", func() {
limit, err := queue.InflightLimit()
c.Expect(limit, Equals, 0)
c.Expect(err, IsNil)
queue.SetInflightLimit(1)
limit, err = queue.InflightLimit()
c.Expect(limit, Equals, 1)
c.Expect(err, IsNil)
})
c.Specify("limits messages inflight", func() {
r := config.Pool.Get()
defer r.Close()
config.Facet = func(msg *Msg) string {
str, _ := msg.Get("facet").String()
return str
}
msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
msg3, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage3"})
conn.Deliver(msg1)
conn.Deliver(msg2)
conn.Deliver(msg3)
queue.SetInflightLimit(1)
_, message := queue.Pull(2)
c.Expect(message.json(), Equals, msg1.json())
count, _ := redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
c.Expect(count, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg3.json())
count, _ = redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
c.Expect(count, Equals, 1)
count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
c.Expect(count, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
count, _ = redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
c.Expect(count, Equals, 1)
count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
c.Expect(count, Equals, 1)
queue.Ack(msg1)
queue.Ack(msg1)
queue.Ack(msg1)
queue.Ack(msg1)
queue.Ack(msg1)
count, err := redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
c.Expect(count, Equals, 0)
c.Expect(err.Error(), Equals, "redigo: nil returned")
count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
c.Expect(count, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg2.json())
count, _ = redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
c.Expect(count, Equals, 1)
count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
c.Expect(count, Equals, 1)
})
c.Specify("prevents overlimit messages when all messages are inflight", func() {
r := config.Pool.Get()
defer r.Close()
config.Facet = func(msg *Msg) string {
str, _ := msg.Get("facet").String()
return str
}
msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
msg3, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage3"})
queue.SetInflightLimit(1)
active, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
conn.Deliver(msg1)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message := queue.Pull(2)
c.Expect(message.json(), Equals, msg1.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
conn.Deliver(msg2)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
queue.Ack(msg1)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg2.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
conn.Deliver(msg3)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
queue.Ack(msg2)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg3.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
queue.Ack(msg3)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
msg4, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage4"})
conn.Deliver(msg4)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg4.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
queue.Ack(msg4)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
})
c.Specify("if inflight limit is 0, no limit", func() {
r := config.Pool.Get()
defer r.Close()
config.Facet = func(msg *Msg) string {
str, _ := msg.Get("facet").String()
return str
}
msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
msg3, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage3"})
queue.SetInflightLimit(0)
conn.Deliver(msg1)
conn.Deliver(msg2)
conn.Deliver(msg3)
active, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 2)
fqueue, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 2)
_, message := queue.Pull(2)
c.Expect(message.json(), Equals, msg1.json())
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg3.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg2.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
msg4, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage4"})
conn.Deliver(msg4)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
queue.Ack(msg1)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 1)
_, message = queue.Pull(2)
c.Expect(message.json(), Equals, msg4.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
c.Expect(fqueue, Equals, 0)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
queue.Ack(msg2)
queue.Ack(msg3)
queue.Ack(msg4)
})
c.Specify("doesn't place pulled message on inflight sorted set if inflight is disabled", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
conn.Deliver(msg1)
c.Expect(len(queue.Inflight()), Equals, 0)
queueName, message := queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
c.Expect(len(queue.Inflight()), Equals, 0)
})
c.Specify("doesn't pull from inflight message set if inflight is disabled", func() {
msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
conn.Deliver(msg1)
conn.Deliver(msg2)
queueName, message := queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg1.json())
queueName, message = queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
c.Expect(message.json(), Equals, msg2.json())
})
c.Specify("pulls from facets of the queue in round-robin", func() {
r := config.Pool.Get()
defer r.Close()
config.Facet = func(msg *Msg) string {
str, _ := msg.Get("facet").String()
return str
}
msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
msg3, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage3"})
active, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
conn.Deliver(msg1)
conn.Deliver(msg2)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
conn.Deliver(msg3)
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 2)
c.Expect(active[0], Equals, "1")
c.Expect(active[1], Equals, "2")
_, message := queue.Pull(-1)
c.Expect(message.json(), Equals, msg1.json())
_, message = queue.Pull(-1)
c.Expect(message.json(), Equals, msg3.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 1)
c.Expect(active[0], Equals, "1")
_, message = queue.Pull(-1)
c.Expect(message.json(), Equals, msg2.json())
active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
c.Expect(len(active), Equals, 0)
_, message = queue.Pull(2)
c.Expect(message, IsNil)
})
c.Specify("removes facet from active list if it becomes empty", func() {
r := config.Pool.Get()
defer r.Close()
msg, _ := NewMsg(map[string]string{})
conn.Deliver(msg)
count, _ := redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
c.Expect(count, Equals, 1)
queue.Pull(-1)
count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
c.Expect(count, Equals, 0)
})
c.Specify("returns nil if there are no messages to receive", func() {
msg, _ := NewMsg(map[string]string{})
conn.Deliver(msg)
queueName, message := queue.Pull(-1)
c.Expect(queueName, Equals, "myqueue")
queueName, message = queue.Pull(-1)
c.Expect(queueName, Equals, "")
c.Expect(message, IsNil)
})
})
}