Exemple #1
0
func (s *scripts) ack(queueName string, facet string, message *Msg) error {
	conn := s.config.Pool.Get()
	defer conn.Close()

	_, err := redis.Strings(s.ackScript.Do(conn, s.namespace(), queueName, facet, message.Original))

	return err
}
Exemple #2
0
func (s *scripts) ping(queueName string, message *Msg, wait int) error {
	conn := s.config.Pool.Get()
	defer conn.Close()

	_, err := redis.Strings(s.pingScript.Do(conn, s.namespace(), int(time.Now().Unix()),
		wait, queueName, message.Original))

	return err
}
Exemple #3
0
func (s *scripts) inflight(queueName string) []string {
	conn := s.config.Pool.Get()
	defer conn.Close()

	result, err := redis.Strings(s.inflightScript.Do(conn, s.namespace(), queueName))

	if err != nil {
		return []string{}
	}

	return result
}
Exemple #4
0
func (s *scripts) pull(queueName string, wait int) (string, *Msg) {
	conn := s.config.Pool.Get()
	defer conn.Close()

	script := s.findScript(FairwayPull, 3)

	result, err := redis.Strings(script.Do(conn, s.namespace(), int(time.Now().Unix()), wait, queueName))

	if err != nil {
		return "", nil
	}

	queue := result[0]
	message, _ := NewMsgFromString(result[1])

	return queue, message
}
Exemple #5
0
func (s *scripts) registeredQueues() ([]string, error) {
	conn := s.config.Pool.Get()
	defer conn.Close()
	return redis.Strings(conn.Do("hkeys", s.registeredQueuesKey()))
}
func ConnectionSpec(c gospec.Context) {
	config := NewConfig("localhost:6379", "15", 2)
	config.AddQueue("myqueue", ".*")
	conn := NewConnection(config)

	c.Specify("NewConnection", func() {
		c.Specify("registers any queues defined in configuration", func() {
			c.Expect(len(conn.Queues()), Equals, 1)
			config.AddQueue("myqueue2", ".*")
			conn.RegisterQueues()
			c.Expect(len(conn.Queues()), Equals, 2)
		})

		c.Specify("stores registered queues in redis", func() {
			r := config.Pool.Get()
			defer r.Close()

			values, _ := redis.Strings(r.Do("hgetall", "fairway:registered_queues"))

			expected := []string{"myqueue", ".*"}

			for i, str := range values {
				c.Expect(str, Equals, expected[i])
			}
		})
	})

	c.Specify("Queues", func() {
		c.Specify("returns a Queue for every currently registered queue", func() {
			c.Expect(*conn.Queues()[0], Equals, *NewQueue(conn, "myqueue"))
		})
	})

	c.Specify("Deliver", func() {
		c.Specify("adds message to the facet for the queue", func() {
			r := config.Pool.Get()
			defer r.Close()

			count, _ := redis.Int(r.Do("llen", "fairway:myqueue:default"))
			c.Expect(count, Equals, 0)

			msg, _ := NewMsg(map[string]string{"name": "mymessage"})

			conn.Deliver(msg)

			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:default"))
			c.Expect(count, Equals, 1)

			value, _ := redis.String(r.Do("lindex", "fairway:myqueue:default", 0))
			c.Expect(value, Equals, msg.json())
		})

		c.Specify("adds facets to the list of active facets", func() {
			r := config.Pool.Get()
			defer r.Close()

			facets, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(facets), Equals, 0)

			msg, _ := NewMsg(map[string]string{})

			conn.Deliver(msg)

			facets, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(facets), Equals, 1)
			c.Expect(facets[0], Equals, "default")
		})

		c.Specify("pushes facet onto the facet queue", func() {
			r := config.Pool.Get()
			defer r.Close()

			count, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(count, Equals, 0)

			msg, _ := NewMsg(map[string]string{})

			conn.Deliver(msg)

			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(count, Equals, 1)

			value, _ := redis.String(r.Do("lindex", "fairway:myqueue:facet_queue", 0))
			c.Expect(value, Equals, "default")
		})

		c.Specify("doesn't push facet if already active", func() {
			r := config.Pool.Get()
			defer r.Close()

			r.Do("sadd", "fairway:myqueue:active_facets", "default")
			r.Do("hset", "fairway:myqueue:facet_pool", "default", "1")

			msg, _ := NewMsg(map[string]string{})

			conn.Deliver(msg)

			count, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(count, Equals, 0)
		})

		c.Specify("returns nil if delivery succeeds", func() {
			msg, _ := NewMsg(map[string]string{})
			err := conn.Deliver(msg)
			c.Expect(err, IsNil)
		})

		c.Specify("returns error if delivery fails", func() {
			config := NewConfig("localhost:9999", "15", 2)
			conn := NewConnection(config)

			msg, _ := NewMsg(map[string]string{})
			err := conn.Deliver(msg)
			c.Expect(err.Error(), Equals, "dial tcp 127.0.0.1:9999: connection refused")
		})
	})
}
Exemple #7
0
func QueueSpec(c gospec.Context) {
	config := NewConfig("localhost:6379", "15", 2)
	config.AddQueue("myqueue", ".*")
	conn := NewConnection(config)
	queue := NewQueue(conn, "myqueue")

	c.Specify("NewQueue", func() {
	})

	c.Specify("Pull", func() {
		c.Specify("pulls a message off the queue using FIFO", func() {
			msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
			msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})

			conn.Deliver(msg1)
			conn.Deliver(msg2)

			queueName, message := queue.Pull(-1)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg1.json())

			queueName, message = queue.Pull(-1)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg2.json())
		})

		c.Specify("places pulled message on inflight sorted set until acknowledged", func() {
			msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})

			conn.Deliver(msg1)

			c.Expect(len(queue.Inflight()), Equals, 0)

			queueName, message := queue.Pull(100)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg1.json())

			c.Expect(len(queue.Inflight()), Equals, 1)
			c.Expect(queue.Inflight()[0], Equals, msg1.json())

			queue.Ack(msg1)

			c.Expect(len(queue.Inflight()), Equals, 0)
		})

		c.Specify("pulls from inflight message set if messages are unacknowledged", func() {
			msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
			msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})

			conn.Deliver(msg1)
			conn.Deliver(msg2)

			queueName, message := queue.Pull(0)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg1.json())

			queueName, message = queue.Pull(10)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg1.json())

			queueName, message = queue.Pull(10)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg2.json())
		})

		c.Specify("allows puller to ping to keep message inflight", func() {
			msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
			msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})

			conn.Deliver(msg1)
			conn.Deliver(msg2)

			queueName, message := queue.Pull(0)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg1.json())

			// Extends time before message is resent
			queue.Ping(msg1, 10)

			queueName, message = queue.Pull(10)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg2.json())

			// Sets time for message to resend to now
			queue.Ping(msg1, 0)

			queueName, message = queue.Pull(10)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg1.json())
		})

		c.Specify("set limits messages inflight", func() {
			limit, err := queue.InflightLimit()

			c.Expect(limit, Equals, 0)
			c.Expect(err, IsNil)

			queue.SetInflightLimit(1)

			limit, err = queue.InflightLimit()

			c.Expect(limit, Equals, 1)
			c.Expect(err, IsNil)
		})

		c.Specify("limits messages inflight", func() {
			r := config.Pool.Get()
			defer r.Close()

			config.Facet = func(msg *Msg) string {
				str, _ := msg.Get("facet").String()
				return str
			}

			msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
			msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
			msg3, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage3"})

			conn.Deliver(msg1)
			conn.Deliver(msg2)
			conn.Deliver(msg3)

			queue.SetInflightLimit(1)

			_, message := queue.Pull(2)
			c.Expect(message.json(), Equals, msg1.json())

			count, _ := redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
			c.Expect(count, Equals, 1)

			_, message = queue.Pull(2)
			c.Expect(message.json(), Equals, msg3.json())

			count, _ = redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
			c.Expect(count, Equals, 1)

			count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
			c.Expect(count, Equals, 1)

			_, message = queue.Pull(2)
			c.Expect(message, IsNil)
			_, message = queue.Pull(2)
			c.Expect(message, IsNil)

			count, _ = redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
			c.Expect(count, Equals, 1)

			count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
			c.Expect(count, Equals, 1)

			queue.Ack(msg1)
			queue.Ack(msg1)
			queue.Ack(msg1)
			queue.Ack(msg1)
			queue.Ack(msg1)

			count, err := redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
			c.Expect(count, Equals, 0)
			c.Expect(err.Error(), Equals, "redigo: nil returned")

			count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
			c.Expect(count, Equals, 1)

			_, message = queue.Pull(2)
			c.Expect(message.json(), Equals, msg2.json())

			count, _ = redis.Int(r.Do("get", "fairway:myqueue:1:inflight"))
			c.Expect(count, Equals, 1)

			count, _ = redis.Int(r.Do("get", "fairway:myqueue:2:inflight"))
			c.Expect(count, Equals, 1)
		})

		c.Specify("prevents overlimit messages when all messages are inflight", func() {
			r := config.Pool.Get()
			defer r.Close()

			config.Facet = func(msg *Msg) string {
				str, _ := msg.Get("facet").String()
				return str
			}

			msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
			msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
			msg3, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage3"})

			queue.SetInflightLimit(1)

			active, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 0)
			fqueue, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			conn.Deliver(msg1)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			c.Expect(active[0], Equals, "1")
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 1)

			_, message := queue.Pull(2)
			c.Expect(message.json(), Equals, msg1.json())

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			c.Expect(active[0], Equals, "1")
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			conn.Deliver(msg2)

			_, message = queue.Pull(2)
			c.Expect(message, IsNil)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			c.Expect(active[0], Equals, "1")
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			queue.Ack(msg1)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			c.Expect(active[0], Equals, "1")
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 1)

			_, message = queue.Pull(2)
			c.Expect(message.json(), Equals, msg2.json())

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			c.Expect(active[0], Equals, "1")
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			conn.Deliver(msg3)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			c.Expect(active[0], Equals, "1")
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			_, message = queue.Pull(2)
			c.Expect(message, IsNil)

			queue.Ack(msg2)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			c.Expect(active[0], Equals, "1")
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 1)

			_, message = queue.Pull(2)
			c.Expect(message.json(), Equals, msg3.json())

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			c.Expect(active[0], Equals, "1")
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			_, message = queue.Pull(2)
			c.Expect(message, IsNil)

			queue.Ack(msg3)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 0)
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			msg4, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage4"})

			conn.Deliver(msg4)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 1)

			_, message = queue.Pull(2)
			c.Expect(message.json(), Equals, msg4.json())

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			queue.Ack(msg4)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 0)
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			_, message = queue.Pull(2)
			c.Expect(message, IsNil)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 0)
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)
		})

		c.Specify("if inflight limit is 0, no limit", func() {
			r := config.Pool.Get()
			defer r.Close()

			config.Facet = func(msg *Msg) string {
				str, _ := msg.Get("facet").String()
				return str
			}

			msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
			msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
			msg3, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage3"})

			queue.SetInflightLimit(0)

			conn.Deliver(msg1)
			conn.Deliver(msg2)
			conn.Deliver(msg3)

			active, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 2)
			fqueue, _ := redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 2)

			_, message := queue.Pull(2)
			c.Expect(message.json(), Equals, msg1.json())

			_, message = queue.Pull(2)
			c.Expect(message.json(), Equals, msg3.json())

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 1)

			_, message = queue.Pull(2)
			c.Expect(message.json(), Equals, msg2.json())

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 0)
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			msg4, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage4"})

			conn.Deliver(msg4)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 1)

			queue.Ack(msg1)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 1)

			_, message = queue.Pull(2)
			c.Expect(message.json(), Equals, msg4.json())

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 0)
			fqueue, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
			c.Expect(fqueue, Equals, 0)

			_, message = queue.Pull(2)
			c.Expect(message, IsNil)

			queue.Ack(msg2)
			queue.Ack(msg3)
			queue.Ack(msg4)
		})

		c.Specify("doesn't place pulled message on inflight sorted set if inflight is disabled", func() {
			msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})

			conn.Deliver(msg1)

			c.Expect(len(queue.Inflight()), Equals, 0)

			queueName, message := queue.Pull(-1)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg1.json())

			c.Expect(len(queue.Inflight()), Equals, 0)
		})

		c.Specify("doesn't pull from inflight message set if inflight is disabled", func() {
			msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
			msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})

			conn.Deliver(msg1)
			conn.Deliver(msg2)

			queueName, message := queue.Pull(-1)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg1.json())

			queueName, message = queue.Pull(-1)
			c.Expect(queueName, Equals, "myqueue")
			c.Expect(message.json(), Equals, msg2.json())
		})

		c.Specify("pulls from facets of the queue in round-robin", func() {
			r := config.Pool.Get()
			defer r.Close()

			config.Facet = func(msg *Msg) string {
				str, _ := msg.Get("facet").String()
				return str
			}

			msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
			msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
			msg3, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage3"})

			active, _ := redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 0)

			conn.Deliver(msg1)
			conn.Deliver(msg2)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			c.Expect(active[0], Equals, "1")

			conn.Deliver(msg3)

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 2)
			c.Expect(active[0], Equals, "1")
			c.Expect(active[1], Equals, "2")

			_, message := queue.Pull(-1)
			c.Expect(message.json(), Equals, msg1.json())

			_, message = queue.Pull(-1)
			c.Expect(message.json(), Equals, msg3.json())

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 1)
			c.Expect(active[0], Equals, "1")

			_, message = queue.Pull(-1)
			c.Expect(message.json(), Equals, msg2.json())

			active, _ = redis.Strings(r.Do("smembers", "fairway:myqueue:active_facets"))
			c.Expect(len(active), Equals, 0)

			_, message = queue.Pull(2)
			c.Expect(message, IsNil)
		})

		c.Specify("removes facet from active list if it becomes empty", func() {
			r := config.Pool.Get()
			defer r.Close()

			msg, _ := NewMsg(map[string]string{})
			conn.Deliver(msg)

			count, _ := redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
			c.Expect(count, Equals, 1)

			queue.Pull(-1)

			count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
			c.Expect(count, Equals, 0)
		})

		c.Specify("returns nil if there are no messages to receive", func() {
			msg, _ := NewMsg(map[string]string{})
			conn.Deliver(msg)

			queueName, message := queue.Pull(-1)
			c.Expect(queueName, Equals, "myqueue")
			queueName, message = queue.Pull(-1)
			c.Expect(queueName, Equals, "")
			c.Expect(message, IsNil)
		})
	})
}