Exemplo n.º 1
0
Arquivo: commands.go Projeto: mc0/okq
func qflush(client *clients.Client, args []string) (interface{}, error) {
	queue := args[0]
	unclaimedKey := db.UnclaimedKey(queue)
	claimedKey := db.ClaimedKey(queue)
	itemsKey := db.ItemsKey(queue)

	err := db.Inst.Cmd("DEL", unclaimedKey, claimedKey, itemsKey).Err
	if err != nil {
		return nil, fmt.Errorf("QFLUSH DEL: %s", err)
	}
	return okSS, nil
}
Exemplo n.º 2
0
Arquivo: commands.go Projeto: mc0/okq
func qstatus(client *clients.Client, args []string) (interface{}, error) {
	var queueNames []string
	if len(args) == 0 {
		queueNames = db.AllQueueNames()
		sort.Strings(queueNames)
	} else {
		queueNames = args
	}

	queueInfos := make([][]interface{}, 0, len(queueNames))
	for i := range queueNames {
		queueName := queueNames[i]

		claimedCount := 0
		availableCount := 0
		totalCount := 0
		consumerCount := int64(0)

		unclaimedKey := db.UnclaimedKey(queueName)
		claimedKey := db.ClaimedKey(queueName)

		claimedCount, err := db.Inst.Cmd("LLEN", claimedKey).Int()
		if err != nil {
			return nil, fmt.Errorf("QSTATUS LLEN claimed: %s", err)
		}

		availableCount, err = db.Inst.Cmd("LLEN", unclaimedKey).Int()
		if err != nil {
			return nil, fmt.Errorf("QSTATUS LLEN unclaimed: %s", err)
		}

		totalCount = availableCount + claimedCount

		consumerCount, err = consumers.QueueConsumerCount(queueName)
		if err != nil {
			return nil, fmt.Errorf("QSTATUS QueueConsumerCount: %s", err)
		}

		queueInfos = append(queueInfos, []interface{}{
			queueName,
			int64(totalCount),
			int64(claimedCount),
			consumerCount,
		})
	}

	return queueInfos, nil
}
Exemplo n.º 3
0
Arquivo: restore.go Projeto: mc0/okq
func validateClaimedEvents() {
	log.L.Debug("validating claimed events")

	queueNames := db.AllQueueNames()

	for i := range queueNames {
		queueName := queueNames[i]
		claimedKey := db.ClaimedKey(queueName)

		// get the presumably oldest 50 items
		var eventIDs []string
		eventIDs, err := db.Inst.Cmd("LRANGE", claimedKey, -50, -1).List()
		if err != nil {
			log.L.Printf("ERR lrange redis replied %q", err)
			return
		} else if len(eventIDs) == 0 {
			continue
		}

		var locks []interface{}
		for i := range eventIDs {
			lockKey := db.ItemLockKey(queueName, eventIDs[i])
			locks = append(locks, lockKey)
		}

		var locksList [][]byte
		locksList, err = db.Inst.Cmd("MGET", locks...).ListBytes()
		if err != nil {
			log.L.Printf("ERR mget redis replied %q", err)
			return
		}

		for i := range locksList {
			if locksList[i] == nil {
				err = restoreEventToQueue(queueName, eventIDs[i])
				if err != nil {
					return
				}
			}
		}
	}
}
Exemplo n.º 4
0
func TestQFlush(t *T) {
	client := newClient()
	queue := clients.RandQueueName()

	// Ensure adding a single event to a queue and then flushing destroys it
	Dispatch(client, "qlpush", []string{queue, "0", "foo"})
	readAndAssertStr(t, client, "OK")
	Dispatch(client, "qflush", []string{queue})
	readAndAssertStr(t, client, "OK")
	Dispatch(client, "qstatus", []string{queue})
	readAndAssertQStatus(t, client, []queueInfo{
		queueInfo{queue, 0, 0, 0},
	})

	// Ensure adding a multiple items, having one in the claimed queue, and then
	// flushing still destroys everything
	Dispatch(client, "qlpush", []string{queue, "0", "foo"})
	readAndAssertStr(t, client, "OK")
	Dispatch(client, "qlpush", []string{queue, "1", "foo"})
	readAndAssertStr(t, client, "OK")
	Dispatch(client, "qrpop", []string{queue})
	readAndAssertArr(t, client, []string{"0", "foo"})
	Dispatch(client, "qflush", []string{queue})
	readAndAssertStr(t, client, "OK")
	Dispatch(client, "qstatus", []string{queue})
	readAndAssertQStatus(t, client, []queueInfo{
		queueInfo{queue, 0, 0, 0},
	})

	// Make sure the actual redis keys are destroyed
	keys := []string{
		db.UnclaimedKey(queue),
		db.ClaimedKey(queue),
		db.ItemsKey(queue),
	}
	for _, key := range keys {
		exists, err := db.Inst.Cmd("EXISTS", key).Int()
		assert.Nil(t, err)
		assert.Equal(t, 0, exists)
	}
}
Exemplo n.º 5
0
// There's a race condition where an item get's ackd just after being found by
// the restore process. The process checks if the item lock exists, it doesn't
// because the ack just deleted it, and then attempts to move the item from the
// claimed to the unclaimed queue. We need to make sure it doesn't appear in the
// unclaimed queue in this case
func TestRestoreRace(t *T) {

	q := clients.RandQueueName()
	unclaimedKey := db.UnclaimedKey(q)
	claimedKey := db.ClaimedKey(q)

	require.Nil(t, db.Inst.Cmd("LPUSH", unclaimedKey, "buz", "boz").Err)
	require.Nil(t, db.Inst.Cmd("LPUSH", claimedKey, "bar", "baz").Err)

	// restore an event which is in the claimed list, and one which is not
	require.Nil(t, restoreEventToQueue(q, "bar"))
	require.Nil(t, restoreEventToQueue(q, "foo"))

	l, err := db.Inst.Cmd("LRANGE", claimedKey, 0, -1).List()
	require.Nil(t, err)
	assert.Equal(t, []string{"baz"}, l)

	l, err = db.Inst.Cmd("LRANGE", unclaimedKey, 0, -1).List()
	require.Nil(t, err)
	assert.Equal(t, []string{"boz", "buz", "bar"}, l)
}
Exemplo n.º 6
0
func TestRestore(t *T) {

	// Add an item to claimed list and call validateClaimedEvents. Since it
	// doesn't have a lock it should be moved to the unclaimed list
	q := clients.RandQueueName()
	unclaimedKey := db.UnclaimedKey(q)
	claimedKey := db.ClaimedKey(q)
	itemsKey := db.ItemsKey(q)
	require.Nil(t, db.Inst.Cmd("HSET", itemsKey, "foo", "bar").Err)
	require.Nil(t, db.Inst.Cmd("LPUSH", claimedKey, "foo").Err)

	validateClaimedEvents()

	l, err := db.Inst.Cmd("LRANGE", claimedKey, 0, -1).List()
	require.Nil(t, err)
	assert.Empty(t, l)

	l, err = db.Inst.Cmd("LRANGE", unclaimedKey, 0, -1).List()
	require.Nil(t, err)
	assert.Equal(t, []string{"foo"}, l)
}
Exemplo n.º 7
0
Arquivo: commands.go Projeto: mc0/okq
func qack(client *clients.Client, args []string) (interface{}, error) {
	queueName, eventID := args[0], args[1]
	claimedKey := db.ClaimedKey(queueName)

	var numRemoved int
	numRemoved, err := db.Inst.Cmd("LREM", claimedKey, -1, eventID).Int()
	if err != nil {
		return nil, fmt.Errorf("QACK LREM (claimed): %s", err)
	}

	// If we didn't removed the eventID from the claimed events we see if it can
	// be found in unclaimed. We only do this in the uncommon case that the
	// eventID isn't in claimed since unclaimed can be really large, so LREM is
	// slow on it
	if numRemoved == 0 {
		unclaimedKey := db.UnclaimedKey(queueName)
		var err error
		numRemoved, err = db.Inst.Cmd("LREM", unclaimedKey, -1, eventID).Int()
		if err != nil {
			return nil, fmt.Errorf("QACK LREM (unclaimed): %s", err)
		}
	}

	// We only remove the object data itself if the eventID was actually removed
	// from something
	if numRemoved > 0 {
		itemsKey := db.ItemsKey(queueName)
		lockKey := db.ItemLockKey(queueName, eventID)
		_, err := db.Inst.Pipe(
			db.PP("HDEL", itemsKey, eventID),
			db.PP("DEL", lockKey),
		)
		if err != nil {
			return nil, fmt.Errorf("QACK HDEL/DEL: %s", err)
		}
	}

	return numRemoved, nil
}
Exemplo n.º 8
0
Arquivo: commands.go Projeto: mc0/okq
func qrpop(client *clients.Client, args []string) (interface{}, error) {
	var err error
	queueName := args[0]
	expires := 30

	noack := false
	args = args[1:]
	if len(args) > 1 && isEqualUpper("EX", args[0]) {
		if expires, err = parseInt(args[1], "expires"); err != nil {
			return err, nil
		}
		// The expiry of 0 is special and turns the read into at-most-once
		if expires == 0 {
			noack = true
			expires = 30
		}
	}

	unclaimedKey := db.UnclaimedKey(queueName)
	claimedKey := db.ClaimedKey(queueName)
	itemsKey := db.ItemsKey(queueName)

	reply := db.Inst.Lua("RPOPLPUSH_LOCK_HGET", 3, unclaimedKey, claimedKey, itemsKey, queueName, expires)
	if reply.IsType(redis.Nil) {
		return nil, nil
	}

	r, err := reply.List()
	if err != nil {
		return nil, fmt.Errorf("QRPOP RPOPLPUSH_LOCK_HGET: %s", err)
	}

	if noack {
		qack(client, []string{queueName, r[0]})
	}

	return r, nil
}
Exemplo n.º 9
0
Arquivo: restore.go Projeto: mc0/okq
func restoreEventToQueue(queueName string, eventID string) error {
	unclaimedKey := db.UnclaimedKey(queueName)
	claimedKey := db.ClaimedKey(queueName)
	r := db.Inst.Lua("LREMRPUSH", 2, claimedKey, unclaimedKey, eventID)
	return r.Err
}