// request2Response converts will interpret VBucket field as Status field and
// re-interpret the request packate as response packet. This is required for
// UPR streams which are full duplex.
func request2Response(req *mcd.MCRequest) *mcd.MCResponse {
return &mcd.MCResponse{
Opcode: req.Opcode,
Cas: req.Cas,
Opaque: req.Opaque,
Status: mcd.Status(req.VBucket),
Extras: req.Extras,
Key: req.Key,
Body: req.Body,
}
}
func grokHeader(hdrBytes []byte) (rv *gomemcached.MCResponse, err error) {
if hdrBytes[0] != gomemcached.RES_MAGIC {
return rv, fmt.Errorf("Bad magic: 0x%02x", hdrBytes[0])
}
rv = &gomemcached.MCResponse{
Opcode: gomemcached.CommandCode(hdrBytes[1]),
Key: make([]byte, binary.BigEndian.Uint16(hdrBytes[2:4])),
Extras: make([]byte, hdrBytes[4]),
Status: gomemcached.Status(binary.BigEndian.Uint16(hdrBytes[6:8])),
Opaque: binary.BigEndian.Uint32(hdrBytes[12:16]),
Cas: binary.BigEndian.Uint64(hdrBytes[16:24]),
}
bodyLen := binary.BigEndian.Uint32(hdrBytes[8:12]) -
uint32(len(rv.Key)+len(rv.Extras))
rv.Body = make([]byte, bodyLen)
return
}
UPR_OPEN = gomemcached.CommandCode(0x50)
UPR_ADD_STREAM = gomemcached.CommandCode(0x51)
UPR_CLOSE_STREAM = gomemcached.CommandCode(0x52)
UPR_FAILOVER_LOG = gomemcached.CommandCode(0x54)
UPR_STREAM_REQ = gomemcached.CommandCode(0x53)
UPR_STREAM_END = gomemcached.CommandCode(0x55)
UPR_SNAPSHOTM = gomemcached.CommandCode(0x56)
UPR_MUTATION = gomemcached.CommandCode(0x57)
UPR_DELETION = gomemcached.CommandCode(0x58)
UPR_EXPIRATION = gomemcached.CommandCode(0x59)
UPR_FLUSH = gomemcached.CommandCode(0x5a)
)
const (
// UPR Status
ROLLBACK = gomemcached.Status(0x23)
)
func init() {
gomemcached.CommandNames[UPR_OPEN] = "UPR_OPEN"
gomemcached.CommandNames[UPR_ADD_STREAM] = "UPR_ADD_STREAM"
gomemcached.CommandNames[UPR_CLOSE_STREAM] = "UPR_CLOSE_STREAM"
gomemcached.CommandNames[UPR_FAILOVER_LOG] = "UPR_FAILOVER_LOG"
gomemcached.CommandNames[UPR_STREAM_REQ] = "UPR_STREAM_REQ"
gomemcached.CommandNames[UPR_STREAM_END] = "UPR_STREAM_END"
gomemcached.CommandNames[UPR_SNAPSHOTM] = "UPR_SNAPSHOTM"
gomemcached.CommandNames[UPR_MUTATION] = "UPR_MUTATION"
gomemcached.CommandNames[UPR_DELETION] = "UPR_DELETION"
gomemcached.CommandNames[UPR_EXPIRATION] = "UPR_EXPIRATION"
gomemcached.CommandNames[UPR_FLUSH] = "UPR_FLUSH"
}
func TestBasicOps(t *testing.T) {
empty := []byte{}
active := uint16(3)
ignored := gomemcached.Status(32768)
tests := []struct {
op gomemcached.CommandCode
vb uint16
key string
val string
expStatus gomemcached.Status
expValue []byte
}{
{255, active, "", "",
gomemcached.UNKNOWN_COMMAND, nil},
{gomemcached.SET, active, "a", "aye",
gomemcached.SUCCESS, empty},
{gomemcached.GET, active, "a", "",
gomemcached.SUCCESS, []byte("aye")},
{gomemcached.GETK, active, "a", "", // TODO: Assert the key?
gomemcached.SUCCESS, []byte("aye")},
{gomemcached.GET, 2, "a", "",
gomemcached.NOT_MY_VBUCKET, empty},
{gomemcached.GET, active, "b", "",
gomemcached.KEY_ENOENT, empty},
{gomemcached.DELETE, active, "a", "",
gomemcached.SUCCESS, empty},
{gomemcached.DELETE, active, "a", "",
gomemcached.KEY_ENOENT, empty},
{gomemcached.GET, active, "a", "",
gomemcached.KEY_ENOENT, empty},
// quiet
{gomemcached.GETQ, active, "a", "aye",
ignored, empty},
{gomemcached.DELETEQ, active, "a", "",
ignored, empty},
{gomemcached.SETQ, active, "a", "aye",
ignored, empty},
{gomemcached.GETQ, active, "a", "",
gomemcached.SUCCESS, []byte("aye")},
}
expStats := Stats{
Items: 1,
Ops: int64(len(tests)) - 1, // Don't count the NOT_MY_VBUCKET.
Gets: 6,
GetMisses: 3,
Mutations: 2,
Sets: 2,
Deletes: 3,
Creates: 2,
Unknowns: 1,
IncomingValueBytes: 6,
OutgoingValueBytes: 9,
ItemBytes: 144,
}
testBucketDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(testBucketDir)
testBucket, _ := NewBucket(testBucketDir,
&BucketSettings{
NumPartitions: MAX_VBUCKETS,
})
defer testBucket.Close()
rh := reqHandler{currentBucket: testBucket}
vb, _ := testBucket.CreateVBucket(3)
testBucket.SetVBState(3, VBActive)
for _, x := range tests {
req := &gomemcached.MCRequest{
Opcode: x.op,
VBucket: x.vb,
Key: []byte(x.key),
Body: []byte(x.val),
}
res := rh.HandleMessage(ioutil.Discard, nil, req)
if res == nil && x.expStatus == ignored {
// this was a "normal" quiet command
continue
}
if res.Status != x.expStatus {
t.Errorf("Expected %v for %v:%v/%v, got %v",
x.expStatus, x.op, x.vb, x.key, res.Status)
}
if x.expValue != nil && !bytes.Equal(x.expValue, res.Body) {
t.Errorf("Expected body of %v:%v/%v to be\n%#v\ngot\n%#v",
x.op, x.vb, x.key, x.expValue, res.Body)
}
}
time.Sleep(10 * time.Millisecond) // Let async stats catch up.
if !reflect.DeepEqual(&expStats, &vb.stats) {
t.Errorf("Expected stats of %#v, got %#v", expStats, vb.stats)
}
expStatItems := make(chan statItem)
actStatItems := make(chan statItem)
go func() {
expStats.Send(expStatItems)
close(expStatItems)
}()
go func() {
AggregateStats(testBucket, "").Send(actStatItems)
close(actStatItems)
}()
for expitem := range expStatItems {
actitem := <-actStatItems
if expitem.key != actitem.key {
t.Errorf("agg stats expected key %v, got %v",
expitem.key, actitem.key)
}
if expitem.val != actitem.val {
t.Errorf("agg stats expected val %v, got %v for key %v",
expitem.val, actitem.val, expitem.key)
}
}
}
func TestArithOps(t *testing.T) {
empty := []byte{}
notempty := []byte("sentinel-for-not-empty")
active := uint16(3)
ignored := gomemcached.Status(32768)
tests := []struct {
op gomemcached.CommandCode
vb uint16
key string
amt uint64
def uint64
expStatus gomemcached.Status
expValue []byte
}{
{gomemcached.INCREMENT, active, "a", 111, 222,
gomemcached.SUCCESS, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 222}},
{gomemcached.GET, active, "a", 0, 0,
gomemcached.SUCCESS, []byte("222")},
{gomemcached.INCREMENT, active, "a", 444, 333,
gomemcached.SUCCESS, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x9a}},
{gomemcached.GET, active, "a", 0, 0,
gomemcached.SUCCESS, []byte("666")},
{gomemcached.DECREMENT, active, "a", 555, 777,
gomemcached.SUCCESS, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 111}},
{gomemcached.GET, active, "a", 0, 0,
gomemcached.SUCCESS, []byte("111")},
// quiet
{gomemcached.INCREMENTQ, active, "a", 10, 888,
ignored, empty},
{gomemcached.INCREMENTQ, active, "a", 3, 999,
ignored, empty},
{gomemcached.DECREMENTQ, active, "a", 1, 222,
ignored, empty},
{gomemcached.GET, active, "a", 0, 0,
gomemcached.SUCCESS, []byte("123")},
}
expStats := Stats{
Items: 1,
Ops: int64(len(tests)),
Gets: 4,
GetMisses: 0,
Mutations: 6,
Sets: 0,
Adds: 0,
Replaces: 0,
Appends: 0,
Prepends: 0,
Incrs: 4,
Decrs: 2,
Deletes: 0,
Creates: 1,
Updates: 5,
Unknowns: 0,
IncomingValueBytes: 0,
OutgoingValueBytes: 12,
ItemBytes: 113,
}
testBucketDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(testBucketDir)
testBucket, _ := NewBucket(testBucketDir,
&BucketSettings{
NumPartitions: MAX_VBUCKETS,
})
defer testBucket.Close()
rh := reqHandler{currentBucket: testBucket}
vb, _ := testBucket.CreateVBucket(3)
testBucket.SetVBState(3, VBActive)
for idx, x := range tests {
req := &gomemcached.MCRequest{
Opcode: x.op,
VBucket: x.vb,
Key: []byte(x.key),
Body: []byte{},
}
if x.op != gomemcached.GET {
req.Extras = make([]byte, 8+8+4)
binary.BigEndian.PutUint64(req.Extras[:8], x.amt)
binary.BigEndian.PutUint64(req.Extras[8:16], x.def)
binary.BigEndian.PutUint32(req.Extras[16:20], uint32(0))
}
res := rh.HandleMessage(ioutil.Discard, nil, req)
if res == nil && x.expStatus == ignored {
// this was a "normal" quiet command
continue
}
if res.Status != x.expStatus {
t.Errorf("Expected %v for %v - %v:%v/%v, got %v",
x.expStatus, idx, x.op, x.vb, x.key, res.Status)
}
if x.expValue != nil {
if bytes.Equal(x.expValue, notempty) {
if len(res.Body) <= 0 {
t.Errorf("Expected non-empty body of %v - %v:%v/%v, got: %#v",
idx, x.op, x.vb, x.key, res)
}
} else if !bytes.Equal(x.expValue, res.Body) {
t.Errorf("Expected body of %v - %v:%v/%v to be\n%#v\ngot\n%#v",
idx, x.op, x.vb, x.key, x.expValue, res.Body)
}
}
}
time.Sleep(10 * time.Millisecond) // Let async stats catch up.
if !reflect.DeepEqual(&expStats, &vb.stats) {
t.Errorf("Expected stats of %#v, got %#v", expStats, vb.stats)
}
expStatItems := make(chan statItem)
actStatItems := make(chan statItem)
go func() {
expStats.Send(expStatItems)
close(expStatItems)
}()
go func() {
AggregateStats(testBucket, "").Send(actStatItems)
close(actStatItems)
}()
for expitem := range expStatItems {
actitem := <-actStatItems
if expitem.key != actitem.key {
t.Errorf("agg stats expected key %v, got %v",
expitem.key, actitem.key)
}
if expitem.val != actitem.val {
t.Errorf("agg stats expected val %v, got %v for key %v",
expitem.val, actitem.val, expitem.key)
}
}
}
func TestMutationOps(t *testing.T) {
empty := []byte{}
notempty := []byte("sentinel-for-not-empty")
active := uint16(3)
ignored := gomemcached.Status(32768)
tests := []struct {
op gomemcached.CommandCode
vb uint16
key string
val string
expStatus gomemcached.Status
expValue []byte
}{
{gomemcached.REPLACE, active, "a", "irreplacable",
gomemcached.KEY_ENOENT, notempty},
{gomemcached.GET, active, "a", "",
gomemcached.KEY_ENOENT, empty},
{gomemcached.ADD, active, "a", "should-be-added",
gomemcached.SUCCESS, empty},
{gomemcached.GET, active, "a", "",
gomemcached.SUCCESS, []byte("should-be-added")},
{gomemcached.APPEND, active, "a", "_suffix",
gomemcached.SUCCESS, empty},
{gomemcached.GET, active, "a", "",
gomemcached.SUCCESS, []byte("should-be-added_suffix")},
{gomemcached.PREPEND, active, "a", "prefix_",
gomemcached.SUCCESS, empty},
{gomemcached.GET, active, "a", "",
gomemcached.SUCCESS, []byte("prefix_should-be-added_suffix")},
{gomemcached.REPLACE, active, "a", "replacement",
gomemcached.SUCCESS, empty},
{gomemcached.GET, active, "a", "",
gomemcached.SUCCESS, []byte("replacement")},
{gomemcached.ADD, active, "a", "not-added",
gomemcached.KEY_EEXISTS, notempty},
{gomemcached.GET, active, "a", "",
gomemcached.SUCCESS, []byte("replacement")},
// quiet
{gomemcached.ADDQ, active, "a", "not-added",
gomemcached.KEY_EEXISTS, notempty},
{gomemcached.GET, active, "a", "",
gomemcached.SUCCESS, []byte("replacement")},
{gomemcached.REPLACEQ, active, "a", "replacement2",
ignored, empty},
{gomemcached.GET, active, "a", "",
gomemcached.SUCCESS, []byte("replacement2")},
{gomemcached.APPENDQ, active, "a", "_suffix2",
ignored, empty},
{gomemcached.PREPENDQ, active, "a", "prefix2_",
ignored, empty},
{gomemcached.GET, active, "a", "",
gomemcached.SUCCESS, []byte("prefix2_replacement2_suffix2")},
{gomemcached.DELETE, active, "a", "",
gomemcached.SUCCESS, empty},
{gomemcached.REPLACEQ, active, "a", "replacement2",
gomemcached.KEY_ENOENT, notempty},
}
expStats := Stats{
Items: 0,
Ops: int64(len(tests)),
Gets: 9,
GetMisses: 1,
Mutations: 11,
Sets: 0,
Adds: 3,
Replaces: 4,
Appends: 2,
Prepends: 2,
Deletes: 1,
Creates: 1,
Updates: 6,
Unknowns: 0,
IncomingValueBytes: 68,
OutgoingValueBytes: 139,
ItemBytes: 110,
}
testBucketDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(testBucketDir)
testBucket, _ := NewBucket(testBucketDir,
&BucketSettings{
NumPartitions: MAX_VBUCKETS,
})
defer testBucket.Close()
rh := reqHandler{currentBucket: testBucket}
vb, _ := testBucket.CreateVBucket(3)
testBucket.SetVBState(3, VBActive)
for _, x := range tests {
req := &gomemcached.MCRequest{
Opcode: x.op,
VBucket: x.vb,
Key: []byte(x.key),
Body: []byte(x.val),
}
res := rh.HandleMessage(ioutil.Discard, nil, req)
if res == nil && x.expStatus == ignored {
// this was a "normal" quiet command
continue
}
if res.Status != x.expStatus {
t.Errorf("Expected %v for %v:%v/%v, got %v",
x.expStatus, x.op, x.vb, x.key, res.Status)
}
if x.expValue != nil {
if bytes.Equal(x.expValue, notempty) {
if len(res.Body) <= 0 {
t.Errorf("Expected non-empty body of %v:%v/%v, got: %#v",
x.op, x.vb, x.key, res)
}
} else if !bytes.Equal(x.expValue, res.Body) {
t.Errorf("Expected body of %v:%v/%v to be\n%#v\ngot\n%#v",
x.op, x.vb, x.key, x.expValue, res.Body)
}
}
}
time.Sleep(10 * time.Millisecond) // Let async stats catch up.
if !reflect.DeepEqual(&expStats, &vb.stats) {
t.Errorf("Expected stats of %#v, got %#v", expStats, vb.stats)
}
expStatItems := make(chan statItem)
actStatItems := make(chan statItem)
go func() {
expStats.Send(expStatItems)
close(expStatItems)
}()
go func() {
AggregateStats(testBucket, "").Send(actStatItems)
close(actStatItems)
}()
for expitem := range expStatItems {
actitem := <-actStatItems
if expitem.key != actitem.key {
t.Errorf("agg stats expected key %v, got %v",
expitem.key, actitem.key)
}
if expitem.val != actitem.val {
t.Errorf("agg stats expected val %v, got %v for key %v",
expitem.val, actitem.val, expitem.key)
}
}
}
// constants used for memcached protocol
const (
uprOPEN = mcd.CommandCode(0x50) // Open a upr connection with `name`
uprAddSTREAM = mcd.CommandCode(0x51) // Sent by ebucketmigrator to upr consumer
uprCloseSTREAM = mcd.CommandCode(0x52) // Sent by ebucketmigrator to upr consumer
uprFailoverLOG = mcd.CommandCode(0x54) // Request all known failover ids for restart
uprStreamREQ = mcd.CommandCode(0x53) // Stream request from consumer to producer
uprStreamEND = mcd.CommandCode(0x55) // Sent by producer when it is going to end stream
uprSnapshotM = mcd.CommandCode(0x56) // Sent by producer for a new snapshot
uprMUTATION = mcd.CommandCode(0x57) // Notifies SET/ADD/REPLACE/etc. on the server
uprDELETION = mcd.CommandCode(0x58) // Notifies DELETE on the server
uprEXPIRATION = mcd.CommandCode(0x59) // Notifies key expiration
uprFLUSH = mcd.CommandCode(0x5a) // Notifies vbucket flush
)
const (
rollBack = mcd.Status(0x23)
)
// FailoverLog is a slice of 2 element array, containing a list of,
// [[vuuid, sequence-no], [vuuid, sequence-no] ...]
type FailoverLog [][2]uint64
// UprStream will maintain stream information per vbucket
type UprStream struct {
Vbucket uint16 // vbucket id
Vuuid uint64 // vbucket uuid
Opaque uint32 // messages from producer to this stream have same value
Highseq uint64 // to be supplied by the application
Startseq uint64 // to be supplied by the application
Endseq uint64 // to be supplied by the application
Flog FailoverLog