// request2Response converts will interpret VBucket field as Status field and // re-interpret the request packate as response packet. This is required for // UPR streams which are full duplex. func request2Response(req *mcd.MCRequest) *mcd.MCResponse { return &mcd.MCResponse{ Opcode: req.Opcode, Cas: req.Cas, Opaque: req.Opaque, Status: mcd.Status(req.VBucket), Extras: req.Extras, Key: req.Key, Body: req.Body, } }
func grokHeader(hdrBytes []byte) (rv *gomemcached.MCResponse, err error) { if hdrBytes[0] != gomemcached.RES_MAGIC { return rv, fmt.Errorf("Bad magic: 0x%02x", hdrBytes[0]) } rv = &gomemcached.MCResponse{ Opcode: gomemcached.CommandCode(hdrBytes[1]), Key: make([]byte, binary.BigEndian.Uint16(hdrBytes[2:4])), Extras: make([]byte, hdrBytes[4]), Status: gomemcached.Status(binary.BigEndian.Uint16(hdrBytes[6:8])), Opaque: binary.BigEndian.Uint32(hdrBytes[12:16]), Cas: binary.BigEndian.Uint64(hdrBytes[16:24]), } bodyLen := binary.BigEndian.Uint32(hdrBytes[8:12]) - uint32(len(rv.Key)+len(rv.Extras)) rv.Body = make([]byte, bodyLen) return }
UPR_OPEN = gomemcached.CommandCode(0x50) UPR_ADD_STREAM = gomemcached.CommandCode(0x51) UPR_CLOSE_STREAM = gomemcached.CommandCode(0x52) UPR_FAILOVER_LOG = gomemcached.CommandCode(0x54) UPR_STREAM_REQ = gomemcached.CommandCode(0x53) UPR_STREAM_END = gomemcached.CommandCode(0x55) UPR_SNAPSHOTM = gomemcached.CommandCode(0x56) UPR_MUTATION = gomemcached.CommandCode(0x57) UPR_DELETION = gomemcached.CommandCode(0x58) UPR_EXPIRATION = gomemcached.CommandCode(0x59) UPR_FLUSH = gomemcached.CommandCode(0x5a) ) const ( // UPR Status ROLLBACK = gomemcached.Status(0x23) ) func init() { gomemcached.CommandNames[UPR_OPEN] = "UPR_OPEN" gomemcached.CommandNames[UPR_ADD_STREAM] = "UPR_ADD_STREAM" gomemcached.CommandNames[UPR_CLOSE_STREAM] = "UPR_CLOSE_STREAM" gomemcached.CommandNames[UPR_FAILOVER_LOG] = "UPR_FAILOVER_LOG" gomemcached.CommandNames[UPR_STREAM_REQ] = "UPR_STREAM_REQ" gomemcached.CommandNames[UPR_STREAM_END] = "UPR_STREAM_END" gomemcached.CommandNames[UPR_SNAPSHOTM] = "UPR_SNAPSHOTM" gomemcached.CommandNames[UPR_MUTATION] = "UPR_MUTATION" gomemcached.CommandNames[UPR_DELETION] = "UPR_DELETION" gomemcached.CommandNames[UPR_EXPIRATION] = "UPR_EXPIRATION" gomemcached.CommandNames[UPR_FLUSH] = "UPR_FLUSH" }
func TestBasicOps(t *testing.T) { empty := []byte{} active := uint16(3) ignored := gomemcached.Status(32768) tests := []struct { op gomemcached.CommandCode vb uint16 key string val string expStatus gomemcached.Status expValue []byte }{ {255, active, "", "", gomemcached.UNKNOWN_COMMAND, nil}, {gomemcached.SET, active, "a", "aye", gomemcached.SUCCESS, empty}, {gomemcached.GET, active, "a", "", gomemcached.SUCCESS, []byte("aye")}, {gomemcached.GETK, active, "a", "", // TODO: Assert the key? gomemcached.SUCCESS, []byte("aye")}, {gomemcached.GET, 2, "a", "", gomemcached.NOT_MY_VBUCKET, empty}, {gomemcached.GET, active, "b", "", gomemcached.KEY_ENOENT, empty}, {gomemcached.DELETE, active, "a", "", gomemcached.SUCCESS, empty}, {gomemcached.DELETE, active, "a", "", gomemcached.KEY_ENOENT, empty}, {gomemcached.GET, active, "a", "", gomemcached.KEY_ENOENT, empty}, // quiet {gomemcached.GETQ, active, "a", "aye", ignored, empty}, {gomemcached.DELETEQ, active, "a", "", ignored, empty}, {gomemcached.SETQ, active, "a", "aye", ignored, empty}, {gomemcached.GETQ, active, "a", "", gomemcached.SUCCESS, []byte("aye")}, } expStats := Stats{ Items: 1, Ops: int64(len(tests)) - 1, // Don't count the NOT_MY_VBUCKET. Gets: 6, GetMisses: 3, Mutations: 2, Sets: 2, Deletes: 3, Creates: 2, Unknowns: 1, IncomingValueBytes: 6, OutgoingValueBytes: 9, ItemBytes: 144, } testBucketDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(testBucketDir) testBucket, _ := NewBucket(testBucketDir, &BucketSettings{ NumPartitions: MAX_VBUCKETS, }) defer testBucket.Close() rh := reqHandler{currentBucket: testBucket} vb, _ := testBucket.CreateVBucket(3) testBucket.SetVBState(3, VBActive) for _, x := range tests { req := &gomemcached.MCRequest{ Opcode: x.op, VBucket: x.vb, Key: []byte(x.key), Body: []byte(x.val), } res := rh.HandleMessage(ioutil.Discard, nil, req) if res == nil && x.expStatus == ignored { // this was a "normal" quiet command continue } if res.Status != x.expStatus { t.Errorf("Expected %v for %v:%v/%v, got %v", x.expStatus, x.op, x.vb, x.key, res.Status) } if x.expValue != nil && !bytes.Equal(x.expValue, res.Body) { t.Errorf("Expected body of %v:%v/%v to be\n%#v\ngot\n%#v", x.op, x.vb, x.key, x.expValue, res.Body) } } time.Sleep(10 * time.Millisecond) // Let async stats catch up. if !reflect.DeepEqual(&expStats, &vb.stats) { t.Errorf("Expected stats of %#v, got %#v", expStats, vb.stats) } expStatItems := make(chan statItem) actStatItems := make(chan statItem) go func() { expStats.Send(expStatItems) close(expStatItems) }() go func() { AggregateStats(testBucket, "").Send(actStatItems) close(actStatItems) }() for expitem := range expStatItems { actitem := <-actStatItems if expitem.key != actitem.key { t.Errorf("agg stats expected key %v, got %v", expitem.key, actitem.key) } if expitem.val != actitem.val { t.Errorf("agg stats expected val %v, got %v for key %v", expitem.val, actitem.val, expitem.key) } } }
func TestArithOps(t *testing.T) { empty := []byte{} notempty := []byte("sentinel-for-not-empty") active := uint16(3) ignored := gomemcached.Status(32768) tests := []struct { op gomemcached.CommandCode vb uint16 key string amt uint64 def uint64 expStatus gomemcached.Status expValue []byte }{ {gomemcached.INCREMENT, active, "a", 111, 222, gomemcached.SUCCESS, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 222}}, {gomemcached.GET, active, "a", 0, 0, gomemcached.SUCCESS, []byte("222")}, {gomemcached.INCREMENT, active, "a", 444, 333, gomemcached.SUCCESS, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x9a}}, {gomemcached.GET, active, "a", 0, 0, gomemcached.SUCCESS, []byte("666")}, {gomemcached.DECREMENT, active, "a", 555, 777, gomemcached.SUCCESS, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 111}}, {gomemcached.GET, active, "a", 0, 0, gomemcached.SUCCESS, []byte("111")}, // quiet {gomemcached.INCREMENTQ, active, "a", 10, 888, ignored, empty}, {gomemcached.INCREMENTQ, active, "a", 3, 999, ignored, empty}, {gomemcached.DECREMENTQ, active, "a", 1, 222, ignored, empty}, {gomemcached.GET, active, "a", 0, 0, gomemcached.SUCCESS, []byte("123")}, } expStats := Stats{ Items: 1, Ops: int64(len(tests)), Gets: 4, GetMisses: 0, Mutations: 6, Sets: 0, Adds: 0, Replaces: 0, Appends: 0, Prepends: 0, Incrs: 4, Decrs: 2, Deletes: 0, Creates: 1, Updates: 5, Unknowns: 0, IncomingValueBytes: 0, OutgoingValueBytes: 12, ItemBytes: 113, } testBucketDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(testBucketDir) testBucket, _ := NewBucket(testBucketDir, &BucketSettings{ NumPartitions: MAX_VBUCKETS, }) defer testBucket.Close() rh := reqHandler{currentBucket: testBucket} vb, _ := testBucket.CreateVBucket(3) testBucket.SetVBState(3, VBActive) for idx, x := range tests { req := &gomemcached.MCRequest{ Opcode: x.op, VBucket: x.vb, Key: []byte(x.key), Body: []byte{}, } if x.op != gomemcached.GET { req.Extras = make([]byte, 8+8+4) binary.BigEndian.PutUint64(req.Extras[:8], x.amt) binary.BigEndian.PutUint64(req.Extras[8:16], x.def) binary.BigEndian.PutUint32(req.Extras[16:20], uint32(0)) } res := rh.HandleMessage(ioutil.Discard, nil, req) if res == nil && x.expStatus == ignored { // this was a "normal" quiet command continue } if res.Status != x.expStatus { t.Errorf("Expected %v for %v - %v:%v/%v, got %v", x.expStatus, idx, x.op, x.vb, x.key, res.Status) } if x.expValue != nil { if bytes.Equal(x.expValue, notempty) { if len(res.Body) <= 0 { t.Errorf("Expected non-empty body of %v - %v:%v/%v, got: %#v", idx, x.op, x.vb, x.key, res) } } else if !bytes.Equal(x.expValue, res.Body) { t.Errorf("Expected body of %v - %v:%v/%v to be\n%#v\ngot\n%#v", idx, x.op, x.vb, x.key, x.expValue, res.Body) } } } time.Sleep(10 * time.Millisecond) // Let async stats catch up. if !reflect.DeepEqual(&expStats, &vb.stats) { t.Errorf("Expected stats of %#v, got %#v", expStats, vb.stats) } expStatItems := make(chan statItem) actStatItems := make(chan statItem) go func() { expStats.Send(expStatItems) close(expStatItems) }() go func() { AggregateStats(testBucket, "").Send(actStatItems) close(actStatItems) }() for expitem := range expStatItems { actitem := <-actStatItems if expitem.key != actitem.key { t.Errorf("agg stats expected key %v, got %v", expitem.key, actitem.key) } if expitem.val != actitem.val { t.Errorf("agg stats expected val %v, got %v for key %v", expitem.val, actitem.val, expitem.key) } } }
func TestMutationOps(t *testing.T) { empty := []byte{} notempty := []byte("sentinel-for-not-empty") active := uint16(3) ignored := gomemcached.Status(32768) tests := []struct { op gomemcached.CommandCode vb uint16 key string val string expStatus gomemcached.Status expValue []byte }{ {gomemcached.REPLACE, active, "a", "irreplacable", gomemcached.KEY_ENOENT, notempty}, {gomemcached.GET, active, "a", "", gomemcached.KEY_ENOENT, empty}, {gomemcached.ADD, active, "a", "should-be-added", gomemcached.SUCCESS, empty}, {gomemcached.GET, active, "a", "", gomemcached.SUCCESS, []byte("should-be-added")}, {gomemcached.APPEND, active, "a", "_suffix", gomemcached.SUCCESS, empty}, {gomemcached.GET, active, "a", "", gomemcached.SUCCESS, []byte("should-be-added_suffix")}, {gomemcached.PREPEND, active, "a", "prefix_", gomemcached.SUCCESS, empty}, {gomemcached.GET, active, "a", "", gomemcached.SUCCESS, []byte("prefix_should-be-added_suffix")}, {gomemcached.REPLACE, active, "a", "replacement", gomemcached.SUCCESS, empty}, {gomemcached.GET, active, "a", "", gomemcached.SUCCESS, []byte("replacement")}, {gomemcached.ADD, active, "a", "not-added", gomemcached.KEY_EEXISTS, notempty}, {gomemcached.GET, active, "a", "", gomemcached.SUCCESS, []byte("replacement")}, // quiet {gomemcached.ADDQ, active, "a", "not-added", gomemcached.KEY_EEXISTS, notempty}, {gomemcached.GET, active, "a", "", gomemcached.SUCCESS, []byte("replacement")}, {gomemcached.REPLACEQ, active, "a", "replacement2", ignored, empty}, {gomemcached.GET, active, "a", "", gomemcached.SUCCESS, []byte("replacement2")}, {gomemcached.APPENDQ, active, "a", "_suffix2", ignored, empty}, {gomemcached.PREPENDQ, active, "a", "prefix2_", ignored, empty}, {gomemcached.GET, active, "a", "", gomemcached.SUCCESS, []byte("prefix2_replacement2_suffix2")}, {gomemcached.DELETE, active, "a", "", gomemcached.SUCCESS, empty}, {gomemcached.REPLACEQ, active, "a", "replacement2", gomemcached.KEY_ENOENT, notempty}, } expStats := Stats{ Items: 0, Ops: int64(len(tests)), Gets: 9, GetMisses: 1, Mutations: 11, Sets: 0, Adds: 3, Replaces: 4, Appends: 2, Prepends: 2, Deletes: 1, Creates: 1, Updates: 6, Unknowns: 0, IncomingValueBytes: 68, OutgoingValueBytes: 139, ItemBytes: 110, } testBucketDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(testBucketDir) testBucket, _ := NewBucket(testBucketDir, &BucketSettings{ NumPartitions: MAX_VBUCKETS, }) defer testBucket.Close() rh := reqHandler{currentBucket: testBucket} vb, _ := testBucket.CreateVBucket(3) testBucket.SetVBState(3, VBActive) for _, x := range tests { req := &gomemcached.MCRequest{ Opcode: x.op, VBucket: x.vb, Key: []byte(x.key), Body: []byte(x.val), } res := rh.HandleMessage(ioutil.Discard, nil, req) if res == nil && x.expStatus == ignored { // this was a "normal" quiet command continue } if res.Status != x.expStatus { t.Errorf("Expected %v for %v:%v/%v, got %v", x.expStatus, x.op, x.vb, x.key, res.Status) } if x.expValue != nil { if bytes.Equal(x.expValue, notempty) { if len(res.Body) <= 0 { t.Errorf("Expected non-empty body of %v:%v/%v, got: %#v", x.op, x.vb, x.key, res) } } else if !bytes.Equal(x.expValue, res.Body) { t.Errorf("Expected body of %v:%v/%v to be\n%#v\ngot\n%#v", x.op, x.vb, x.key, x.expValue, res.Body) } } } time.Sleep(10 * time.Millisecond) // Let async stats catch up. if !reflect.DeepEqual(&expStats, &vb.stats) { t.Errorf("Expected stats of %#v, got %#v", expStats, vb.stats) } expStatItems := make(chan statItem) actStatItems := make(chan statItem) go func() { expStats.Send(expStatItems) close(expStatItems) }() go func() { AggregateStats(testBucket, "").Send(actStatItems) close(actStatItems) }() for expitem := range expStatItems { actitem := <-actStatItems if expitem.key != actitem.key { t.Errorf("agg stats expected key %v, got %v", expitem.key, actitem.key) } if expitem.val != actitem.val { t.Errorf("agg stats expected val %v, got %v for key %v", expitem.val, actitem.val, expitem.key) } } }
// constants used for memcached protocol const ( uprOPEN = mcd.CommandCode(0x50) // Open a upr connection with `name` uprAddSTREAM = mcd.CommandCode(0x51) // Sent by ebucketmigrator to upr consumer uprCloseSTREAM = mcd.CommandCode(0x52) // Sent by ebucketmigrator to upr consumer uprFailoverLOG = mcd.CommandCode(0x54) // Request all known failover ids for restart uprStreamREQ = mcd.CommandCode(0x53) // Stream request from consumer to producer uprStreamEND = mcd.CommandCode(0x55) // Sent by producer when it is going to end stream uprSnapshotM = mcd.CommandCode(0x56) // Sent by producer for a new snapshot uprMUTATION = mcd.CommandCode(0x57) // Notifies SET/ADD/REPLACE/etc. on the server uprDELETION = mcd.CommandCode(0x58) // Notifies DELETE on the server uprEXPIRATION = mcd.CommandCode(0x59) // Notifies key expiration uprFLUSH = mcd.CommandCode(0x5a) // Notifies vbucket flush ) const ( rollBack = mcd.Status(0x23) ) // FailoverLog is a slice of 2 element array, containing a list of, // [[vuuid, sequence-no], [vuuid, sequence-no] ...] type FailoverLog [][2]uint64 // UprStream will maintain stream information per vbucket type UprStream struct { Vbucket uint16 // vbucket id Vuuid uint64 // vbucket uuid Opaque uint32 // messages from producer to this stream have same value Highseq uint64 // to be supplied by the application Startseq uint64 // to be supplied by the application Endseq uint64 // to be supplied by the application Flog FailoverLog