Beispiel #1
0
// request2Response converts will interpret VBucket field as Status field and
// re-interpret the request packate as response packet. This is required for
// UPR streams which are full duplex.
func request2Response(req *mcd.MCRequest) *mcd.MCResponse {
	return &mcd.MCResponse{
		Opcode: req.Opcode,
		Cas:    req.Cas,
		Opaque: req.Opaque,
		Status: mcd.Status(req.VBucket),
		Extras: req.Extras,
		Key:    req.Key,
		Body:   req.Body,
	}
}
Beispiel #2
0
func grokHeader(hdrBytes []byte) (rv *gomemcached.MCResponse, err error) {
	if hdrBytes[0] != gomemcached.RES_MAGIC {
		return rv, fmt.Errorf("Bad magic: 0x%02x", hdrBytes[0])
	}
	rv = &gomemcached.MCResponse{
		Opcode: gomemcached.CommandCode(hdrBytes[1]),
		Key:    make([]byte, binary.BigEndian.Uint16(hdrBytes[2:4])),
		Extras: make([]byte, hdrBytes[4]),
		Status: gomemcached.Status(binary.BigEndian.Uint16(hdrBytes[6:8])),
		Opaque: binary.BigEndian.Uint32(hdrBytes[12:16]),
		Cas:    binary.BigEndian.Uint64(hdrBytes[16:24]),
	}
	bodyLen := binary.BigEndian.Uint32(hdrBytes[8:12]) -
		uint32(len(rv.Key)+len(rv.Extras))
	rv.Body = make([]byte, bodyLen)

	return
}
Beispiel #3
0
	UPR_OPEN         = gomemcached.CommandCode(0x50)
	UPR_ADD_STREAM   = gomemcached.CommandCode(0x51)
	UPR_CLOSE_STREAM = gomemcached.CommandCode(0x52)
	UPR_FAILOVER_LOG = gomemcached.CommandCode(0x54)
	UPR_STREAM_REQ   = gomemcached.CommandCode(0x53)
	UPR_STREAM_END   = gomemcached.CommandCode(0x55)
	UPR_SNAPSHOTM    = gomemcached.CommandCode(0x56)
	UPR_MUTATION     = gomemcached.CommandCode(0x57)
	UPR_DELETION     = gomemcached.CommandCode(0x58)
	UPR_EXPIRATION   = gomemcached.CommandCode(0x59)
	UPR_FLUSH        = gomemcached.CommandCode(0x5a)
)

const (
	// UPR Status
	ROLLBACK = gomemcached.Status(0x23)
)

func init() {
	gomemcached.CommandNames[UPR_OPEN] = "UPR_OPEN"
	gomemcached.CommandNames[UPR_ADD_STREAM] = "UPR_ADD_STREAM"
	gomemcached.CommandNames[UPR_CLOSE_STREAM] = "UPR_CLOSE_STREAM"
	gomemcached.CommandNames[UPR_FAILOVER_LOG] = "UPR_FAILOVER_LOG"
	gomemcached.CommandNames[UPR_STREAM_REQ] = "UPR_STREAM_REQ"
	gomemcached.CommandNames[UPR_STREAM_END] = "UPR_STREAM_END"
	gomemcached.CommandNames[UPR_SNAPSHOTM] = "UPR_SNAPSHOTM"
	gomemcached.CommandNames[UPR_MUTATION] = "UPR_MUTATION"
	gomemcached.CommandNames[UPR_DELETION] = "UPR_DELETION"
	gomemcached.CommandNames[UPR_EXPIRATION] = "UPR_EXPIRATION"
	gomemcached.CommandNames[UPR_FLUSH] = "UPR_FLUSH"
}
Beispiel #4
0
func TestBasicOps(t *testing.T) {
	empty := []byte{}
	active := uint16(3)
	ignored := gomemcached.Status(32768)

	tests := []struct {
		op  gomemcached.CommandCode
		vb  uint16
		key string
		val string

		expStatus gomemcached.Status
		expValue  []byte
	}{
		{255, active, "", "",
			gomemcached.UNKNOWN_COMMAND, nil},
		{gomemcached.SET, active, "a", "aye",
			gomemcached.SUCCESS, empty},
		{gomemcached.GET, active, "a", "",
			gomemcached.SUCCESS, []byte("aye")},
		{gomemcached.GETK, active, "a", "", // TODO: Assert the key?
			gomemcached.SUCCESS, []byte("aye")},
		{gomemcached.GET, 2, "a", "",
			gomemcached.NOT_MY_VBUCKET, empty},
		{gomemcached.GET, active, "b", "",
			gomemcached.KEY_ENOENT, empty},
		{gomemcached.DELETE, active, "a", "",
			gomemcached.SUCCESS, empty},
		{gomemcached.DELETE, active, "a", "",
			gomemcached.KEY_ENOENT, empty},
		{gomemcached.GET, active, "a", "",
			gomemcached.KEY_ENOENT, empty},

		// quiet
		{gomemcached.GETQ, active, "a", "aye",
			ignored, empty},
		{gomemcached.DELETEQ, active, "a", "",
			ignored, empty},
		{gomemcached.SETQ, active, "a", "aye",
			ignored, empty},
		{gomemcached.GETQ, active, "a", "",
			gomemcached.SUCCESS, []byte("aye")},
	}

	expStats := Stats{
		Items:              1,
		Ops:                int64(len(tests)) - 1, // Don't count the NOT_MY_VBUCKET.
		Gets:               6,
		GetMisses:          3,
		Mutations:          2,
		Sets:               2,
		Deletes:            3,
		Creates:            2,
		Unknowns:           1,
		IncomingValueBytes: 6,
		OutgoingValueBytes: 9,
		ItemBytes:          144,
	}

	testBucketDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(testBucketDir)
	testBucket, _ := NewBucket(testBucketDir,
		&BucketSettings{
			NumPartitions: MAX_VBUCKETS,
		})
	defer testBucket.Close()
	rh := reqHandler{currentBucket: testBucket}
	vb, _ := testBucket.CreateVBucket(3)
	testBucket.SetVBState(3, VBActive)

	for _, x := range tests {
		req := &gomemcached.MCRequest{
			Opcode:  x.op,
			VBucket: x.vb,
			Key:     []byte(x.key),
			Body:    []byte(x.val),
		}

		res := rh.HandleMessage(ioutil.Discard, nil, req)

		if res == nil && x.expStatus == ignored {
			// this was a "normal" quiet command
			continue
		}

		if res.Status != x.expStatus {
			t.Errorf("Expected %v for %v:%v/%v, got %v",
				x.expStatus, x.op, x.vb, x.key, res.Status)
		}

		if x.expValue != nil && !bytes.Equal(x.expValue, res.Body) {
			t.Errorf("Expected body of %v:%v/%v to be\n%#v\ngot\n%#v",
				x.op, x.vb, x.key, x.expValue, res.Body)
		}
	}

	time.Sleep(10 * time.Millisecond) // Let async stats catch up.

	if !reflect.DeepEqual(&expStats, &vb.stats) {
		t.Errorf("Expected stats of %#v, got %#v", expStats, vb.stats)
	}

	expStatItems := make(chan statItem)
	actStatItems := make(chan statItem)

	go func() {
		expStats.Send(expStatItems)
		close(expStatItems)
	}()
	go func() {
		AggregateStats(testBucket, "").Send(actStatItems)
		close(actStatItems)
	}()

	for expitem := range expStatItems {
		actitem := <-actStatItems
		if expitem.key != actitem.key {
			t.Errorf("agg stats expected key %v, got %v",
				expitem.key, actitem.key)
		}
		if expitem.val != actitem.val {
			t.Errorf("agg stats expected val %v, got %v for key %v",
				expitem.val, actitem.val, expitem.key)
		}
	}
}
Beispiel #5
0
func TestArithOps(t *testing.T) {
	empty := []byte{}
	notempty := []byte("sentinel-for-not-empty")
	active := uint16(3)
	ignored := gomemcached.Status(32768)

	tests := []struct {
		op  gomemcached.CommandCode
		vb  uint16
		key string
		amt uint64
		def uint64

		expStatus gomemcached.Status
		expValue  []byte
	}{
		{gomemcached.INCREMENT, active, "a", 111, 222,
			gomemcached.SUCCESS, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 222}},
		{gomemcached.GET, active, "a", 0, 0,
			gomemcached.SUCCESS, []byte("222")},
		{gomemcached.INCREMENT, active, "a", 444, 333,
			gomemcached.SUCCESS, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x9a}},
		{gomemcached.GET, active, "a", 0, 0,
			gomemcached.SUCCESS, []byte("666")},
		{gomemcached.DECREMENT, active, "a", 555, 777,
			gomemcached.SUCCESS, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 111}},
		{gomemcached.GET, active, "a", 0, 0,
			gomemcached.SUCCESS, []byte("111")},

		// quiet
		{gomemcached.INCREMENTQ, active, "a", 10, 888,
			ignored, empty},
		{gomemcached.INCREMENTQ, active, "a", 3, 999,
			ignored, empty},
		{gomemcached.DECREMENTQ, active, "a", 1, 222,
			ignored, empty},
		{gomemcached.GET, active, "a", 0, 0,
			gomemcached.SUCCESS, []byte("123")},
	}

	expStats := Stats{
		Items:              1,
		Ops:                int64(len(tests)),
		Gets:               4,
		GetMisses:          0,
		Mutations:          6,
		Sets:               0,
		Adds:               0,
		Replaces:           0,
		Appends:            0,
		Prepends:           0,
		Incrs:              4,
		Decrs:              2,
		Deletes:            0,
		Creates:            1,
		Updates:            5,
		Unknowns:           0,
		IncomingValueBytes: 0,
		OutgoingValueBytes: 12,
		ItemBytes:          113,
	}

	testBucketDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(testBucketDir)
	testBucket, _ := NewBucket(testBucketDir,
		&BucketSettings{
			NumPartitions: MAX_VBUCKETS,
		})
	defer testBucket.Close()
	rh := reqHandler{currentBucket: testBucket}
	vb, _ := testBucket.CreateVBucket(3)
	testBucket.SetVBState(3, VBActive)

	for idx, x := range tests {
		req := &gomemcached.MCRequest{
			Opcode:  x.op,
			VBucket: x.vb,
			Key:     []byte(x.key),
			Body:    []byte{},
		}
		if x.op != gomemcached.GET {
			req.Extras = make([]byte, 8+8+4)
			binary.BigEndian.PutUint64(req.Extras[:8], x.amt)
			binary.BigEndian.PutUint64(req.Extras[8:16], x.def)
			binary.BigEndian.PutUint32(req.Extras[16:20], uint32(0))
		}

		res := rh.HandleMessage(ioutil.Discard, nil, req)

		if res == nil && x.expStatus == ignored {
			// this was a "normal" quiet command
			continue
		}

		if res.Status != x.expStatus {
			t.Errorf("Expected %v for %v -  %v:%v/%v, got %v",
				x.expStatus, idx, x.op, x.vb, x.key, res.Status)
		}

		if x.expValue != nil {
			if bytes.Equal(x.expValue, notempty) {
				if len(res.Body) <= 0 {
					t.Errorf("Expected non-empty body of %v - %v:%v/%v, got: %#v",
						idx, x.op, x.vb, x.key, res)
				}
			} else if !bytes.Equal(x.expValue, res.Body) {
				t.Errorf("Expected body of %v - %v:%v/%v to be\n%#v\ngot\n%#v",
					idx, x.op, x.vb, x.key, x.expValue, res.Body)
			}
		}
	}

	time.Sleep(10 * time.Millisecond) // Let async stats catch up.

	if !reflect.DeepEqual(&expStats, &vb.stats) {
		t.Errorf("Expected stats of %#v, got %#v", expStats, vb.stats)
	}

	expStatItems := make(chan statItem)
	actStatItems := make(chan statItem)

	go func() {
		expStats.Send(expStatItems)
		close(expStatItems)
	}()
	go func() {
		AggregateStats(testBucket, "").Send(actStatItems)
		close(actStatItems)
	}()

	for expitem := range expStatItems {
		actitem := <-actStatItems
		if expitem.key != actitem.key {
			t.Errorf("agg stats expected key %v, got %v",
				expitem.key, actitem.key)
		}
		if expitem.val != actitem.val {
			t.Errorf("agg stats expected val %v, got %v for key %v",
				expitem.val, actitem.val, expitem.key)
		}
	}
}
Beispiel #6
0
func TestMutationOps(t *testing.T) {
	empty := []byte{}
	notempty := []byte("sentinel-for-not-empty")
	active := uint16(3)
	ignored := gomemcached.Status(32768)

	tests := []struct {
		op  gomemcached.CommandCode
		vb  uint16
		key string
		val string

		expStatus gomemcached.Status
		expValue  []byte
	}{
		{gomemcached.REPLACE, active, "a", "irreplacable",
			gomemcached.KEY_ENOENT, notempty},
		{gomemcached.GET, active, "a", "",
			gomemcached.KEY_ENOENT, empty},
		{gomemcached.ADD, active, "a", "should-be-added",
			gomemcached.SUCCESS, empty},
		{gomemcached.GET, active, "a", "",
			gomemcached.SUCCESS, []byte("should-be-added")},
		{gomemcached.APPEND, active, "a", "_suffix",
			gomemcached.SUCCESS, empty},
		{gomemcached.GET, active, "a", "",
			gomemcached.SUCCESS, []byte("should-be-added_suffix")},
		{gomemcached.PREPEND, active, "a", "prefix_",
			gomemcached.SUCCESS, empty},
		{gomemcached.GET, active, "a", "",
			gomemcached.SUCCESS, []byte("prefix_should-be-added_suffix")},
		{gomemcached.REPLACE, active, "a", "replacement",
			gomemcached.SUCCESS, empty},
		{gomemcached.GET, active, "a", "",
			gomemcached.SUCCESS, []byte("replacement")},
		{gomemcached.ADD, active, "a", "not-added",
			gomemcached.KEY_EEXISTS, notempty},
		{gomemcached.GET, active, "a", "",
			gomemcached.SUCCESS, []byte("replacement")},

		// quiet
		{gomemcached.ADDQ, active, "a", "not-added",
			gomemcached.KEY_EEXISTS, notempty},
		{gomemcached.GET, active, "a", "",
			gomemcached.SUCCESS, []byte("replacement")},
		{gomemcached.REPLACEQ, active, "a", "replacement2",
			ignored, empty},
		{gomemcached.GET, active, "a", "",
			gomemcached.SUCCESS, []byte("replacement2")},
		{gomemcached.APPENDQ, active, "a", "_suffix2",
			ignored, empty},
		{gomemcached.PREPENDQ, active, "a", "prefix2_",
			ignored, empty},
		{gomemcached.GET, active, "a", "",
			gomemcached.SUCCESS, []byte("prefix2_replacement2_suffix2")},
		{gomemcached.DELETE, active, "a", "",
			gomemcached.SUCCESS, empty},
		{gomemcached.REPLACEQ, active, "a", "replacement2",
			gomemcached.KEY_ENOENT, notempty},
	}

	expStats := Stats{
		Items:              0,
		Ops:                int64(len(tests)),
		Gets:               9,
		GetMisses:          1,
		Mutations:          11,
		Sets:               0,
		Adds:               3,
		Replaces:           4,
		Appends:            2,
		Prepends:           2,
		Deletes:            1,
		Creates:            1,
		Updates:            6,
		Unknowns:           0,
		IncomingValueBytes: 68,
		OutgoingValueBytes: 139,
		ItemBytes:          110,
	}

	testBucketDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(testBucketDir)
	testBucket, _ := NewBucket(testBucketDir,
		&BucketSettings{
			NumPartitions: MAX_VBUCKETS,
		})
	defer testBucket.Close()
	rh := reqHandler{currentBucket: testBucket}
	vb, _ := testBucket.CreateVBucket(3)
	testBucket.SetVBState(3, VBActive)

	for _, x := range tests {
		req := &gomemcached.MCRequest{
			Opcode:  x.op,
			VBucket: x.vb,
			Key:     []byte(x.key),
			Body:    []byte(x.val),
		}

		res := rh.HandleMessage(ioutil.Discard, nil, req)

		if res == nil && x.expStatus == ignored {
			// this was a "normal" quiet command
			continue
		}

		if res.Status != x.expStatus {
			t.Errorf("Expected %v for %v:%v/%v, got %v",
				x.expStatus, x.op, x.vb, x.key, res.Status)
		}

		if x.expValue != nil {
			if bytes.Equal(x.expValue, notempty) {
				if len(res.Body) <= 0 {
					t.Errorf("Expected non-empty body of %v:%v/%v, got: %#v",
						x.op, x.vb, x.key, res)
				}
			} else if !bytes.Equal(x.expValue, res.Body) {
				t.Errorf("Expected body of %v:%v/%v to be\n%#v\ngot\n%#v",
					x.op, x.vb, x.key, x.expValue, res.Body)
			}
		}
	}

	time.Sleep(10 * time.Millisecond) // Let async stats catch up.

	if !reflect.DeepEqual(&expStats, &vb.stats) {
		t.Errorf("Expected stats of %#v, got %#v", expStats, vb.stats)
	}

	expStatItems := make(chan statItem)
	actStatItems := make(chan statItem)

	go func() {
		expStats.Send(expStatItems)
		close(expStatItems)
	}()
	go func() {
		AggregateStats(testBucket, "").Send(actStatItems)
		close(actStatItems)
	}()

	for expitem := range expStatItems {
		actitem := <-actStatItems
		if expitem.key != actitem.key {
			t.Errorf("agg stats expected key %v, got %v",
				expitem.key, actitem.key)
		}
		if expitem.val != actitem.val {
			t.Errorf("agg stats expected val %v, got %v for key %v",
				expitem.val, actitem.val, expitem.key)
		}
	}
}
Beispiel #7
0
// constants used for memcached protocol
const (
	uprOPEN        = mcd.CommandCode(0x50) // Open a upr connection with `name`
	uprAddSTREAM   = mcd.CommandCode(0x51) // Sent by ebucketmigrator to upr consumer
	uprCloseSTREAM = mcd.CommandCode(0x52) // Sent by ebucketmigrator to upr consumer
	uprFailoverLOG = mcd.CommandCode(0x54) // Request all known failover ids for restart
	uprStreamREQ   = mcd.CommandCode(0x53) // Stream request from consumer to producer
	uprStreamEND   = mcd.CommandCode(0x55) // Sent by producer when it is going to end stream
	uprSnapshotM   = mcd.CommandCode(0x56) // Sent by producer for a new snapshot
	uprMUTATION    = mcd.CommandCode(0x57) // Notifies SET/ADD/REPLACE/etc.  on the server
	uprDELETION    = mcd.CommandCode(0x58) // Notifies DELETE on the server
	uprEXPIRATION  = mcd.CommandCode(0x59) // Notifies key expiration
	uprFLUSH       = mcd.CommandCode(0x5a) // Notifies vbucket flush
)
const (
	rollBack = mcd.Status(0x23)
)

// FailoverLog is a slice of 2 element array, containing a list of,
// [[vuuid, sequence-no], [vuuid, sequence-no] ...]
type FailoverLog [][2]uint64

// UprStream will maintain stream information per vbucket
type UprStream struct {
	Vbucket  uint16 // vbucket id
	Vuuid    uint64 // vbucket uuid
	Opaque   uint32 // messages from producer to this stream have same value
	Highseq  uint64 // to be supplied by the application
	Startseq uint64 // to be supplied by the application
	Endseq   uint64 // to be supplied by the application
	Flog     FailoverLog