Example #1
0
func (db *KVAutobus) RawPut(key storage.Key, value []byte) error {
	b64key := encodeKey(key)
	url := fmt.Sprintf("%s/kvautobus/api/value/%s/%s/", db.host, db.collection, b64key)
	var bin Binary
	if value == nil {
		bin = Binary{}
	} else {
		bin = Binary(value)
	}

	storage.StoreKeyBytesWritten <- len(key)
	storage.StoreValueBytesWritten <- len(value)

	// Create pipe from encoding to posting
	pr, pw := io.Pipe()
	w := msgp.NewWriter(pw)
	go func() {
		bin.EncodeMsg(w)
		w.Flush()
		pw.Close()
	}()

	resp, err := db.client.Post(url, "application/x-msgpack", pr)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusConflict {
		return fmt.Errorf("Can't POST to an already stored key.  KVAutobus returned status %d (%s)", resp.StatusCode, url)
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Bad status code returned (%d) from put request: %s", resp.StatusCode, url)
	}
	return nil
}
Example #2
0
func (c *MsgpackDeltasCodec) WriteTo(w io.Writer) error {
	c.encode()
	mw := msgp.NewWriter(w)
	err := c.docs.EncodeMsg(mw)
	mw.Flush()
	return err
}
Example #3
0
func (db *KVAutobus) putRange(kvs []storage.KeyValue) error {
	// Transform to an encodable type
	mkvs := make(KVs, len(kvs))
	for i, kv := range kvs {
		storage.StoreKeyBytesWritten <- len(kv.K)
		storage.StoreValueBytesWritten <- len(kv.V)
		mkvs[i] = KV{Binary(kv.K), Binary(kv.V)}
	}

	// Create pipe from encoding to posting
	pr, pw := io.Pipe()
	w := msgp.NewWriter(pw)
	go func() {
		mkvs.EncodeMsg(w)
		w.Flush()
		pw.Close()
	}()

	// Send the data
	url := fmt.Sprintf("%s/kvautobus/api/keyvalue_range/", db.host)
	resp, err := http.Post(url, "application/x-msgpack", pr)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusConflict {
		return fmt.Errorf("Can't POST to an already stored key.  KVAutobus returned status %d (%s)", resp.StatusCode, url)
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Bad status code returned (%d) from put range request: %s", resp.StatusCode, url)
	}
	return nil
}
Example #4
0
func (db *KVAutobus) RawPut(key storage.Key, value []byte) error {
	b64key := encodeKey(key)
	url := fmt.Sprintf("%s/kvautobus/api/value/%s/", db.host, b64key)
	bin := Binary(value)

	dvid.Debugf("Begin RawPut on key %s (%d bytes)\n", hex.EncodeToString(key), len(bin))

	// Create pipe from encoding to posting
	pr, pw := io.Pipe()
	w := msgp.NewWriter(pw)
	go func() {
		dvid.Debugf("Starting msgpack encoding...\n")
		bin.EncodeMsg(w)
		w.Flush()
		pw.Close()
		dvid.Debugf("Done msgpack encoding.\n")
	}()

	dvid.Debugf("Beginning POST to kvautobus: %s\n", url)
	resp, err := http.Post(url, "application/x-msgpack", pr)
	dvid.Debugf("Done POST with err %v\n", err)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusConflict {
		return fmt.Errorf("Can't POST to an already stored key.  KVAutobus returned status %d (%s)", resp.StatusCode, url)
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Bad status code returned (%d) from put request: %s", resp.StatusCode, url)
	}
	return nil
}
Example #5
0
func NewCodec(conn io.ReadWriteCloser) *codec {
	c := &codec{
		conn: conn,
		r:    msgp.NewReader(conn),
		w:    msgp.NewWriter(conn),
	}
	return c
}
Example #6
0
func NewCodec(ws *websocket.Conn) *codec {
	c := &codec{
		ws: ws,
		r:  msgp.NewReader(nil),
		w:  msgp.NewWriter(nil),
	}
	return c
}
Example #7
0
func NewCodec(db *redis.Database, channel string) *codec {
	c := &codec{
		db: db,
		ch: channel,
		r:  msgp.NewReader(nil),
		w:  msgp.NewWriter(nil),
	}
	return c
}
Example #8
0
func (s *testKinesisShard) AddRecord(sn SequenceNumber, rec map[string]interface{}) {
	b := bytes.NewBuffer(make([]byte, 0, 1024))
	w := msgp.NewWriter(b)
	err := w.WriteMapStrIntf(rec)
	if err != nil {
		panic(err)
	}
	w.Flush()
	rs := testKinesisRecords{sn, [][]byte{b.Bytes()}}
	s.records = append(s.records, rs)
}
Example #9
0
func (adp *AddDataPool) put(ad *AddData) {
	ad.buffer.Reset()
	ad.msgpBuffer.Reset()
	if ad.buffer.Len() > gAddDataBufferMaxLen {
		ad.buffer = &bytes.Buffer{}
	}
	if ad.msgpBuffer.Len() > gAddDataBufferMaxLen {
		ad.msgpBuffer = &bytes.Buffer{}
		ad.msgpWriter = msgp.NewWriter(ad.msgpBuffer)
	}
	adp.pool.Put(ad)
}
Example #10
0
func BenchmarkIncidentEncode(b *testing.B) {
	v := Incident{}
	var buf bytes.Buffer
	msgp.Encode(&buf, &v)
	b.SetBytes(int64(buf.Len()))
	en := msgp.NewWriter(msgp.Nowhere)
	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		v.EncodeMsg(en)
	}
	en.Flush()
}
Example #11
0
func newAddData() interface{} {
	ad := &AddData{
		buffer:         &bytes.Buffer{},
		msgpBuffer:     &bytes.Buffer{},
		fnv1a:          fnv.New32a(),
		md5:            md5.New(),
		Key:            "",
		pmsg:           &sarama.ProducerMessage{Topic: gBrokerTopic},
		brokerDoneChan: make(chan int, 1),
	}
	ad.msgpWriter = msgp.NewWriter(ad.msgpBuffer)
	return ad
}
func BenchmarkEncodebitmapContainerShortIterator(b *testing.B) {
	v := bitmapContainerShortIterator{}
	var buf bytes.Buffer
	msgp.Encode(&buf, &v)
	b.SetBytes(int64(buf.Len()))
	en := msgp.NewWriter(msgp.Nowhere)
	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		v.EncodeMsg(en)
	}
	en.Flush()
}
Example #13
0
func BenchmarkNotificationEncode(b *testing.B) {
	r := Notification{
		Method: "Call",
		Body:   nil,
	}

	var buf bytes.Buffer
	w := msgp.NewWriter(&buf)

	for i := 0; i < b.N; i++ {
		r.EncodeMsg(w)
		w.Flush()
		buf.Reset()
	}
}
Example #14
0
func BenchmarkRequestEncode(b *testing.B) {
	r := Request{
		ID:     100,
		Method: "Call",
		Body:   nil,
	}

	var buf bytes.Buffer
	w := msgp.NewWriter(&buf)

	for i := 0; i < b.N; i++ {
		r.EncodeMsg(w)
		w.Flush()
		buf.Reset()
	}
}
Example #15
0
func BenchmarkResponseEncode(b *testing.B) {
	r := Response{
		ID:    100,
		Error: "error",
		Body:  nil,
	}

	var buf bytes.Buffer
	w := msgp.NewWriter(&buf)

	for i := 0; i < b.N; i++ {
		r.EncodeMsg(w)
		w.Flush()
		buf.Reset()
	}
}
Example #16
0
// benchmark encoding a small, "fast" type.
// the point here is to see how much garbage
// is generated intrinsically by the encoding/
// decoding process as opposed to the nature
// of the struct.
func BenchmarkFastEncode(b *testing.B) {
	v := &TestFast{
		Lat:  40.12398,
		Long: -41.9082,
		Alt:  201.08290,
		Data: []byte("whaaaaargharbl"),
	}
	var buf bytes.Buffer
	msgp.Encode(&buf, v)
	en := msgp.NewWriter(msgp.Nowhere)
	b.SetBytes(int64(buf.Len()))
	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		v.EncodeMsg(en)
	}
	en.Flush()
}
Example #17
0
func (d *Deleter) response(w http.ResponseWriter, statusCode int, errMsg string) {
	var buf bytes.Buffer
	buf.Reset()
	res := map[string]interface{}{
		"error": map[string]interface{}{
			"errno": statusCode,
			/* some msgpack lib cannot support new encoding-spec, so let len(string) < 31 */
			//			"errmsg": "success process: /del?topic=ap_",
			"errmsg": "see errno please",
		},
	}
	wr := msgp.NewWriter(&buf)
	err := wr.WriteIntf(res)
	if err != nil {
		logger.Warning("fail to msgp.wr.WriteIntf: %s", err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	wr.Flush()
	w.WriteHeader(statusCode)
	_, err = w.Write(buf.Bytes())
	if err != nil {
		logger.Warning("fail to http.ResponseWriter.Write(): %s", err.Error())
		return
	}
	//	/* unpack test */
	//	b := bytes.NewReader(buf.Bytes())
	//	b.Seek(0, 0)
	//	msgr := msgp.NewReader(b)
	//	reqi, err := msgr.ReadIntf()
	//	if err != nil {
	//        msg := fmt.Sprintf("fail to decode post data: %s", err.Error())
	//		logger.Warning("%s", msg)
	//		return
	//	}
	//	logger.Warning("unpack: %v", reqi)
}
Example #18
0
func (mc *MqClient) addKeys(keys []interface{}, doneServers map[string]interface{}, delTime, nextDelay int64, firstDel bool) (err error) {
	/* pack data */
	data := map[string]interface{}{
		gDelKeys:     keys,
		gDelDelTime:  delTime,
		gDelCurDelay: nextDelay,
		gDelFromMq:   true,
		gFirstDel:    firstDel,
		"method":     fmt.Sprintf("%s%d", gDelMethodPrefix, nextDelay), // ktransfer need it
	}
	if len(doneServers) > 0 {
		data[gDoneServers] = doneServers
	}
	var buf bytes.Buffer
	wr := msgp.NewWriter(&buf)
	err = wr.WriteIntf(data)
	if err != nil {
		logger.Warning("fail to msgp.WriteIntf: %s", err.Error())
		return
	}
	wr.Flush()
	r := bytes.NewReader(buf.Bytes())
	/* write mq */
	var mqServerAddrs []string
	if firstDel && len(mc.idcMqServerAddrs) > 0 {
		mqServerAddrs = mc.idcMqServerAddrs
	} else {
		mqServerAddrs = mc.delMqServerAddrs
	}
	toc := make(chan bool, 1)
	go mc.checkMqTimeout(toc)
	ns := len(mqServerAddrs)
	mc.lastServerId = (mc.lastServerId + 1) % ns
	i := 0
	for {
		r.Seek(0, 0)
		url := fmt.Sprintf("http://%s?topic=%s&method=%s%d", mqServerAddrs[mc.lastServerId], mc.clusterName, gDelMethodPrefix, nextDelay)
		req, err1 := http.NewRequest("POST", url, r)
		if err1 != nil {
			logger.Warning("fail to http.NewRequest: %s, %s", url, err1.Error())
			err = err1
			return
		}
		rsp, err1 := mc.client.Do(req)
		if err1 != nil {
			logger.Warning("fail to http.Client.Do: %s, %s", url, err1.Error())
			select {
			case <-toc:
				err = errors.New("add back to mq timeout")
				return
			default:
				/* do nothing */
			}
			i++
			if i == ns {
				err = errors.New("fail to add back to mq, all servers have been retried")
				return
			}
			mc.lastServerId = (mc.lastServerId + 1) % ns
			continue
		} else {
			rsp.Body.Close()
			logger.Notice("success add back to mq: %s", url)
			return nil
		}
	}
}