func (db *KVAutobus) RawPut(key storage.Key, value []byte) error {
b64key := encodeKey(key)
url := fmt.Sprintf("%s/kvautobus/api/value/%s/%s/", db.host, db.collection, b64key)
var bin Binary
if value == nil {
bin = Binary{}
} else {
bin = Binary(value)
}
storage.StoreKeyBytesWritten <- len(key)
storage.StoreValueBytesWritten <- len(value)
// Create pipe from encoding to posting
pr, pw := io.Pipe()
w := msgp.NewWriter(pw)
go func() {
bin.EncodeMsg(w)
w.Flush()
pw.Close()
}()
resp, err := db.client.Post(url, "application/x-msgpack", pr)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusConflict {
return fmt.Errorf("Can't POST to an already stored key. KVAutobus returned status %d (%s)", resp.StatusCode, url)
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("Bad status code returned (%d) from put request: %s", resp.StatusCode, url)
}
return nil
}
func (c *MsgpackDeltasCodec) WriteTo(w io.Writer) error {
c.encode()
mw := msgp.NewWriter(w)
err := c.docs.EncodeMsg(mw)
mw.Flush()
return err
}
func (db *KVAutobus) putRange(kvs []storage.KeyValue) error {
// Transform to an encodable type
mkvs := make(KVs, len(kvs))
for i, kv := range kvs {
storage.StoreKeyBytesWritten <- len(kv.K)
storage.StoreValueBytesWritten <- len(kv.V)
mkvs[i] = KV{Binary(kv.K), Binary(kv.V)}
}
// Create pipe from encoding to posting
pr, pw := io.Pipe()
w := msgp.NewWriter(pw)
go func() {
mkvs.EncodeMsg(w)
w.Flush()
pw.Close()
}()
// Send the data
url := fmt.Sprintf("%s/kvautobus/api/keyvalue_range/", db.host)
resp, err := http.Post(url, "application/x-msgpack", pr)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusConflict {
return fmt.Errorf("Can't POST to an already stored key. KVAutobus returned status %d (%s)", resp.StatusCode, url)
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("Bad status code returned (%d) from put range request: %s", resp.StatusCode, url)
}
return nil
}
func (db *KVAutobus) RawPut(key storage.Key, value []byte) error {
b64key := encodeKey(key)
url := fmt.Sprintf("%s/kvautobus/api/value/%s/", db.host, b64key)
bin := Binary(value)
dvid.Debugf("Begin RawPut on key %s (%d bytes)\n", hex.EncodeToString(key), len(bin))
// Create pipe from encoding to posting
pr, pw := io.Pipe()
w := msgp.NewWriter(pw)
go func() {
dvid.Debugf("Starting msgpack encoding...\n")
bin.EncodeMsg(w)
w.Flush()
pw.Close()
dvid.Debugf("Done msgpack encoding.\n")
}()
dvid.Debugf("Beginning POST to kvautobus: %s\n", url)
resp, err := http.Post(url, "application/x-msgpack", pr)
dvid.Debugf("Done POST with err %v\n", err)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusConflict {
return fmt.Errorf("Can't POST to an already stored key. KVAutobus returned status %d (%s)", resp.StatusCode, url)
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("Bad status code returned (%d) from put request: %s", resp.StatusCode, url)
}
return nil
}
func NewCodec(conn io.ReadWriteCloser) *codec {
c := &codec{
conn: conn,
r: msgp.NewReader(conn),
w: msgp.NewWriter(conn),
}
return c
}
func NewCodec(ws *websocket.Conn) *codec {
c := &codec{
ws: ws,
r: msgp.NewReader(nil),
w: msgp.NewWriter(nil),
}
return c
}
func NewCodec(db *redis.Database, channel string) *codec {
c := &codec{
db: db,
ch: channel,
r: msgp.NewReader(nil),
w: msgp.NewWriter(nil),
}
return c
}
func (s *testKinesisShard) AddRecord(sn SequenceNumber, rec map[string]interface{}) {
b := bytes.NewBuffer(make([]byte, 0, 1024))
w := msgp.NewWriter(b)
err := w.WriteMapStrIntf(rec)
if err != nil {
panic(err)
}
w.Flush()
rs := testKinesisRecords{sn, [][]byte{b.Bytes()}}
s.records = append(s.records, rs)
}
func (adp *AddDataPool) put(ad *AddData) {
ad.buffer.Reset()
ad.msgpBuffer.Reset()
if ad.buffer.Len() > gAddDataBufferMaxLen {
ad.buffer = &bytes.Buffer{}
}
if ad.msgpBuffer.Len() > gAddDataBufferMaxLen {
ad.msgpBuffer = &bytes.Buffer{}
ad.msgpWriter = msgp.NewWriter(ad.msgpBuffer)
}
adp.pool.Put(ad)
}
func BenchmarkIncidentEncode(b *testing.B) {
v := Incident{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func newAddData() interface{} {
ad := &AddData{
buffer: &bytes.Buffer{},
msgpBuffer: &bytes.Buffer{},
fnv1a: fnv.New32a(),
md5: md5.New(),
Key: "",
pmsg: &sarama.ProducerMessage{Topic: gBrokerTopic},
brokerDoneChan: make(chan int, 1),
}
ad.msgpWriter = msgp.NewWriter(ad.msgpBuffer)
return ad
}
func BenchmarkEncodebitmapContainerShortIterator(b *testing.B) {
v := bitmapContainerShortIterator{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkNotificationEncode(b *testing.B) {
r := Notification{
Method: "Call",
Body: nil,
}
var buf bytes.Buffer
w := msgp.NewWriter(&buf)
for i := 0; i < b.N; i++ {
r.EncodeMsg(w)
w.Flush()
buf.Reset()
}
}
func BenchmarkRequestEncode(b *testing.B) {
r := Request{
ID: 100,
Method: "Call",
Body: nil,
}
var buf bytes.Buffer
w := msgp.NewWriter(&buf)
for i := 0; i < b.N; i++ {
r.EncodeMsg(w)
w.Flush()
buf.Reset()
}
}
func BenchmarkResponseEncode(b *testing.B) {
r := Response{
ID: 100,
Error: "error",
Body: nil,
}
var buf bytes.Buffer
w := msgp.NewWriter(&buf)
for i := 0; i < b.N; i++ {
r.EncodeMsg(w)
w.Flush()
buf.Reset()
}
}
// benchmark encoding a small, "fast" type.
// the point here is to see how much garbage
// is generated intrinsically by the encoding/
// decoding process as opposed to the nature
// of the struct.
func BenchmarkFastEncode(b *testing.B) {
v := &TestFast{
Lat: 40.12398,
Long: -41.9082,
Alt: 201.08290,
Data: []byte("whaaaaargharbl"),
}
var buf bytes.Buffer
msgp.Encode(&buf, v)
en := msgp.NewWriter(msgp.Nowhere)
b.SetBytes(int64(buf.Len()))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func (d *Deleter) response(w http.ResponseWriter, statusCode int, errMsg string) {
var buf bytes.Buffer
buf.Reset()
res := map[string]interface{}{
"error": map[string]interface{}{
"errno": statusCode,
/* some msgpack lib cannot support new encoding-spec， so let len(string) < 31 */
// "errmsg": "success process: /del?topic=ap_",
"errmsg": "see errno please",
},
}
wr := msgp.NewWriter(&buf)
err := wr.WriteIntf(res)
if err != nil {
logger.Warning("fail to msgp.wr.WriteIntf: %s", err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
wr.Flush()
w.WriteHeader(statusCode)
_, err = w.Write(buf.Bytes())
if err != nil {
logger.Warning("fail to http.ResponseWriter.Write(): %s", err.Error())
return
}
// /* unpack test */
// b := bytes.NewReader(buf.Bytes())
// b.Seek(0, 0)
// msgr := msgp.NewReader(b)
// reqi, err := msgr.ReadIntf()
// if err != nil {
// msg := fmt.Sprintf("fail to decode post data: %s", err.Error())
// logger.Warning("%s", msg)
// return
// }
// logger.Warning("unpack: %v", reqi)
}
func (mc *MqClient) addKeys(keys []interface{}, doneServers map[string]interface{}, delTime, nextDelay int64, firstDel bool) (err error) {
/* pack data */
data := map[string]interface{}{
gDelKeys: keys,
gDelDelTime: delTime,
gDelCurDelay: nextDelay,
gDelFromMq: true,
gFirstDel: firstDel,
"method": fmt.Sprintf("%s%d", gDelMethodPrefix, nextDelay), // ktransfer need it
}
if len(doneServers) > 0 {
data[gDoneServers] = doneServers
}
var buf bytes.Buffer
wr := msgp.NewWriter(&buf)
err = wr.WriteIntf(data)
if err != nil {
logger.Warning("fail to msgp.WriteIntf: %s", err.Error())
return
}
wr.Flush()
r := bytes.NewReader(buf.Bytes())
/* write mq */
var mqServerAddrs []string
if firstDel && len(mc.idcMqServerAddrs) > 0 {
mqServerAddrs = mc.idcMqServerAddrs
} else {
mqServerAddrs = mc.delMqServerAddrs
}
toc := make(chan bool, 1)
go mc.checkMqTimeout(toc)
ns := len(mqServerAddrs)
mc.lastServerId = (mc.lastServerId + 1) % ns
i := 0
for {
r.Seek(0, 0)
url := fmt.Sprintf("http://%s?topic=%s&method=%s%d", mqServerAddrs[mc.lastServerId], mc.clusterName, gDelMethodPrefix, nextDelay)
req, err1 := http.NewRequest("POST", url, r)
if err1 != nil {
logger.Warning("fail to http.NewRequest: %s, %s", url, err1.Error())
err = err1
return
}
rsp, err1 := mc.client.Do(req)
if err1 != nil {
logger.Warning("fail to http.Client.Do: %s, %s", url, err1.Error())
select {
case <-toc:
err = errors.New("add back to mq timeout")
return
default:
/* do nothing */
}
i++
if i == ns {
err = errors.New("fail to add back to mq, all servers have been retried")
return
}
mc.lastServerId = (mc.lastServerId + 1) % ns
continue
} else {
rsp.Body.Close()
logger.Notice("success add back to mq: %s", url)
return nil
}
}
}
