func (t *LeaseMetadata) Marshal(wire io.Writer) {
var b [10]byte
var bs []byte
bs = b[:]
alen1 := int64(len(t.Quorum))
if wlen := binary.PutVarint(bs, alen1); wlen >= 0 {
wire.Write(b[0:wlen])
}
for i := int64(0); i < alen1; i++ {
bs = b[:4]
tmp32 := t.Quorum[i]
bs[0] = byte(tmp32)
bs[1] = byte(tmp32 >> 8)
bs[2] = byte(tmp32 >> 16)
bs[3] = byte(tmp32 >> 24)
wire.Write(bs)
}
bs = b[:]
alen2 := int64(len(t.ObjectKeys))
if wlen := binary.PutVarint(bs, alen2); wlen >= 0 {
wire.Write(b[0:wlen])
}
for i := int64(0); i < alen2; i++ {
t.ObjectKeys[i].Marshal(wire)
}
bs = b[:2]
bs[0] = byte(t.IgnoreReplicas)
bs[1] = byte(t.ReinstateReplicas)
wire.Write(bs)
}
func (b *BoltDB) Increment() int64 {
buf := make([]byte, 8)
var retVal int64
err := b.db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists(demoBucket)
if err != nil {
log.Fatal(err)
}
v := b.Get(demoKey)
if v == nil {
binary.PutVarint(buf, 0)
retVal = 0
} else {
i, err := binary.ReadVarint(bytes.NewBuffer(v))
if err != nil {
log.Fatal(err)
}
i++
retVal = i
binary.PutVarint(buf, i)
}
err = b.Put(demoKey, buf)
return err
})
if err != nil {
log.Fatal(err)
}
return retVal
}
func (b *binaryNomsWriter) writeNumber(v Number) {
b.ensureCapacity(binary.MaxVarintLen64 * 2)
i, exp := float64ToIntExp(float64(v))
count := binary.PutVarint(b.buff[b.offset:], i)
b.offset += uint32(count)
count = binary.PutVarint(b.buff[b.offset:], int64(exp))
b.offset += uint32(count)
}
func zoneFilename(x, y int64, z int8) string {
var buf [binary.MaxVarintLen64*2 + 1]byte
i := binary.PutVarint(buf[:], x)
i += binary.PutVarint(buf[i:], y)
buf[i] = uint8(z)
i++
encoded := base32.StdEncoding.EncodeToString(buf[:i])
return filepath.Join("rnoadm-AA", "zone"+encoded+".gz")
}
func putFunc(cmd *cobra.Command, args []string) {
if keySpaceSize <= 0 {
fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize)
os.Exit(1)
}
requests := make(chan v3.Op, totalClients)
clients := mustCreateClients(totalClients, totalConns)
k, v := make([]byte, keySize), string(mustRandBytes(valSize))
bar = pb.New(putTotal)
bar.Format("Bom !")
bar.Start()
r := newReport()
for i := range clients {
wg.Add(1)
go func(c *v3.Client) {
defer wg.Done()
for op := range requests {
st := time.Now()
_, err := c.Do(context.Background(), op)
r.Results() <- report.Result{Err: err, Start: st, End: time.Now()}
bar.Increment()
}
}(clients[i])
}
go func() {
for i := 0; i < putTotal; i++ {
if seqKeys {
binary.PutVarint(k, int64(i%keySpaceSize))
} else {
binary.PutVarint(k, int64(rand.Intn(keySpaceSize)))
}
requests <- v3.OpPut(string(k), v)
}
close(requests)
}()
if compactInterval > 0 {
go func() {
for {
time.Sleep(compactInterval)
compactKV(clients)
}
}()
}
rc := r.Run()
wg.Wait()
close(r.Results())
bar.Finish()
fmt.Println(<-rc)
}
func putFunc(cmd *cobra.Command, args []string) {
if keySpaceSize <= 0 {
fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize)
os.Exit(1)
}
results = make(chan result)
requests := make(chan v3.Op, totalClients)
bar = pb.New(putTotal)
k, v := make([]byte, keySize), string(mustRandBytes(valSize))
clients := mustCreateClients(totalClients, totalConns)
bar.Format("Bom !")
bar.Start()
for i := range clients {
wg.Add(1)
go doPut(context.Background(), clients[i], requests)
}
pdoneC := printReport(results)
go func() {
for i := 0; i < putTotal; i++ {
if seqKeys {
binary.PutVarint(k, int64(i%keySpaceSize))
} else {
binary.PutVarint(k, int64(rand.Intn(keySpaceSize)))
}
requests <- v3.OpPut(string(k), v)
}
close(requests)
}()
if compactInterval > 0 {
go func() {
for {
time.Sleep(compactInterval)
compactKV(clients)
}
}()
}
wg.Wait()
bar.Finish()
close(results)
<-pdoneC
}
func zoneFilenameV1(x, y int64) string {
var buf [binary.MaxVarintLen64 * 2]byte
i := binary.PutVarint(buf[:], x)
i += binary.PutVarint(buf[i:], y)
encoded := base32.StdEncoding.EncodeToString(buf[:i])
for i := range encoded {
if encoded[i] == '=' {
encoded = encoded[:i]
break
}
}
return filepath.Join("rnoadm-AA", "zone"+encoded+".gz")
}
func putFunc(cmd *cobra.Command, args []string) {
if keySpaceSize <= 0 {
fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize)
os.Exit(1)
}
results = make(chan result)
requests := make(chan etcdserverpb.PutRequest, totalClients)
bar = pb.New(putTotal)
k, v := make([]byte, keySize), mustRandBytes(valSize)
conns := make([]*grpc.ClientConn, totalConns)
for i := range conns {
conns[i] = mustCreateConn()
}
clients := make([]etcdserverpb.KVClient, totalClients)
for i := range clients {
clients[i] = etcdserverpb.NewKVClient(conns[i%int(totalConns)])
}
bar.Format("Bom !")
bar.Start()
for i := range clients {
wg.Add(1)
go doPut(context.Background(), clients[i], requests)
}
pdoneC := printReport(results)
go func() {
for i := 0; i < putTotal; i++ {
if seqKeys {
binary.PutVarint(k, int64(i%keySpaceSize))
} else {
binary.PutVarint(k, int64(rand.Intn(keySpaceSize)))
}
requests <- etcdserverpb.PutRequest{Key: k, Value: v}
}
close(requests)
}()
wg.Wait()
bar.Finish()
close(results)
<-pdoneC
}
func (e *CacheEntry) Encode() []byte {
hl := 0
for key, vals := range e.Header {
hl = hl + binary.MaxVarintLen32*2 + len(key)
for _, v := range vals {
hl += binary.MaxVarintLen32 + len(v)
}
}
size := 5*binary.MaxVarintLen32 + len(e.Body) + hl
buffer := make([]byte, size)
n := 0
n += binary.PutVarint(buffer[n:], int64(e.Status))
n += binary.PutVarint(buffer[n:], int64(e.AddTime))
n += binary.PutVarint(buffer[n:], int64(e.Expire))
n += binary.PutVarint(buffer[n:], int64(len(e.Body)))
n += copy(buffer[n:], e.Body)
n += binary.PutVarint(buffer[n:], int64(len(e.Header)))
for key, vals := range e.Header {
n += binary.PutVarint(buffer[n:], int64(len(key)))
n += copy(buffer[n:], key)
n += binary.PutVarint(buffer[n:], int64(len(vals)))
for _, v := range vals {
n += binary.PutVarint(buffer[n:], int64(len(v)))
n += copy(buffer[n:], v)
}
}
return buffer[:n]
}
func MarshalDeltaNodes(nodes []element.Node, buf []byte) []byte {
estimatedLength := len(nodes)*4*3 + binary.MaxVarintLen64
if cap(buf) < estimatedLength {
buf = make([]byte, estimatedLength)
} else {
// resize slice to full capacity
buf = buf[:cap(buf)-1]
}
lastId := int64(0)
nextPos := binary.PutUvarint(buf, uint64(len(nodes)))
for i := range nodes {
if len(buf)-nextPos < binary.MaxVarintLen64 {
tmp := make([]byte, len(buf)*3/2)
copy(tmp, buf)
buf = tmp
}
nextPos += binary.PutVarint(buf[nextPos:], nodes[i].Id-lastId)
lastId = nodes[i].Id
}
lastLong := int64(0)
for i := range nodes {
if len(buf)-nextPos < binary.MaxVarintLen64 {
tmp := make([]byte, len(buf)*3/2)
copy(tmp, buf)
buf = tmp
}
long := int64(CoordToInt(nodes[i].Long))
nextPos += binary.PutVarint(buf[nextPos:], long-lastLong)
lastLong = long
}
lastLat := int64(0)
for i := range nodes {
if len(buf)-nextPos < binary.MaxVarintLen64 {
tmp := make([]byte, len(buf)*3/2)
copy(tmp, buf)
buf = tmp
}
lat := int64(CoordToInt(nodes[i].Lat))
nextPos += binary.PutVarint(buf[nextPos:], lat-lastLat)
lastLat = lat
}
return buf[:nextPos]
}
func TestIntSliceEncode(t *testing.T) {
var result []byte
result = strconv.AppendInt(result, 124, 10)
if string(result) != "124" {
t.Errorf("%v\n", result)
}
result = strconv.AppendInt(result, 12, 10)
another := make([]byte, 14)
nul := binary.PutVarint(another, 2)
nul = binary.PutVarint(another[4:], 111111111)
if nul < 0 {
t.Error()
}
}
// Create and chain a new bucket.
func (ht *HashTable) growBucket(bucket int) {
ht.EnsureSize(BUCKET_SIZE)
lastBucketAddr := ht.lastBucket(bucket) * BUCKET_SIZE
binary.PutVarint(ht.Buf[lastBucketAddr:lastBucketAddr+10], int64(ht.numBuckets))
ht.Used += BUCKET_SIZE
ht.numBuckets++
}
func handleWrite(c net.Conn) {
for angle := range out {
m := Message{angle}
if buf, err := json.Marshal(m); err != nil {
log.Printf("Could not json encode angle: %+v\n", m)
} else {
var frame []byte
mode := byte(0x81)
mask := byte(0x80)
if len(buf) <= 125 {
// can fit in the mask
frame = []byte{mode, mask | byte(len(buf))}
} else if len(buf) > 0xffff {
// needs 8 extra bytes
frame = make([]byte, 10)
frame[0] = mode
frame[1] = mask | 127
// for now, len is 32-bit
// TODO: update when 64-bit is supported
// this should never happen anyway
binary.PutVarint(frame[2:], int64(len(buf)))
} else {
// needs 2 extra bytes
frame = []byte{
mode,
mask | 128,
byte(len(buf)&0xff00) >> 8,
byte(len(buf) & 0xff)}
}
c.Write(append(frame, buf...))
}
}
}
func (rv *rollingValueHasher) hashVarint(n int64) {
buff := [binary.MaxVarintLen64]byte{}
count := binary.PutVarint(buff[:], n)
for i := 0; i < count; i++ {
rv.HashByte(buff[i])
}
}
func (v *Verifier) VerifySpace(challenges []int64, hashes [][]byte, parents [][][]byte, proofs [][][]byte, pProofs [][][][]byte) bool {
for i := range challenges {
buf := make([]byte, hashSize)
binary.PutVarint(buf, challenges[i]+v.pow2)
val := append(v.pk, buf...)
for _, ph := range parents[i] {
val = append(val, ph...)
}
exp := sha3.Sum256(val)
for j := range exp {
if exp[j] != hashes[i][j] {
return false
}
}
if !v.Verify(challenges[i], hashes[i], proofs[i]) {
return false
}
ps := v.graph.GetParents(challenges[i], v.index)
for j := range ps {
if !v.Verify(ps[j], parents[i][j], pProofs[i][j]) {
return false
}
}
}
return true
}
func IntToBytes(v int64) []byte {
buf := make([]byte, indexOffsetSize)
binary.PutVarint(buf, v)
return buf
}
func (s *Signer) Sign(msg []byte) []byte {
b := make([]byte, 8+len(msg))
binary.PutVarint(b, int64(len(msg)))
b = append(b[:8], msg...)
b = append(b, s.signature(b)...)
return b
}
// TODO: verify. this needs to be audited
// we start with crypt/rand, then XOR in additional entropy from OS
func GetEntropyMixed(n int) []byte {
startTime := time.Now().UnixNano()
// for each source, we take SHA3 of the source and use it as seed to math/rand
// then read bytes from it and XOR them onto the bytes read from crypto/rand
mainBuff := GetEntropyCSPRNG(n)
// 1. OS entropy sources
startTimeBytes := make([]byte, 32)
binary.PutVarint(startTimeBytes, startTime)
startTimeHash := Sha3(startTimeBytes)
mixBytes(mainBuff, startTimeHash)
pid := os.Getpid()
pidBytes := make([]byte, 32)
binary.PutUvarint(pidBytes, uint64(pid))
pidHash := Sha3(pidBytes)
mixBytes(mainBuff, pidHash)
osEnv := os.Environ()
osEnvBytes := []byte(strings.Join(osEnv, ""))
osEnvHash := Sha3(osEnvBytes)
mixBytes(mainBuff, osEnvHash)
// not all OS have hostname in env variables
osHostName, err := os.Hostname()
if err != nil {
osHostNameBytes := []byte(osHostName)
osHostNameHash := Sha3(osHostNameBytes)
mixBytes(mainBuff, osHostNameHash)
}
return mainBuff
}
// increment fetches the varint encoded int64 value specified by key
// and adds "inc" to it then re-encodes as varint and puts the new
// value to key using the timestamp "ts". The newly incremented value
// is returned.
func increment(engine Engine, key Key, inc int64, ts int64) (int64, error) {
// First retrieve existing value.
val, err := engine.get(key)
if err != nil {
return 0, err
}
var int64Val int64
// If the value exists, attempt to decode it as a varint.
if len(val.Bytes) != 0 {
var numBytes int
int64Val, numBytes = binary.Varint(val.Bytes)
if numBytes == 0 {
return 0, util.Errorf("key %q cannot be incremented; not varint-encoded", key)
} else if numBytes < 0 {
return 0, util.Errorf("key %q cannot be incremented; integer overflow", key)
}
}
// Check for overflow and underflow.
r := int64Val + inc
if (r < int64Val) != (inc < 0) {
return 0, util.Errorf("key %q with value %d incremented by %d results in overflow", key, int64Val, inc)
}
encoded := make([]byte, binary.MaxVarintLen64)
numBytes := binary.PutVarint(encoded, r)
encoded = encoded[:numBytes]
if err = engine.put(key, Value{Bytes: encoded, Timestamp: ts}); err != nil {
return 0, err
}
return r, nil
}
// UUID returns a global unique identifier for the given literal. It is
// implemented as the SHA1 UUID of the literal value.
func (l *Literal) UUID() uuid.UUID {
var buffer bytes.Buffer
switch v := l.v.(type) {
case bool:
if v {
buffer.WriteString("true")
} else {
buffer.WriteString("false")
}
case int64:
b := make([]byte, 8)
binary.PutVarint(b, v)
buffer.Write(b)
case float64:
bs := math.Float64bits(v)
b := make([]byte, 8)
binary.LittleEndian.PutUint64(b, bs)
buffer.Write(b)
case string:
buffer.Write([]byte(v))
case []byte:
buffer.Write(v)
}
return uuid.NewSHA1(uuid.NIL, buffer.Bytes())
}
func (c *Cache) SetVersion(version int) error {
return c.db.Update(func(tx *bolt.Tx) error {
buf := make([]byte, 10)
n := binary.PutVarint(buf, int64(version))
return tx.Bucket(geoMetaBucket).Put([]byte("version"), buf[:n])
})
}
func intToBuf(data int32) []byte {
buf := make([]byte, 8)
if wrote := binary.PutVarint(buf, int64(data)); wrote < 1 {
panic(fmt.Sprintf("failed to write %i in CMS insert, was %i", data, wrote))
}
return buf
}
func (state *ComputePeerState) Sync(q chan int) {
// Subscribe to HELO messages
state.SubSock.Subscribe([]byte("HELO"))
//fmt.Println("Starting to wait to receive HELO")
// Wait to receive one HELO message
_, err := state.SubSock.Recv()
if err != nil {
//fmt.Println("Receiving over subscription socket failed", err)
q <- 1
return
}
//fmt.Println("Received HELO, client ", int64(state.Client))
// HELO messages are now useless, unsubscribe
state.SubSock.Unsubscribe([]byte("HELO"))
// Inform master of our presence on the network
resp := make([][]byte, 1)
resp[0] = make([]byte, binary.MaxVarintLen32)
binary.PutVarint(resp[0], int64(state.Client))
r, _ := binary.Varint(resp[0])
_ = r
//fmt.Printf("Client = %d\n", r)
err = state.CoordSock.Send(resp)
if err != nil {
//fmt.Println("Error sending on coordination socket", err)
q <- 1
}
}
func stress(mb int) error {
time.Sleep(5 * time.Second)
cfg := client.Config{
Endpoints: []string{"http://localhost:12379", "http://localhost:22379", "http://localhost:32379"},
Transport: client.DefaultTransport,
// set timeout per request to fail fast when the target endpoint is unavailable
HeaderTimeoutPerRequest: time.Second,
}
c, err := client.New(cfg)
if err != nil {
return err
}
kapi := client.NewKeysAPI(c)
for i := 0; i < mb*2; i++ {
fmt.Println("stressing", i)
k := make([]byte, 100)
binary.PutVarint(k, int64(rand.Intn(putSize)))
_, err = kapi.Set(context.Background(), string(k), "", nil)
if err != nil {
if i < 2 {
return err
}
}
time.Sleep(500 * time.Millisecond)
}
return nil
}
func (t *PrepareReply) Marshal(wire io.Writer) {
var b [10]byte
var bs []byte
bs = b[:9]
tmp32 := t.Balnum
bs[0] = byte(tmp32)
bs[1] = byte(tmp32 >> 8)
bs[2] = byte(tmp32 >> 16)
bs[3] = byte(tmp32 >> 24)
bs[4] = byte(t.OK)
tmp32 = t.Ballot
bs[5] = byte(tmp32)
bs[6] = byte(tmp32 >> 8)
bs[7] = byte(tmp32 >> 16)
bs[8] = byte(tmp32 >> 24)
wire.Write(bs)
bs = b[:]
alen1 := int64(len(t.Cstruct))
if wlen := binary.PutVarint(bs, alen1); wlen >= 0 {
wire.Write(b[0:wlen])
}
for i := int64(0); i < alen1; i++ {
bs = b[:4]
tmp32 = t.Cstruct[i]
bs[0] = byte(tmp32)
bs[1] = byte(tmp32 >> 8)
bs[2] = byte(tmp32 >> 16)
bs[3] = byte(tmp32 >> 24)
wire.Write(bs)
}
}
func (t *Commit) Marshal(wire io.Writer) {
var b [12]byte
var bs []byte
bs = b[:12]
tmp32 := t.LeaderId
bs[0] = byte(tmp32)
bs[1] = byte(tmp32 >> 8)
bs[2] = byte(tmp32 >> 16)
bs[3] = byte(tmp32 >> 24)
tmp32 = t.Instance
bs[4] = byte(tmp32)
bs[5] = byte(tmp32 >> 8)
bs[6] = byte(tmp32 >> 16)
bs[7] = byte(tmp32 >> 24)
tmp32 = t.Ballot
bs[8] = byte(tmp32)
bs[9] = byte(tmp32 >> 8)
bs[10] = byte(tmp32 >> 16)
bs[11] = byte(tmp32 >> 24)
wire.Write(bs)
bs = b[:]
alen1 := int64(len(t.LeaseUpdate))
if wlen := binary.PutVarint(bs, alen1); wlen >= 0 {
wire.Write(b[0:wlen])
}
for i := int64(0); i < alen1; i++ {
t.LeaseUpdate[i].Marshal(wire)
}
}
// Encrypts and writes to the writer outputing the following structure:
// The first two bytes are the length(x) of the encrypted message.
// The next 24 bytes are a random nonce.
// The next x bytes is the encrypted message.
func (s *SecureWriter) Write(p []byte) (int, error) {
nonce, err := generateNonce()
if err != nil {
return 0, err
}
payload := box.Seal(nil, p, nonce, s.pub, s.priv)
plen := make([]byte, 2)
binary.PutVarint(plen, int64(len(payload)))
ew := &errWriter{w: s.writer}
ew.write(plen)
ew.write(nonce[:])
ew.write(payload)
if ew.err != nil {
// If an error occurred, bytes may have been written
// but they do not correspond to bytes of `p` written,
// which is what the io.Writer interface cares about.
// SecureWriter is pretty much all or nothing.
return 0, ew.err
}
return len(p), ew.err
}
func (t *Commit) Marshal(wire io.Writer) {
var b [24]byte
var bs []byte
bs = b[:12]
tmp32 := t.LeaderId
bs[0] = byte(tmp32)
bs[1] = byte(tmp32 >> 8)
bs[2] = byte(tmp32 >> 16)
bs[3] = byte(tmp32 >> 24)
tmp32 = t.Replica
bs[4] = byte(tmp32)
bs[5] = byte(tmp32 >> 8)
bs[6] = byte(tmp32 >> 16)
bs[7] = byte(tmp32 >> 24)
tmp32 = t.Instance
bs[8] = byte(tmp32)
bs[9] = byte(tmp32 >> 8)
bs[10] = byte(tmp32 >> 16)
bs[11] = byte(tmp32 >> 24)
wire.Write(bs)
bs = b[:]
alen1 := int64(len(t.Command))
if wlen := binary.PutVarint(bs, alen1); wlen >= 0 {
wire.Write(b[0:wlen])
}
for i := int64(0); i < alen1; i++ {
t.Command[i].Marshal(wire)
}
tmp32 = t.Seq
bs[0] = byte(tmp32)
bs[1] = byte(tmp32 >> 8)
bs[2] = byte(tmp32 >> 16)
bs[3] = byte(tmp32 >> 24)
tmp32 = t.Deps[0]
bs[4] = byte(tmp32)
bs[5] = byte(tmp32 >> 8)
bs[6] = byte(tmp32 >> 16)
bs[7] = byte(tmp32 >> 24)
tmp32 = t.Deps[1]
bs[8] = byte(tmp32)
bs[9] = byte(tmp32 >> 8)
bs[10] = byte(tmp32 >> 16)
bs[11] = byte(tmp32 >> 24)
tmp32 = t.Deps[2]
bs[12] = byte(tmp32)
bs[13] = byte(tmp32 >> 8)
bs[14] = byte(tmp32 >> 16)
bs[15] = byte(tmp32 >> 24)
tmp32 = t.Deps[3]
bs[16] = byte(tmp32)
bs[17] = byte(tmp32 >> 8)
bs[18] = byte(tmp32 >> 16)
bs[19] = byte(tmp32 >> 24)
tmp32 = t.Deps[4]
bs[20] = byte(tmp32)
bs[21] = byte(tmp32 >> 8)
bs[22] = byte(tmp32 >> 16)
bs[23] = byte(tmp32 >> 24)
wire.Write(bs)
}
// rawInt64 should only be used by low-level encoders
func (p *exporter) rawInt64(x int64) {
var tmp [binary.MaxVarintLen64]byte
n := binary.PutVarint(tmp[:], x)
for i := 0; i < n; i++ {
p.rawByte(tmp[i])
}
}
func generateRandFile(size int) (p string, m string) {
p = filepath.Join(os.TempDir(), "fcgict"+strconv.Itoa(rand.Int()))
// open output file
fo, err := os.Create(p)
if err != nil {
panic(err)
}
// close fo on exit and check for its returned error
defer func() {
if err := fo.Close(); err != nil {
panic(err)
}
}()
h := md5.New()
for i := 0; i < size/16; i++ {
buf := make([]byte, 16)
binary.PutVarint(buf, rand.Int63())
fo.Write(buf)
h.Write(buf)
}
m = fmt.Sprintf("%x", h.Sum(nil))
return
}