Exemple #1
0
// Creates a new log entry associated with a log.
func newLogEntry(log *Log, event *ev, index uint64, term uint64, command Command) (*LogEntry, error) {
	var buf bytes.Buffer
	var commandName string
	if command != nil {
		commandName = command.CommandName()
		if encoder, ok := command.(CommandEncoder); ok {
			if err := encoder.Encode(&buf); err != nil {
				return nil, err
			}
		} else {
			if err := json.NewEncoder(&buf).Encode(command); err != nil {
				return nil, err
			}
		}
	}

	pb := &protobuf.LogEntry{
		Index:       proto.Uint64(index),
		Term:        proto.Uint64(term),
		CommandName: proto.String(commandName),
		Command:     buf.Bytes(),
	}

	e := &LogEntry{
		pb:    pb,
		log:   log,
		event: event,
	}

	return e, nil
}
func createStartStopMessage(requestId uint64, peerType events.PeerType) *events.Envelope {
	return &events.Envelope{
		Origin:    proto.String("fake-origin-2"),
		EventType: events.Envelope_HttpStartStop.Enum(),
		HttpStartStop: &events.HttpStartStop{
			StartTimestamp: proto.Int64(1),
			StopTimestamp:  proto.Int64(100),
			RequestId: &events.UUID{
				Low:  proto.Uint64(requestId),
				High: proto.Uint64(requestId + 1),
			},
			PeerType:      &peerType,
			Method:        events.Method_GET.Enum(),
			Uri:           proto.String("fake-uri-1"),
			RemoteAddress: proto.String("fake-remote-addr-1"),
			UserAgent:     proto.String("fake-user-agent-1"),
			StatusCode:    proto.Int32(103),
			ContentLength: proto.Int64(104),
			ApplicationId: &events.UUID{
				Low:  proto.Uint64(105),
				High: proto.Uint64(106),
			},
			InstanceIndex: proto.Int32(6),
			InstanceId:    proto.String("fake-instance-id-1"),
		},
	}
}
func generateCounterEvent(delta, total uint64) *events.CounterEvent {
	return &events.CounterEvent{
		Name:  proto.String("TruncatingBuffer.DroppedMessages"),
		Delta: proto.Uint64(delta),
		Total: proto.Uint64(total),
	}
}
func (d *DroppedCounter) encodeCounterEvent(droppedCount, totalDropped int64) []byte {
	message := &events.Envelope{
		Origin:     proto.String(d.origin),
		Timestamp:  proto.Int64(time.Now().UnixNano()),
		Ip:         proto.String(d.ip),
		Deployment: proto.String(d.conf.Deployment),
		Index:      proto.String(d.conf.Index),
		Job:        proto.String(d.conf.Job),
		EventType:  events.Envelope_CounterEvent.Enum(),
		CounterEvent: &events.CounterEvent{
			Name:  proto.String("DroppedCounter.droppedMessageCount"),
			Delta: proto.Uint64(uint64(droppedCount)),
			Total: proto.Uint64(uint64(totalDropped)),
		},
	}

	bytes, err := message.Marshal()
	if err != nil {
		d.incrementer.BatchIncrementCounter("droppedCounter.sendErrors")
		d.timer.Reset(time.Millisecond)
		return nil
	}

	return prefixMessage(bytes)
}
Exemple #5
0
// handleFetchData handles a request for the current nodes meta data
func (r *rpc) handleFetchData(req *internal.FetchDataRequest) (*internal.FetchDataResponse, error) {
	var (
		b    []byte
		data *Data
		err  error
	)

	for {
		data = r.store.cachedData()
		if data.Index != req.GetIndex() {
			b, err = data.MarshalBinary()
			if err != nil {
				return nil, err
			}
			break
		}

		if !req.GetBlocking() {
			break
		}

		if err := r.store.WaitForDataChanged(); err != nil {
			return nil, err
		}
	}

	return &internal.FetchDataResponse{
		Header: &internal.ResponseHeader{
			OK: proto.Bool(true),
		},
		Index: proto.Uint64(data.Index),
		Term:  proto.Uint64(data.Term),
		Data:  b}, nil
}
func TestUnmarshalPartiallyPopulatedOptionalFieldsFails(t *testing.T) {
	// Fill in all fields, then randomly remove one.
	dataOut := &test.NinOptNative{
		Field1:  proto.Float64(0),
		Field2:  proto.Float32(0),
		Field3:  proto.Int32(0),
		Field4:  proto.Int64(0),
		Field5:  proto.Uint32(0),
		Field6:  proto.Uint64(0),
		Field7:  proto.Int32(0),
		Field8:  proto.Int64(0),
		Field9:  proto.Uint32(0),
		Field10: proto.Int32(0),
		Field11: proto.Uint64(0),
		Field12: proto.Int64(0),
		Field13: proto.Bool(false),
		Field14: proto.String("0"),
		Field15: []byte("0"),
	}
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	fieldName := "Field" + strconv.Itoa(r.Intn(15)+1)
	field := reflect.ValueOf(dataOut).Elem().FieldByName(fieldName)
	fieldType := field.Type()
	field.Set(reflect.Zero(fieldType))
	encodedMessage, err := proto.Marshal(dataOut)
	if err != nil {
		t.Fatalf("Unexpected error when marshalling dataOut: %v", err)
	}
	dataIn := NidOptNative{}
	err = proto.Unmarshal(encodedMessage, &dataIn)
	if err.Error() != `proto: required field "`+fieldName+`" not set` {
		t.Fatalf(`err.Error() != "proto: required field "`+fieldName+`" not set"; was "%s" instead`, err.Error())
	}
}
Exemple #7
0
// Encodes the SnapshotRecoveryRequest to a buffer. Returns the number of bytes
// written and any error that may have occurred.
func (req *SnapshotRecoveryRequest) Encode(w io.Writer) (int, error) {

	protoPeers := make([]*protobuf.SnapshotRecoveryRequest_Peer, len(req.Peers))

	for i, peer := range req.Peers {
		protoPeers[i] = &protobuf.SnapshotRecoveryRequest_Peer{
			Name:             proto.String(peer.Name),
			ConnectionString: proto.String(peer.ConnectionString),
		}
	}

	pb := &protobuf.SnapshotRecoveryRequest{
		LeaderName: proto.String(req.LeaderName),
		LastIndex:  proto.Uint64(req.LastIndex),
		LastTerm:   proto.Uint64(req.LastTerm),
		Peers:      protoPeers,
		State:      req.State,
	}
	p, err := proto.Marshal(pb)
	if err != nil {
		return -1, err
	}

	return w.Write(p)
}
Exemple #8
0
func NewHeartbeat(sentCount, receivedCount, errorCount uint64) *events.Heartbeat {
	return &events.Heartbeat{
		SentCount:     proto.Uint64(sentCount),
		ReceivedCount: proto.Uint64(receivedCount),
		ErrorCount:    proto.Uint64(errorCount),
	}
}
Exemple #9
0
func ParseRanges(text string) (*mesosproto.Value_Ranges, error) {
	reg := regexp.MustCompile(`([\d]+)`)
	ports := reg.FindAllString(text, -1)
	ranges := &mesosproto.Value_Ranges{}

	if len(ports)%2 != 0 {
		return nil, errors.New("Cannot parse range string: " + text)
	}

	for i := 0; i < len(ports)-1; i += 2 {
		i64Begin, err := strconv.Atoi(ports[i])
		if err != nil {
			return nil, errors.New("Cannot parse range string: " + text)
		}
		i64End, err := strconv.Atoi(ports[i+1])
		if err != nil {
			return nil, errors.New("Cannot parse range string: " + text)
		}

		ranges.Range = append(ranges.Range, &mesosproto.Value_Range{
			Begin: proto.Uint64(uint64(i64Begin)),
			End:   proto.Uint64(uint64(i64End)),
		})
	}
	return ranges, nil
}
Exemple #10
0
func (d *LauncherData) terminate(row *RunQueueEntry, action string) {
	if d.killedRecently[row.Id] {
		return
	}

	if action == KILL_ACTION_NO_ACTION {
		d.call(&badoo_phproxyd.RequestTerminate{Hash: proto.Uint64(row.Id)})
	} else {
		params := []string{
			`\ScriptFramework\Script_Kill`,
			fmt.Sprintf("--force-sf-db=%s", db.GetDbName()),
			fmt.Sprintf("--kill-run-id=%d", row.Id),
			fmt.Sprintf("--kill-action=%s", action),
			fmt.Sprintf("--kill-class-name=%s", row.ClassName),
			fmt.Sprintf("--kill-timetable-id=%d", row.timetable_id.Int64),
		}

		d.call(&badoo_phproxyd.RequestRun{
			Script:       proto.String(getScriptPath(row.settings)),
			Hash:         proto.Uint64(0),
			Tag:          proto.String(PHPROXY_TAG),
			Force:        proto.Int32(1),
			Params:       params,
			Store:        badoo_phproxyd.StoreT_MEMORY.Enum(),
			FreeAfterRun: proto.Bool(true),
		})
	}

	d.killedRecently[row.Id] = true
}
func BasicCounterEvent(origin string) *events.Envelope {
	return &events.Envelope{
		Origin:    proto.String(origin),
		EventType: events.Envelope_CounterEvent.Enum(),
		CounterEvent: &events.CounterEvent{
			Name:  proto.String("fake-counter-event"),
			Delta: proto.Uint64(1),
			Total: proto.Uint64(2),
		},
	}
}
func basicCounterEventEnvelope() *events.Envelope {
	return &events.Envelope{
		Origin:    proto.String("fake-origin-2"),
		EventType: events.Envelope_CounterEvent.Enum(),
		CounterEvent: &events.CounterEvent{
			Name:  proto.String("fake-metric-name"),
			Delta: proto.Uint64(3),
			Total: proto.Uint64(3),
		},
	}
}
Exemple #13
0
// Creates a new AppendEntries response.
func newAppendEntriesResponse(term uint64, success bool, index uint64, commitIndex uint64) *AppendEntriesResponse {
	pb := &protobuf.AppendEntriesResponse{
		Term:        proto.Uint64(term),
		Index:       proto.Uint64(index),
		Success:     proto.Bool(success),
		CommitIndex: proto.Uint64(commitIndex),
	}

	return &AppendEntriesResponse{
		pb: pb,
	}
}
func createCounterEvent() *events.Envelope {
	return &events.Envelope{
		Origin:    proto.String(helpers.ORIGIN_NAME),
		EventType: events.Envelope_CounterEvent.Enum(),
		Timestamp: proto.Int64(time.Now().UnixNano()),
		CounterEvent: &events.CounterEvent{
			Name:  proto.String("LATs-Counter"),
			Delta: proto.Uint64(5),
			Total: proto.Uint64(5),
		},
	}
}
Exemple #15
0
// Encodes the SnapshotRequest to a buffer. Returns the number of bytes
// written and any error that may have occurred.
func (req *SnapshotRequest) Encode(w io.Writer) (int, error) {
	pb := &protobuf.SnapshotRequest{
		LeaderName: proto.String(req.LeaderName),
		LastIndex:  proto.Uint64(req.LastIndex),
		LastTerm:   proto.Uint64(req.LastTerm),
	}
	p, err := proto.Marshal(pb)
	if err != nil {
		return -1, err
	}

	return w.Write(p)
}
Exemple #16
0
// Encode writes the response to a writer.
// Returns the number of bytes written and any error that occurs.
func (req *SnapshotRecoveryResponse) Encode(w io.Writer) (int, error) {
	pb := &protobuf.SnapshotRecoveryResponse{
		Term:        proto.Uint64(req.Term),
		Success:     proto.Bool(req.Success),
		CommitIndex: proto.Uint64(req.CommitIndex),
	}
	p, err := proto.Marshal(pb)
	if err != nil {
		return -1, err
	}

	return w.Write(p)
}
func createContainerMetric(appId string) *events.Envelope {
	return &events.Envelope{
		Origin:    proto.String(helpers.ORIGIN_NAME),
		EventType: events.Envelope_ContainerMetric.Enum(),
		Timestamp: proto.Int64(time.Now().UnixNano()),
		ContainerMetric: &events.ContainerMetric{
			ApplicationId: proto.String(appId),
			InstanceIndex: proto.Int32(1),
			CpuPercentage: proto.Float64(20.0),
			MemoryBytes:   proto.Uint64(10),
			DiskBytes:     proto.Uint64(20),
		},
	}
}
func metricFor(instanceId int32, timestamp time.Time, cpu float64, mem uint64, disk uint64) *events.Envelope {
	unixTimestamp := timestamp.UnixNano()
	return &events.Envelope{
		EventType: events.Envelope_ContainerMetric.Enum(),
		Timestamp: proto.Int64(unixTimestamp),
		ContainerMetric: &events.ContainerMetric{
			ApplicationId: proto.String("myApp"),
			InstanceIndex: proto.Int32(instanceId),
			CpuPercentage: proto.Float64(cpu),
			MemoryBytes:   proto.Uint64(mem),
			DiskBytes:     proto.Uint64(disk),
		},
	}
}
Exemple #19
0
// Encodes the RequestVoteRequest to a buffer. Returns the number of bytes
// written and any error that may have occurred.
func (req *RequestVoteRequest) Encode(w io.Writer) (int, error) {
	pb := &protobuf.RequestVoteRequest{
		Term:          proto.Uint64(req.Term),
		LastLogIndex:  proto.Uint64(req.LastLogIndex),
		LastLogTerm:   proto.Uint64(req.LastLogTerm),
		CandidateName: proto.String(req.CandidateName),
	}
	p, err := proto.Marshal(pb)
	if err != nil {
		return -1, err
	}

	return w.Write(p)
}
Exemple #20
0
// BuildBasicResources build basic resources
// including cpus, mem, disk, ports
func BuildBasicResources(task *registry.Task) {
	if task.Build {
		return
	}
	if task.Cpus > 0 {
		task.Resources = append(task.Resources, &mesosproto.Resource{
			Name:   proto.String("cpus"),
			Type:   mesosproto.Value_SCALAR.Enum(),
			Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Cpus)},
		})
	}

	if task.Mem > 0 {
		task.Resources = append(task.Resources, &mesosproto.Resource{
			Name:   proto.String("mem"),
			Type:   mesosproto.Value_SCALAR.Enum(),
			Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Mem)},
		})
	}

	if task.Disk > 0 {
		task.Resources = append(task.Resources, &mesosproto.Resource{
			Name:   proto.String("disk"),
			Type:   mesosproto.Value_SCALAR.Enum(),
			Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Disk)},
		})
	}

	if len(task.Ports) > 0 {
		ranges := &mesosproto.Value_Ranges{}
		for _, port := range task.Ports {
			if port.HostPort == 0 {
				continue
			}
			ranges.Range = append(ranges.Range, &mesosproto.Value_Range{
				Begin: proto.Uint64(uint64(port.HostPort)),
				End:   proto.Uint64(uint64(port.HostPort)),
			})
		}

		task.Resources = append(task.Resources, &mesosproto.Resource{
			Name:   proto.String("ports"),
			Type:   mesosproto.Value_RANGES.Enum(),
			Ranges: ranges,
		})
	}
	task.Build = true
}
Exemple #21
0
// Ensure shards with deprecated "OwnerIDs" can be decoded.
func TestShardInfo_UnmarshalBinary_OwnerIDs(t *testing.T) {
	// Encode deprecated form to bytes.
	buf, err := proto.Marshal(&internal.ShardInfo{
		ID:       proto.Uint64(1),
		OwnerIDs: []uint64{10, 20, 30},
	})
	if err != nil {
		t.Fatal(err)
	}

	// Decode deprecated form.
	var si meta.ShardInfo
	if err := si.UnmarshalBinary(buf); err != nil {
		t.Fatal(err)
	}

	// Verify data is migrated correctly.
	if !reflect.DeepEqual(si, meta.ShardInfo{
		ID: 1,
		Owners: []meta.ShardOwner{
			{NodeID: 10},
			{NodeID: 20},
			{NodeID: 30},
		},
	}) {
		t.Fatalf("unexpected shard info: %s", spew.Sdump(si))
	}
}
Exemple #22
0
// DeleteNode removes a node from the metastore by id.
func (s *Store) DeleteNode(id uint64) error {
	return s.exec(internal.Command_DeleteNodeCommand, internal.E_DeleteNodeCommand_Command,
		&internal.DeleteNodeCommand{
			ID: proto.Uint64(id),
		},
	)
}
// doc bytes % etc
func (ms *MetricSender) ContainerMetric(appID string, instance int32, cpu float64, mem, disk uint64) ContainerMetricChainer {
	chainer := containerMetricChainer{}
	chainer.emitter = ms.eventEmitter
	chainer.envelope = &events.Envelope{
		Origin:    proto.String(ms.eventEmitter.Origin()),
		EventType: events.Envelope_ContainerMetric.Enum(),
		ContainerMetric: &events.ContainerMetric{
			ApplicationId: proto.String(appID),
			InstanceIndex: proto.Int32(instance),
			CpuPercentage: proto.Float64(cpu),
			MemoryBytes:   proto.Uint64(mem),
			DiskBytes:     proto.Uint64(disk),
		},
	}
	return chainer
}
Exemple #24
0
func lock(key, primary string, ts uint64) *kvrpcpb.LockInfo {
	return &kvrpcpb.LockInfo{
		Key:         encodeKey(key),
		PrimaryLock: encodeKey(primary),
		LockVersion: proto.Uint64(ts),
	}
}
Exemple #25
0
func (c *Client) DeleteMetaNode(id uint64) error {
	cmd := &internal.DeleteMetaNodeCommand{
		ID: proto.Uint64(id),
	}

	return c.retryUntilExec(internal.Command_DeleteMetaNodeCommand, internal.E_DeleteMetaNodeCommand_Command, cmd)
}
Exemple #26
0
//NewUintConst returns a new terminal expression containing the given uint value.
//  uint(i)
func NewUintConst(i uint64) *Expr {
	return &Expr{
		Terminal: &Terminal{
			UintValue: proto.Uint64(i),
		},
	}
}
Exemple #27
0
// marshal serializes to a protobuf representation.
func (ni NodeInfo) marshal() *internal.NodeInfo {
	pb := &internal.NodeInfo{}
	pb.ID = proto.Uint64(ni.ID)
	pb.Host = proto.String(ni.Host)
	pb.TCPHost = proto.String(ni.TCPHost)
	return pb
}
Exemple #28
0
func (d *LauncherData) processFinished() {
	var finishedIds []uint64
	var err error

	for run_id := range d.finishedMap {
		d.call(&badoo_phproxyd.RequestFree{Hash: proto.Uint64(run_id)})
		finishedIds = append(finishedIds, run_id)
	}

	if finishedIds == nil || len(finishedIds) == 0 {
		return
	}

	sort.Sort(common.UInt64Slice(finishedIds))

	err = db.DoInLazyTransaction(func(tx *db.LazyTrx) error {
		return deleteFromRunQueue(tx, finishedIds, RUN_STATUS_FINISHED)
	})

	if err != nil {
		log.Errorf("Could not delete rows from run queue for hostname=%s: %s", d.hostname, err.Error())
		return
	}

	for _, v := range d.finishedMap {
		d.delFromMaps(v.Id)
	}
}
Exemple #29
0
// ShardReader returns a reader for streaming shard data.
// Returned ReadCloser must be closed by the caller.
func (c *Client) ShardReader(id uint64) (io.ReadCloser, error) {
	// Connect to remote server.
	conn, err := tcp.Dial("tcp", c.host, MuxHeader)
	if err != nil {
		return nil, err
	}

	// Send request to server.
	if err := c.writeRequest(conn, &internal.Request{ShardID: proto.Uint64(id)}); err != nil {
		return nil, fmt.Errorf("write request: %s", err)
	}

	// Read response from the server.
	resp, err := c.readResponse(conn)
	if err != nil {
		return nil, fmt.Errorf("read response: %s", err)
	}

	// If there was an error then return it and close connection.
	if resp.GetError() != "" {
		conn.Close()
		return nil, errors.New(resp.GetError())
	}

	// Returning remaining stream for caller to consume.
	return conn, nil
}
Exemple #30
0
func (d *LauncherData) processLogFinishRequest(req *LauncherLogFinishRequest) {
	var err error
	var info *FinishResultRunInfo
	defer func() { req.respCh <- &LauncherLogFinishResponse{err: err, info: info} }()

	el := d.allMap[req.RunId]

	if el == nil {
		err = fmt.Errorf("No such rq row id=%d", req.RunId)
		return
	}

	if el.RunStatus != req.PrevStatus {
		err = fmt.Errorf("Previous status mismatch for rq row id=%d: req.prev=%s, actual=%s", req.RunId, req.PrevStatus, el.RunStatus)
		return
	}

	if err = d.persistFinishAndNotify(el, req.Success, req.PrevStatus); err != nil {
		return
	}

	d.updateStatus(el, RUN_STATUS_FINISHED)
	el.finished_ts.Int64 = time.Now().Unix()
	el.finished_ts.Valid = true

	info = d.convertRunQueueEntryToFinishInfo(el)

	d.delFromMaps(req.RunId)
	d.call(&badoo_phproxyd.RequestFree{Hash: proto.Uint64(req.RunId)})
}