// Creates a new log entry associated with a log.
func newLogEntry(log *Log, event *ev, index uint64, term uint64, command Command) (*LogEntry, error) {
var buf bytes.Buffer
var commandName string
if command != nil {
commandName = command.CommandName()
if encoder, ok := command.(CommandEncoder); ok {
if err := encoder.Encode(&buf); err != nil {
return nil, err
}
} else {
if err := json.NewEncoder(&buf).Encode(command); err != nil {
return nil, err
}
}
}
pb := &protobuf.LogEntry{
Index: proto.Uint64(index),
Term: proto.Uint64(term),
CommandName: proto.String(commandName),
Command: buf.Bytes(),
}
e := &LogEntry{
pb: pb,
log: log,
event: event,
}
return e, nil
}
func createStartStopMessage(requestId uint64, peerType events.PeerType) *events.Envelope {
return &events.Envelope{
Origin: proto.String("fake-origin-2"),
EventType: events.Envelope_HttpStartStop.Enum(),
HttpStartStop: &events.HttpStartStop{
StartTimestamp: proto.Int64(1),
StopTimestamp: proto.Int64(100),
RequestId: &events.UUID{
Low: proto.Uint64(requestId),
High: proto.Uint64(requestId + 1),
},
PeerType: &peerType,
Method: events.Method_GET.Enum(),
Uri: proto.String("fake-uri-1"),
RemoteAddress: proto.String("fake-remote-addr-1"),
UserAgent: proto.String("fake-user-agent-1"),
StatusCode: proto.Int32(103),
ContentLength: proto.Int64(104),
ApplicationId: &events.UUID{
Low: proto.Uint64(105),
High: proto.Uint64(106),
},
InstanceIndex: proto.Int32(6),
InstanceId: proto.String("fake-instance-id-1"),
},
}
}
func generateCounterEvent(delta, total uint64) *events.CounterEvent {
return &events.CounterEvent{
Name: proto.String("TruncatingBuffer.DroppedMessages"),
Delta: proto.Uint64(delta),
Total: proto.Uint64(total),
}
}
func (d *DroppedCounter) encodeCounterEvent(droppedCount, totalDropped int64) []byte {
message := &events.Envelope{
Origin: proto.String(d.origin),
Timestamp: proto.Int64(time.Now().UnixNano()),
Ip: proto.String(d.ip),
Deployment: proto.String(d.conf.Deployment),
Index: proto.String(d.conf.Index),
Job: proto.String(d.conf.Job),
EventType: events.Envelope_CounterEvent.Enum(),
CounterEvent: &events.CounterEvent{
Name: proto.String("DroppedCounter.droppedMessageCount"),
Delta: proto.Uint64(uint64(droppedCount)),
Total: proto.Uint64(uint64(totalDropped)),
},
}
bytes, err := message.Marshal()
if err != nil {
d.incrementer.BatchIncrementCounter("droppedCounter.sendErrors")
d.timer.Reset(time.Millisecond)
return nil
}
return prefixMessage(bytes)
}
// handleFetchData handles a request for the current nodes meta data
func (r *rpc) handleFetchData(req *internal.FetchDataRequest) (*internal.FetchDataResponse, error) {
var (
b []byte
data *Data
err error
)
for {
data = r.store.cachedData()
if data.Index != req.GetIndex() {
b, err = data.MarshalBinary()
if err != nil {
return nil, err
}
break
}
if !req.GetBlocking() {
break
}
if err := r.store.WaitForDataChanged(); err != nil {
return nil, err
}
}
return &internal.FetchDataResponse{
Header: &internal.ResponseHeader{
OK: proto.Bool(true),
},
Index: proto.Uint64(data.Index),
Term: proto.Uint64(data.Term),
Data: b}, nil
}
func TestUnmarshalPartiallyPopulatedOptionalFieldsFails(t *testing.T) {
// Fill in all fields, then randomly remove one.
dataOut := &test.NinOptNative{
Field1: proto.Float64(0),
Field2: proto.Float32(0),
Field3: proto.Int32(0),
Field4: proto.Int64(0),
Field5: proto.Uint32(0),
Field6: proto.Uint64(0),
Field7: proto.Int32(0),
Field8: proto.Int64(0),
Field9: proto.Uint32(0),
Field10: proto.Int32(0),
Field11: proto.Uint64(0),
Field12: proto.Int64(0),
Field13: proto.Bool(false),
Field14: proto.String("0"),
Field15: []byte("0"),
}
r := rand.New(rand.NewSource(time.Now().UnixNano()))
fieldName := "Field" + strconv.Itoa(r.Intn(15)+1)
field := reflect.ValueOf(dataOut).Elem().FieldByName(fieldName)
fieldType := field.Type()
field.Set(reflect.Zero(fieldType))
encodedMessage, err := proto.Marshal(dataOut)
if err != nil {
t.Fatalf("Unexpected error when marshalling dataOut: %v", err)
}
dataIn := NidOptNative{}
err = proto.Unmarshal(encodedMessage, &dataIn)
if err.Error() != `proto: required field "`+fieldName+`" not set` {
t.Fatalf(`err.Error() != "proto: required field "`+fieldName+`" not set"; was "%s" instead`, err.Error())
}
}
// Encodes the SnapshotRecoveryRequest to a buffer. Returns the number of bytes
// written and any error that may have occurred.
func (req *SnapshotRecoveryRequest) Encode(w io.Writer) (int, error) {
protoPeers := make([]*protobuf.SnapshotRecoveryRequest_Peer, len(req.Peers))
for i, peer := range req.Peers {
protoPeers[i] = &protobuf.SnapshotRecoveryRequest_Peer{
Name: proto.String(peer.Name),
ConnectionString: proto.String(peer.ConnectionString),
}
}
pb := &protobuf.SnapshotRecoveryRequest{
LeaderName: proto.String(req.LeaderName),
LastIndex: proto.Uint64(req.LastIndex),
LastTerm: proto.Uint64(req.LastTerm),
Peers: protoPeers,
State: req.State,
}
p, err := proto.Marshal(pb)
if err != nil {
return -1, err
}
return w.Write(p)
}
func NewHeartbeat(sentCount, receivedCount, errorCount uint64) *events.Heartbeat {
return &events.Heartbeat{
SentCount: proto.Uint64(sentCount),
ReceivedCount: proto.Uint64(receivedCount),
ErrorCount: proto.Uint64(errorCount),
}
}
func ParseRanges(text string) (*mesosproto.Value_Ranges, error) {
reg := regexp.MustCompile(`([\d]+)`)
ports := reg.FindAllString(text, -1)
ranges := &mesosproto.Value_Ranges{}
if len(ports)%2 != 0 {
return nil, errors.New("Cannot parse range string: " + text)
}
for i := 0; i < len(ports)-1; i += 2 {
i64Begin, err := strconv.Atoi(ports[i])
if err != nil {
return nil, errors.New("Cannot parse range string: " + text)
}
i64End, err := strconv.Atoi(ports[i+1])
if err != nil {
return nil, errors.New("Cannot parse range string: " + text)
}
ranges.Range = append(ranges.Range, &mesosproto.Value_Range{
Begin: proto.Uint64(uint64(i64Begin)),
End: proto.Uint64(uint64(i64End)),
})
}
return ranges, nil
}
func (d *LauncherData) terminate(row *RunQueueEntry, action string) {
if d.killedRecently[row.Id] {
return
}
if action == KILL_ACTION_NO_ACTION {
d.call(&badoo_phproxyd.RequestTerminate{Hash: proto.Uint64(row.Id)})
} else {
params := []string{
`\ScriptFramework\Script_Kill`,
fmt.Sprintf("--force-sf-db=%s", db.GetDbName()),
fmt.Sprintf("--kill-run-id=%d", row.Id),
fmt.Sprintf("--kill-action=%s", action),
fmt.Sprintf("--kill-class-name=%s", row.ClassName),
fmt.Sprintf("--kill-timetable-id=%d", row.timetable_id.Int64),
}
d.call(&badoo_phproxyd.RequestRun{
Script: proto.String(getScriptPath(row.settings)),
Hash: proto.Uint64(0),
Tag: proto.String(PHPROXY_TAG),
Force: proto.Int32(1),
Params: params,
Store: badoo_phproxyd.StoreT_MEMORY.Enum(),
FreeAfterRun: proto.Bool(true),
})
}
d.killedRecently[row.Id] = true
}
func BasicCounterEvent(origin string) *events.Envelope {
return &events.Envelope{
Origin: proto.String(origin),
EventType: events.Envelope_CounterEvent.Enum(),
CounterEvent: &events.CounterEvent{
Name: proto.String("fake-counter-event"),
Delta: proto.Uint64(1),
Total: proto.Uint64(2),
},
}
}
func basicCounterEventEnvelope() *events.Envelope {
return &events.Envelope{
Origin: proto.String("fake-origin-2"),
EventType: events.Envelope_CounterEvent.Enum(),
CounterEvent: &events.CounterEvent{
Name: proto.String("fake-metric-name"),
Delta: proto.Uint64(3),
Total: proto.Uint64(3),
},
}
}
// Creates a new AppendEntries response.
func newAppendEntriesResponse(term uint64, success bool, index uint64, commitIndex uint64) *AppendEntriesResponse {
pb := &protobuf.AppendEntriesResponse{
Term: proto.Uint64(term),
Index: proto.Uint64(index),
Success: proto.Bool(success),
CommitIndex: proto.Uint64(commitIndex),
}
return &AppendEntriesResponse{
pb: pb,
}
}
func createCounterEvent() *events.Envelope {
return &events.Envelope{
Origin: proto.String(helpers.ORIGIN_NAME),
EventType: events.Envelope_CounterEvent.Enum(),
Timestamp: proto.Int64(time.Now().UnixNano()),
CounterEvent: &events.CounterEvent{
Name: proto.String("LATs-Counter"),
Delta: proto.Uint64(5),
Total: proto.Uint64(5),
},
}
}
// Encodes the SnapshotRequest to a buffer. Returns the number of bytes
// written and any error that may have occurred.
func (req *SnapshotRequest) Encode(w io.Writer) (int, error) {
pb := &protobuf.SnapshotRequest{
LeaderName: proto.String(req.LeaderName),
LastIndex: proto.Uint64(req.LastIndex),
LastTerm: proto.Uint64(req.LastTerm),
}
p, err := proto.Marshal(pb)
if err != nil {
return -1, err
}
return w.Write(p)
}
// Encode writes the response to a writer.
// Returns the number of bytes written and any error that occurs.
func (req *SnapshotRecoveryResponse) Encode(w io.Writer) (int, error) {
pb := &protobuf.SnapshotRecoveryResponse{
Term: proto.Uint64(req.Term),
Success: proto.Bool(req.Success),
CommitIndex: proto.Uint64(req.CommitIndex),
}
p, err := proto.Marshal(pb)
if err != nil {
return -1, err
}
return w.Write(p)
}
func createContainerMetric(appId string) *events.Envelope {
return &events.Envelope{
Origin: proto.String(helpers.ORIGIN_NAME),
EventType: events.Envelope_ContainerMetric.Enum(),
Timestamp: proto.Int64(time.Now().UnixNano()),
ContainerMetric: &events.ContainerMetric{
ApplicationId: proto.String(appId),
InstanceIndex: proto.Int32(1),
CpuPercentage: proto.Float64(20.0),
MemoryBytes: proto.Uint64(10),
DiskBytes: proto.Uint64(20),
},
}
}
func metricFor(instanceId int32, timestamp time.Time, cpu float64, mem uint64, disk uint64) *events.Envelope {
unixTimestamp := timestamp.UnixNano()
return &events.Envelope{
EventType: events.Envelope_ContainerMetric.Enum(),
Timestamp: proto.Int64(unixTimestamp),
ContainerMetric: &events.ContainerMetric{
ApplicationId: proto.String("myApp"),
InstanceIndex: proto.Int32(instanceId),
CpuPercentage: proto.Float64(cpu),
MemoryBytes: proto.Uint64(mem),
DiskBytes: proto.Uint64(disk),
},
}
}
// Encodes the RequestVoteRequest to a buffer. Returns the number of bytes
// written and any error that may have occurred.
func (req *RequestVoteRequest) Encode(w io.Writer) (int, error) {
pb := &protobuf.RequestVoteRequest{
Term: proto.Uint64(req.Term),
LastLogIndex: proto.Uint64(req.LastLogIndex),
LastLogTerm: proto.Uint64(req.LastLogTerm),
CandidateName: proto.String(req.CandidateName),
}
p, err := proto.Marshal(pb)
if err != nil {
return -1, err
}
return w.Write(p)
}
// BuildBasicResources build basic resources
// including cpus, mem, disk, ports
func BuildBasicResources(task *registry.Task) {
if task.Build {
return
}
if task.Cpus > 0 {
task.Resources = append(task.Resources, &mesosproto.Resource{
Name: proto.String("cpus"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Cpus)},
})
}
if task.Mem > 0 {
task.Resources = append(task.Resources, &mesosproto.Resource{
Name: proto.String("mem"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Mem)},
})
}
if task.Disk > 0 {
task.Resources = append(task.Resources, &mesosproto.Resource{
Name: proto.String("disk"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Disk)},
})
}
if len(task.Ports) > 0 {
ranges := &mesosproto.Value_Ranges{}
for _, port := range task.Ports {
if port.HostPort == 0 {
continue
}
ranges.Range = append(ranges.Range, &mesosproto.Value_Range{
Begin: proto.Uint64(uint64(port.HostPort)),
End: proto.Uint64(uint64(port.HostPort)),
})
}
task.Resources = append(task.Resources, &mesosproto.Resource{
Name: proto.String("ports"),
Type: mesosproto.Value_RANGES.Enum(),
Ranges: ranges,
})
}
task.Build = true
}
// Ensure shards with deprecated "OwnerIDs" can be decoded.
func TestShardInfo_UnmarshalBinary_OwnerIDs(t *testing.T) {
// Encode deprecated form to bytes.
buf, err := proto.Marshal(&internal.ShardInfo{
ID: proto.Uint64(1),
OwnerIDs: []uint64{10, 20, 30},
})
if err != nil {
t.Fatal(err)
}
// Decode deprecated form.
var si meta.ShardInfo
if err := si.UnmarshalBinary(buf); err != nil {
t.Fatal(err)
}
// Verify data is migrated correctly.
if !reflect.DeepEqual(si, meta.ShardInfo{
ID: 1,
Owners: []meta.ShardOwner{
{NodeID: 10},
{NodeID: 20},
{NodeID: 30},
},
}) {
t.Fatalf("unexpected shard info: %s", spew.Sdump(si))
}
}
// DeleteNode removes a node from the metastore by id.
func (s *Store) DeleteNode(id uint64) error {
return s.exec(internal.Command_DeleteNodeCommand, internal.E_DeleteNodeCommand_Command,
&internal.DeleteNodeCommand{
ID: proto.Uint64(id),
},
)
}
// doc bytes % etc
func (ms *MetricSender) ContainerMetric(appID string, instance int32, cpu float64, mem, disk uint64) ContainerMetricChainer {
chainer := containerMetricChainer{}
chainer.emitter = ms.eventEmitter
chainer.envelope = &events.Envelope{
Origin: proto.String(ms.eventEmitter.Origin()),
EventType: events.Envelope_ContainerMetric.Enum(),
ContainerMetric: &events.ContainerMetric{
ApplicationId: proto.String(appID),
InstanceIndex: proto.Int32(instance),
CpuPercentage: proto.Float64(cpu),
MemoryBytes: proto.Uint64(mem),
DiskBytes: proto.Uint64(disk),
},
}
return chainer
}
func lock(key, primary string, ts uint64) *kvrpcpb.LockInfo {
return &kvrpcpb.LockInfo{
Key: encodeKey(key),
PrimaryLock: encodeKey(primary),
LockVersion: proto.Uint64(ts),
}
}
func (c *Client) DeleteMetaNode(id uint64) error {
cmd := &internal.DeleteMetaNodeCommand{
ID: proto.Uint64(id),
}
return c.retryUntilExec(internal.Command_DeleteMetaNodeCommand, internal.E_DeleteMetaNodeCommand_Command, cmd)
}
//NewUintConst returns a new terminal expression containing the given uint value.
// uint(i)
func NewUintConst(i uint64) *Expr {
return &Expr{
Terminal: &Terminal{
UintValue: proto.Uint64(i),
},
}
}
// marshal serializes to a protobuf representation.
func (ni NodeInfo) marshal() *internal.NodeInfo {
pb := &internal.NodeInfo{}
pb.ID = proto.Uint64(ni.ID)
pb.Host = proto.String(ni.Host)
pb.TCPHost = proto.String(ni.TCPHost)
return pb
}
func (d *LauncherData) processFinished() {
var finishedIds []uint64
var err error
for run_id := range d.finishedMap {
d.call(&badoo_phproxyd.RequestFree{Hash: proto.Uint64(run_id)})
finishedIds = append(finishedIds, run_id)
}
if finishedIds == nil || len(finishedIds) == 0 {
return
}
sort.Sort(common.UInt64Slice(finishedIds))
err = db.DoInLazyTransaction(func(tx *db.LazyTrx) error {
return deleteFromRunQueue(tx, finishedIds, RUN_STATUS_FINISHED)
})
if err != nil {
log.Errorf("Could not delete rows from run queue for hostname=%s: %s", d.hostname, err.Error())
return
}
for _, v := range d.finishedMap {
d.delFromMaps(v.Id)
}
}
// ShardReader returns a reader for streaming shard data.
// Returned ReadCloser must be closed by the caller.
func (c *Client) ShardReader(id uint64) (io.ReadCloser, error) {
// Connect to remote server.
conn, err := tcp.Dial("tcp", c.host, MuxHeader)
if err != nil {
return nil, err
}
// Send request to server.
if err := c.writeRequest(conn, &internal.Request{ShardID: proto.Uint64(id)}); err != nil {
return nil, fmt.Errorf("write request: %s", err)
}
// Read response from the server.
resp, err := c.readResponse(conn)
if err != nil {
return nil, fmt.Errorf("read response: %s", err)
}
// If there was an error then return it and close connection.
if resp.GetError() != "" {
conn.Close()
return nil, errors.New(resp.GetError())
}
// Returning remaining stream for caller to consume.
return conn, nil
}
func (d *LauncherData) processLogFinishRequest(req *LauncherLogFinishRequest) {
var err error
var info *FinishResultRunInfo
defer func() { req.respCh <- &LauncherLogFinishResponse{err: err, info: info} }()
el := d.allMap[req.RunId]
if el == nil {
err = fmt.Errorf("No such rq row id=%d", req.RunId)
return
}
if el.RunStatus != req.PrevStatus {
err = fmt.Errorf("Previous status mismatch for rq row id=%d: req.prev=%s, actual=%s", req.RunId, req.PrevStatus, el.RunStatus)
return
}
if err = d.persistFinishAndNotify(el, req.Success, req.PrevStatus); err != nil {
return
}
d.updateStatus(el, RUN_STATUS_FINISHED)
el.finished_ts.Int64 = time.Now().Unix()
el.finished_ts.Valid = true
info = d.convertRunQueueEntryToFinishInfo(el)
d.delFromMaps(req.RunId)
d.call(&badoo_phproxyd.RequestFree{Hash: proto.Uint64(req.RunId)})
}