// BuildEmptyResources build empty resources
func BuildEmptyResources() []*mesosproto.Resource {
var resources = []*mesosproto.Resource{}
resources = append(resources, &mesosproto.Resource{
Name: proto.String("cpus"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(minCpus)},
})
resources = append(resources, &mesosproto.Resource{
Name: proto.String("mem"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(minMem)},
})
return resources
}
func (driver *MesosExecutorDriver) makeStatusUpdate(taskStatus *mesosproto.TaskStatus) *mesosproto.StatusUpdate {
now := float64(time.Now().Unix())
// Fill in all the fields.
taskStatus.Timestamp = proto.Float64(now)
taskStatus.SlaveId = driver.slaveID
update := &mesosproto.StatusUpdate{
FrameworkId: driver.frameworkID,
ExecutorId: driver.executorID,
SlaveId: driver.slaveID,
Status: taskStatus,
Timestamp: proto.Float64(now),
Uuid: uuid.NewUUID(),
}
return update
}
// BuildBasicResources build basic resources
// including cpus, mem, disk, ports
func BuildBasicResources(task *registry.Task) {
if task.Build {
return
}
if task.Cpus > 0 {
task.Resources = append(task.Resources, &mesosproto.Resource{
Name: proto.String("cpus"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Cpus)},
})
}
if task.Mem > 0 {
task.Resources = append(task.Resources, &mesosproto.Resource{
Name: proto.String("mem"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Mem)},
})
}
if task.Disk > 0 {
task.Resources = append(task.Resources, &mesosproto.Resource{
Name: proto.String("disk"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Disk)},
})
}
if len(task.Ports) > 0 {
ranges := &mesosproto.Value_Ranges{}
for _, port := range task.Ports {
if port.HostPort == 0 {
continue
}
ranges.Range = append(ranges.Range, &mesosproto.Value_Range{
Begin: proto.Uint64(uint64(port.HostPort)),
End: proto.Uint64(uint64(port.HostPort)),
})
}
task.Resources = append(task.Resources, &mesosproto.Resource{
Name: proto.String("ports"),
Type: mesosproto.Value_RANGES.Enum(),
Ranges: ranges,
})
}
task.Build = true
}
func (sched *Scheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
logOffers(offers)
jobs, err := getLaunchableJobs()
if err != nil {
log.Errorf("Unable to get pending jobs! %s\n", err.Error())
return
}
offersAndTasks, err := packJobsInOffers(jobs, offers)
if err != nil {
log.Errorf("Unable to pack jobs into offers! %s\n", err.Error())
return
}
for _, ot := range offersAndTasks {
if len(ot.Tasks) == 0 {
log.Infof("Declining unused offer %s", ot.Offer.Id.GetValue())
driver.DeclineOffer(ot.Offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
continue
} else {
log.Infof("Launching %d tasks for offer %s\n", len(ot.Tasks), ot.Offer.Id.GetValue())
driver.LaunchTasks([]*mesos.OfferID{ot.Offer.Id}, ot.Tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
sched.tasksLaunched = sched.tasksLaunched + len(ot.Tasks)
}
}
}
func TestUnmarshalPartiallyPopulatedOptionalFieldsFails(t *testing.T) {
// Fill in all fields, then randomly remove one.
dataOut := &test.NinOptNative{
Field1: proto.Float64(0),
Field2: proto.Float32(0),
Field3: proto.Int32(0),
Field4: proto.Int64(0),
Field5: proto.Uint32(0),
Field6: proto.Uint64(0),
Field7: proto.Int32(0),
Field8: proto.Int64(0),
Field9: proto.Uint32(0),
Field10: proto.Int32(0),
Field11: proto.Uint64(0),
Field12: proto.Int64(0),
Field13: proto.Bool(false),
Field14: proto.String("0"),
Field15: []byte("0"),
}
r := rand.New(rand.NewSource(time.Now().UnixNano()))
fieldName := "Field" + strconv.Itoa(r.Intn(15)+1)
field := reflect.ValueOf(dataOut).Elem().FieldByName(fieldName)
fieldType := field.Type()
field.Set(reflect.Zero(fieldType))
encodedMessage, err := proto.Marshal(dataOut)
if err != nil {
t.Fatalf("Unexpected error when marshalling dataOut: %v", err)
}
dataIn := NidOptNative{}
err = proto.Unmarshal(encodedMessage, &dataIn)
if err.Error() != `proto: required field "`+fieldName+`" not set` {
t.Fatalf(`err.Error() != "proto: required field "`+fieldName+`" not set"; was "%s" instead`, err.Error())
}
}
func encodeAux(aux []interface{}) []*internal.Aux {
pb := make([]*internal.Aux, len(aux))
for i := range aux {
switch v := aux[i].(type) {
case float64:
pb[i] = &internal.Aux{DataType: proto.Int32(Float), FloatValue: proto.Float64(v)}
case *float64:
pb[i] = &internal.Aux{DataType: proto.Int32(Float)}
case int64:
pb[i] = &internal.Aux{DataType: proto.Int32(Integer), IntegerValue: proto.Int64(v)}
case *int64:
pb[i] = &internal.Aux{DataType: proto.Int32(Integer)}
case string:
pb[i] = &internal.Aux{DataType: proto.Int32(String), StringValue: proto.String(v)}
case *string:
pb[i] = &internal.Aux{DataType: proto.Int32(String)}
case bool:
pb[i] = &internal.Aux{DataType: proto.Int32(Boolean), BooleanValue: proto.Bool(v)}
case *bool:
pb[i] = &internal.Aux{DataType: proto.Int32(Boolean)}
default:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(Unknown))}
}
}
return pb
}
func newScalarAttribute(name string, val float64) *mesos.Attribute {
return &mesos.Attribute{
Name: proto.String(name),
Type: mesos.Value_SCALAR.Enum(),
Scalar: &mesos.Value_Scalar{Value: proto.Float64(val)},
}
}
func basicValueMetric(name string, value float64, unit string) *events.ValueMetric {
return &events.ValueMetric{
Name: proto.String(name),
Value: proto.Float64(value),
Unit: proto.String(unit),
}
}
//See another version of this test in proto/extensions_test.go
func TestGetExtensionStability(t *testing.T) {
check := func(m *NoExtensionsMap) bool {
ext1, err := proto.GetExtension(m, E_FieldB1)
if err != nil {
t.Fatalf("GetExtension() failed: %s", err)
}
ext2, err := proto.GetExtension(m, E_FieldB1)
if err != nil {
t.Fatalf("GetExtension() failed: %s", err)
}
return ext1.(*NinOptNative).Equal(ext2)
}
msg := &NoExtensionsMap{Field1: proto.Int64(2)}
ext0 := &NinOptNative{Field1: proto.Float64(1)}
if err := proto.SetExtension(msg, E_FieldB1, ext0); err != nil {
t.Fatalf("Could not set ext1: %s", ext0)
}
if !check(msg) {
t.Errorf("GetExtension() not stable before marshaling")
}
bb, err := proto.Marshal(msg)
if err != nil {
t.Fatalf("Marshal() failed: %s", err)
}
msg1 := &NoExtensionsMap{}
err = proto.Unmarshal(bb, msg1)
if err != nil {
t.Fatalf("Unmarshal() failed: %s", err)
}
if !check(msg1) {
t.Errorf("GetExtension() not stable after unmarshaling")
}
}
func NewScalarResource(name string, val float64) *mesos.Resource {
return &mesos.Resource{
Name: proto.String(name),
Type: mesos.Value_SCALAR.Enum(),
Scalar: &mesos.Value_Scalar{Value: proto.Float64(val)},
}
}
//NewDoubleConst returns a new terminal expression containing the given double value.
// double(d)
func NewDoubleConst(d float64) *Expr {
return &Expr{
Terminal: &Terminal{
DoubleValue: proto.Float64(d),
},
}
}
func TestExtend(t *testing.T) {
fp, err := fieldpath.NewFloat64Path("test", "MyExtendable", test.ThetestDescription(), "FieldA")
if err != nil {
panic(err)
}
m := &test.MyExtendable{}
err = proto.SetExtension(m, test.E_FieldA, proto.Float64(10.0))
if err != nil {
panic(err)
}
buf, err := proto.Marshal(m)
if err != nil {
panic(err)
}
var unmarshalled float64
f := FuncHandler{
Float64Func: func(v float64) {
t.Logf("unmarshalled %v", v)
unmarshalled = v
},
}
unmarshaler := fieldpath.NewFloat64Unmarshaler(fp, f)
err = unmarshaler.Unmarshal(buf)
if err != nil {
panic(err)
}
if unmarshalled != float64(10.0) {
panic(fmt.Errorf("wtf %v", unmarshalled))
}
}
func writeValueMessage(port int) {
conn, err := net.Dial("udp", fmt.Sprintf("localhost:%d", port))
Expect(err).ToNot(HaveOccurred())
message := &events.Envelope{
EventType: events.Envelope_ValueMetric.Enum(),
Origin: proto.String("someorigin"),
ValueMetric: &events.ValueMetric{
Name: proto.String("some name"),
Value: proto.Float64(24.0),
Unit: proto.String("some unit"),
},
}
messageBytes, err := proto.Marshal(message)
Expect(err).ToNot(HaveOccurred())
// Pad the first 32 bytes of the payload with zeroes
// In reality this would be the signature
padding := make([]byte, 32)
payload := append(padding, messageBytes...)
conn.Write(payload)
}
func NewStatusUpdate(frameworkId *mesos.FrameworkID, taskStatus *mesos.TaskStatus, timestamp float64, uuid []byte) *mesos.StatusUpdate {
return &mesos.StatusUpdate{
FrameworkId: frameworkId,
Status: taskStatus,
Timestamp: proto.Float64(timestamp),
Uuid: uuid,
}
}
func sendEventsThroughFirehose(fakeFirehoseInputChan chan *events.Envelope) {
fakeFirehoseInputChan <- &events.Envelope{
Origin: proto.String("origin"),
Timestamp: proto.Int64(1000000000),
EventType: events.Envelope_ValueMetric.Enum(),
ValueMetric: &events.ValueMetric{
Name: proto.String("metricName"),
Value: proto.Float64(5),
Unit: proto.String("gauge"),
},
Deployment: proto.String("deployment-name"),
Job: proto.String("doppler"),
Index: proto.String("SOME-METRIC-GUID"),
}
fakeFirehoseInputChan <- &events.Envelope{
Origin: proto.String("origin"),
Timestamp: proto.Int64(2000000000),
EventType: events.Envelope_ValueMetric.Enum(),
ValueMetric: &events.ValueMetric{
Name: proto.String("metricName"),
Value: proto.Float64(10),
Unit: proto.String("gauge"),
},
Deployment: proto.String("deployment-name"),
Job: proto.String("gorouter"),
Index: proto.String("SOME-METRIC-GUID-2"),
}
fakeFirehoseInputChan <- &events.Envelope{
Origin: proto.String("origin"),
Timestamp: proto.Int64(3000000000),
EventType: events.Envelope_CounterEvent.Enum(),
CounterEvent: &events.CounterEvent{
Name: proto.String("counterName"),
Delta: proto.Uint64(3),
Total: proto.Uint64(15),
},
Deployment: proto.String("deployment-name"),
Job: proto.String("doppler"),
Index: proto.String("SOME-METRIC-GUID-3"),
}
close(fakeFirehoseInputChan)
}
func (l *logSender) emitCounters() {
l.Lock()
defer l.Unlock()
l.eventEmitter.Emit(&events.ValueMetric{
Name: proto.String("logSenderTotalMessagesRead"),
Value: proto.Float64(l.logMessageTotalCount),
Unit: proto.String("count"),
})
for appID, count := range l.logMessageReceiveCounts {
l.eventEmitter.Emit(&events.ValueMetric{
Name: proto.String("logSenderTotalMessagesRead." + appID),
Value: proto.Float64(count),
Unit: proto.String("count"),
})
}
}
func basicMessage() *events.Envelope {
return &events.Envelope{
EventType: events.Envelope_HttpStartStop.Enum(),
ValueMetric: &events.ValueMetric{
Name: proto.String("metricName"),
Value: proto.Float64(2.0),
Unit: proto.String("seconds"),
},
}
}
// generateComplexData generates more complicated InternalTimeSeriesData, where
// each contained point may have an explicit max and min.
func generateComplexData(
startTimestamp, sampleDuration int64, dps []completeDatapoint,
) roachpb.InternalTimeSeriesData {
result := roachpb.InternalTimeSeriesData{
StartTimestampNanos: startTimestamp,
SampleDurationNanos: sampleDuration,
Samples: make([]roachpb.InternalTimeSeriesSample, len(dps)),
}
for i, dp := range dps {
result.Samples[i] = roachpb.InternalTimeSeriesSample{
Offset: int32((dp.timestamp - startTimestamp) / sampleDuration),
Count: dp.count,
Sum: dp.sum,
Max: proto.Float64(dp.max),
Min: proto.Float64(dp.min),
}
}
return result
}
func (l *logSender) emitCounters() {
l.Lock()
defer l.Unlock()
l.eventEmitter.Emit(&events.ValueMetric{
Name: proto.String("logSenderTotalMessagesRead"),
Value: proto.Float64(l.logMessageTotalCount),
Unit: proto.String("count"),
})
}
func BasicValueMetricEnvelope(origin string) *events.Envelope {
return &events.Envelope{
Origin: proto.String(origin),
EventType: events.Envelope_ValueMetric.Enum(),
ValueMetric: &events.ValueMetric{
Name: proto.String("fake-metric-name"),
Value: proto.Float64(42),
Unit: proto.String("fake-unit"),
},
}
}
func createValueMessage() *events.Envelope {
return &events.Envelope{
Origin: proto.String("fake-origin-2"),
EventType: events.Envelope_ValueMetric.Enum(),
ValueMetric: &events.ValueMetric{
Name: proto.String("fake-metric-name"),
Value: proto.Float64(42),
Unit: proto.String("fake-unit"),
},
}
}
// Code taken from github.com/bigdatadev/goryman
func EventToPbEvent(event *Event) (*proto.Event, error) {
var e proto.Event
t := reflect.ValueOf(&e).Elem()
s := reflect.ValueOf(event).Elem()
typeOfEvent := s.Type()
for i := 0; i < s.NumField(); i++ {
f := s.Field(i)
value := reflect.ValueOf(f.Interface())
if reflect.Zero(f.Type()) != value && f.Interface() != nil {
name := typeOfEvent.Field(i).Name
switch name {
case "State", "Service", "Host", "Description":
tmp := reflect.ValueOf(pb.String(value.String()))
t.FieldByName(name).Set(tmp)
case "Ttl":
tmp := reflect.ValueOf(pb.Float32(float32(value.Float())))
t.FieldByName(name).Set(tmp)
case "Time":
tmp := reflect.ValueOf(pb.Int64(value.Int()))
t.FieldByName(name).Set(tmp)
case "Tags":
tmp := reflect.ValueOf(value.Interface().([]string))
t.FieldByName(name).Set(tmp)
case "Metric":
switch reflect.TypeOf(f.Interface()).Kind() {
case reflect.Int:
tmp := reflect.ValueOf(pb.Int64(int64(value.Int())))
t.FieldByName("MetricSint64").Set(tmp)
case reflect.Float32:
tmp := reflect.ValueOf(pb.Float32(float32(value.Float())))
t.FieldByName("MetricF").Set(tmp)
case reflect.Float64:
tmp := reflect.ValueOf(pb.Float64(value.Float()))
t.FieldByName("MetricD").Set(tmp)
default:
return nil, fmt.Errorf("Metric of invalid type (type %v)",
reflect.TypeOf(f.Interface()).Kind())
}
case "Attributes":
var attrs []*proto.Attribute
for k, v := range value.Interface().(map[string]string) {
k_, v_ := k, v
attrs = append(attrs, &proto.Attribute{
Key: &k_,
Value: &v_,
})
}
t.FieldByName(name).Set(reflect.ValueOf(attrs))
}
}
}
return &e, nil
}
func encodeFloatPoint(p *FloatPoint) *internal.Point {
return &internal.Point{
Name: proto.String(p.Name),
Tags: proto.String(p.Tags.ID()),
Time: proto.Int64(p.Time),
Nil: proto.Bool(p.Nil),
Aux: encodeAux(p.Aux),
FloatValue: proto.Float64(p.Value),
}
}
func (core *Core) generateResource(task *registry.Task) {
var resources = []*mesosproto.Resource{}
resources = append(resources, &mesosproto.Resource{
Name: proto.String("cpus"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Cpus)},
})
resources = append(resources, &mesosproto.Resource{
Name: proto.String("mem"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Mem)},
})
resources = append(resources, &mesosproto.Resource{
Name: proto.String("disk"),
Type: mesosproto.Value_SCALAR.Enum(),
Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Disk)},
})
resources = append(resources, core.MergePorts(task.Ports))
task.Resources = resources
}
func createValueMetric() *events.Envelope {
return &events.Envelope{
Origin: proto.String(helpers.ORIGIN_NAME),
EventType: events.Envelope_ValueMetric.Enum(),
Timestamp: proto.Int64(time.Now().UnixNano()),
ValueMetric: &events.ValueMetric{
Name: proto.String("LATs-Value"),
Value: proto.Float64(10),
Unit: proto.String("test-unit"),
},
}
}
// FillPrometheusMetric fills the appropriate metric fields.
func (h *Histogram) FillPrometheusMetric(promMetric *prometheusgo.MetricFamily) {
// TODO(mjibson): change to a Histogram once bucket counts are reasonable
sum := &prometheusgo.Summary{}
h.mu.Lock()
maybeTick(h)
merged := h.windowed.Merge()
for _, b := range merged.CumulativeDistribution() {
sum.Quantile = append(sum.Quantile, &prometheusgo.Quantile{
Quantile: proto.Float64(b.Quantile),
Value: proto.Float64(float64(b.ValueAt)),
})
}
sum.SampleCount = proto.Uint64(uint64(merged.TotalCount()))
h.mu.Unlock()
promMetric.Type = prometheusgo.MetricType_SUMMARY.Enum()
promMetric.Metric = []*prometheusgo.Metric{
{Summary: sum},
}
}
func (s *DiegoScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
logOffers(offers)
s.offersLock.Lock()
defer s.offersLock.Unlock()
if s.holdOffer {
s.offers = append(s.offers, offers...)
} else {
offerIds := extractOfferIds(offers)
driver.LaunchTasks(offerIds, nil, &mesos.Filters{RefuseSeconds: proto.Float64(30)})
}
}
// decline declines an offer.
func (s *EtcdScheduler) decline(
driver scheduler.SchedulerDriver,
offer *mesos.Offer,
) {
log.V(2).Infof("Declining offer %s.", offer.Id.GetValue())
driver.DeclineOffer(
offer.Id,
&mesos.Filters{
// Decline offers for 5 seconds.
RefuseSeconds: proto.Float64(float64(5)),
},
)
}
func sendValue(name string, value float64, unit string) {
envelope := &events.Envelope{
EventType: events.Envelope_ValueMetric.Enum(),
Timestamp: proto.Int64(time.Now().UnixNano()),
Origin: proto.String(DEFAULT_ORIGIN),
ValueMetric: &events.ValueMetric{
Name: proto.String(name),
Value: proto.Float64(value),
Unit: proto.String(unit),
},
}
sendEnvelope(envelope)
}
func basicValueMessage() []byte {
message, _ := proto.Marshal(&events.Envelope{
Origin: proto.String("fake-origin-2"),
EventType: events.Envelope_ValueMetric.Enum(),
ValueMetric: &events.ValueMetric{
Name: proto.String("fake-metric-name"),
Value: proto.Float64(42),
Unit: proto.String("fake-unit"),
},
})
return message
}