Beispiel #1
0
// BuildEmptyResources build empty resources
func BuildEmptyResources() []*mesosproto.Resource {
	var resources = []*mesosproto.Resource{}
	resources = append(resources, &mesosproto.Resource{
		Name:   proto.String("cpus"),
		Type:   mesosproto.Value_SCALAR.Enum(),
		Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(minCpus)},
	})
	resources = append(resources, &mesosproto.Resource{
		Name:   proto.String("mem"),
		Type:   mesosproto.Value_SCALAR.Enum(),
		Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(minMem)},
	})
	return resources
}
Beispiel #2
0
func (driver *MesosExecutorDriver) makeStatusUpdate(taskStatus *mesosproto.TaskStatus) *mesosproto.StatusUpdate {
	now := float64(time.Now().Unix())
	// Fill in all the fields.
	taskStatus.Timestamp = proto.Float64(now)
	taskStatus.SlaveId = driver.slaveID
	update := &mesosproto.StatusUpdate{
		FrameworkId: driver.frameworkID,
		ExecutorId:  driver.executorID,
		SlaveId:     driver.slaveID,
		Status:      taskStatus,
		Timestamp:   proto.Float64(now),
		Uuid:        uuid.NewUUID(),
	}
	return update
}
Beispiel #3
0
// BuildBasicResources build basic resources
// including cpus, mem, disk, ports
func BuildBasicResources(task *registry.Task) {
	if task.Build {
		return
	}
	if task.Cpus > 0 {
		task.Resources = append(task.Resources, &mesosproto.Resource{
			Name:   proto.String("cpus"),
			Type:   mesosproto.Value_SCALAR.Enum(),
			Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Cpus)},
		})
	}

	if task.Mem > 0 {
		task.Resources = append(task.Resources, &mesosproto.Resource{
			Name:   proto.String("mem"),
			Type:   mesosproto.Value_SCALAR.Enum(),
			Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Mem)},
		})
	}

	if task.Disk > 0 {
		task.Resources = append(task.Resources, &mesosproto.Resource{
			Name:   proto.String("disk"),
			Type:   mesosproto.Value_SCALAR.Enum(),
			Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Disk)},
		})
	}

	if len(task.Ports) > 0 {
		ranges := &mesosproto.Value_Ranges{}
		for _, port := range task.Ports {
			if port.HostPort == 0 {
				continue
			}
			ranges.Range = append(ranges.Range, &mesosproto.Value_Range{
				Begin: proto.Uint64(uint64(port.HostPort)),
				End:   proto.Uint64(uint64(port.HostPort)),
			})
		}

		task.Resources = append(task.Resources, &mesosproto.Resource{
			Name:   proto.String("ports"),
			Type:   mesosproto.Value_RANGES.Enum(),
			Ranges: ranges,
		})
	}
	task.Build = true
}
Beispiel #4
0
func (sched *Scheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	logOffers(offers)
	jobs, err := getLaunchableJobs()
	if err != nil {
		log.Errorf("Unable to get pending jobs! %s\n", err.Error())
		return
	}

	offersAndTasks, err := packJobsInOffers(jobs, offers)
	if err != nil {
		log.Errorf("Unable to pack jobs into offers! %s\n", err.Error())
		return
	}

	for _, ot := range offersAndTasks {
		if len(ot.Tasks) == 0 {
			log.Infof("Declining unused offer %s", ot.Offer.Id.GetValue())
			driver.DeclineOffer(ot.Offer.Id, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
			continue
		} else {
			log.Infof("Launching %d tasks for offer %s\n", len(ot.Tasks), ot.Offer.Id.GetValue())
			driver.LaunchTasks([]*mesos.OfferID{ot.Offer.Id}, ot.Tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
			sched.tasksLaunched = sched.tasksLaunched + len(ot.Tasks)
		}
	}

}
func TestUnmarshalPartiallyPopulatedOptionalFieldsFails(t *testing.T) {
	// Fill in all fields, then randomly remove one.
	dataOut := &test.NinOptNative{
		Field1:  proto.Float64(0),
		Field2:  proto.Float32(0),
		Field3:  proto.Int32(0),
		Field4:  proto.Int64(0),
		Field5:  proto.Uint32(0),
		Field6:  proto.Uint64(0),
		Field7:  proto.Int32(0),
		Field8:  proto.Int64(0),
		Field9:  proto.Uint32(0),
		Field10: proto.Int32(0),
		Field11: proto.Uint64(0),
		Field12: proto.Int64(0),
		Field13: proto.Bool(false),
		Field14: proto.String("0"),
		Field15: []byte("0"),
	}
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	fieldName := "Field" + strconv.Itoa(r.Intn(15)+1)
	field := reflect.ValueOf(dataOut).Elem().FieldByName(fieldName)
	fieldType := field.Type()
	field.Set(reflect.Zero(fieldType))
	encodedMessage, err := proto.Marshal(dataOut)
	if err != nil {
		t.Fatalf("Unexpected error when marshalling dataOut: %v", err)
	}
	dataIn := NidOptNative{}
	err = proto.Unmarshal(encodedMessage, &dataIn)
	if err.Error() != `proto: required field "`+fieldName+`" not set` {
		t.Fatalf(`err.Error() != "proto: required field "`+fieldName+`" not set"; was "%s" instead`, err.Error())
	}
}
Beispiel #6
0
func encodeAux(aux []interface{}) []*internal.Aux {
	pb := make([]*internal.Aux, len(aux))
	for i := range aux {
		switch v := aux[i].(type) {
		case float64:
			pb[i] = &internal.Aux{DataType: proto.Int32(Float), FloatValue: proto.Float64(v)}
		case *float64:
			pb[i] = &internal.Aux{DataType: proto.Int32(Float)}
		case int64:
			pb[i] = &internal.Aux{DataType: proto.Int32(Integer), IntegerValue: proto.Int64(v)}
		case *int64:
			pb[i] = &internal.Aux{DataType: proto.Int32(Integer)}
		case string:
			pb[i] = &internal.Aux{DataType: proto.Int32(String), StringValue: proto.String(v)}
		case *string:
			pb[i] = &internal.Aux{DataType: proto.Int32(String)}
		case bool:
			pb[i] = &internal.Aux{DataType: proto.Int32(Boolean), BooleanValue: proto.Bool(v)}
		case *bool:
			pb[i] = &internal.Aux{DataType: proto.Int32(Boolean)}
		default:
			pb[i] = &internal.Aux{DataType: proto.Int32(int32(Unknown))}
		}
	}
	return pb
}
Beispiel #7
0
func newScalarAttribute(name string, val float64) *mesos.Attribute {
	return &mesos.Attribute{
		Name:   proto.String(name),
		Type:   mesos.Value_SCALAR.Enum(),
		Scalar: &mesos.Value_Scalar{Value: proto.Float64(val)},
	}
}
func basicValueMetric(name string, value float64, unit string) *events.ValueMetric {
	return &events.ValueMetric{
		Name:  proto.String(name),
		Value: proto.Float64(value),
		Unit:  proto.String(unit),
	}
}
//See another version of this test in proto/extensions_test.go
func TestGetExtensionStability(t *testing.T) {
	check := func(m *NoExtensionsMap) bool {
		ext1, err := proto.GetExtension(m, E_FieldB1)
		if err != nil {
			t.Fatalf("GetExtension() failed: %s", err)
		}
		ext2, err := proto.GetExtension(m, E_FieldB1)
		if err != nil {
			t.Fatalf("GetExtension() failed: %s", err)
		}
		return ext1.(*NinOptNative).Equal(ext2)
	}
	msg := &NoExtensionsMap{Field1: proto.Int64(2)}
	ext0 := &NinOptNative{Field1: proto.Float64(1)}
	if err := proto.SetExtension(msg, E_FieldB1, ext0); err != nil {
		t.Fatalf("Could not set ext1: %s", ext0)
	}
	if !check(msg) {
		t.Errorf("GetExtension() not stable before marshaling")
	}
	bb, err := proto.Marshal(msg)
	if err != nil {
		t.Fatalf("Marshal() failed: %s", err)
	}
	msg1 := &NoExtensionsMap{}
	err = proto.Unmarshal(bb, msg1)
	if err != nil {
		t.Fatalf("Unmarshal() failed: %s", err)
	}
	if !check(msg1) {
		t.Errorf("GetExtension() not stable after unmarshaling")
	}
}
Beispiel #10
0
func NewScalarResource(name string, val float64) *mesos.Resource {
	return &mesos.Resource{
		Name:   proto.String(name),
		Type:   mesos.Value_SCALAR.Enum(),
		Scalar: &mesos.Value_Scalar{Value: proto.Float64(val)},
	}
}
Beispiel #11
0
//NewDoubleConst returns a new terminal expression containing the given double value.
//  double(d)
func NewDoubleConst(d float64) *Expr {
	return &Expr{
		Terminal: &Terminal{
			DoubleValue: proto.Float64(d),
		},
	}
}
Beispiel #12
0
func TestExtend(t *testing.T) {
	fp, err := fieldpath.NewFloat64Path("test", "MyExtendable", test.ThetestDescription(), "FieldA")
	if err != nil {
		panic(err)
	}
	m := &test.MyExtendable{}
	err = proto.SetExtension(m, test.E_FieldA, proto.Float64(10.0))
	if err != nil {
		panic(err)
	}
	buf, err := proto.Marshal(m)
	if err != nil {
		panic(err)
	}
	var unmarshalled float64
	f := FuncHandler{
		Float64Func: func(v float64) {
			t.Logf("unmarshalled %v", v)
			unmarshalled = v
		},
	}
	unmarshaler := fieldpath.NewFloat64Unmarshaler(fp, f)
	err = unmarshaler.Unmarshal(buf)
	if err != nil {
		panic(err)
	}
	if unmarshalled != float64(10.0) {
		panic(fmt.Errorf("wtf %v", unmarshalled))
	}
}
func writeValueMessage(port int) {
	conn, err := net.Dial("udp", fmt.Sprintf("localhost:%d", port))
	Expect(err).ToNot(HaveOccurred())

	message := &events.Envelope{
		EventType: events.Envelope_ValueMetric.Enum(),
		Origin:    proto.String("someorigin"),

		ValueMetric: &events.ValueMetric{
			Name:  proto.String("some name"),
			Value: proto.Float64(24.0),
			Unit:  proto.String("some unit"),
		},
	}

	messageBytes, err := proto.Marshal(message)
	Expect(err).ToNot(HaveOccurred())

	// Pad the first 32 bytes of the payload with zeroes
	// In reality this would be the signature

	padding := make([]byte, 32)

	payload := append(padding, messageBytes...)
	conn.Write(payload)
}
Beispiel #14
0
func NewStatusUpdate(frameworkId *mesos.FrameworkID, taskStatus *mesos.TaskStatus, timestamp float64, uuid []byte) *mesos.StatusUpdate {
	return &mesos.StatusUpdate{
		FrameworkId: frameworkId,
		Status:      taskStatus,
		Timestamp:   proto.Float64(timestamp),
		Uuid:        uuid,
	}
}
func sendEventsThroughFirehose(fakeFirehoseInputChan chan *events.Envelope) {
	fakeFirehoseInputChan <- &events.Envelope{
		Origin:    proto.String("origin"),
		Timestamp: proto.Int64(1000000000),
		EventType: events.Envelope_ValueMetric.Enum(),
		ValueMetric: &events.ValueMetric{
			Name:  proto.String("metricName"),
			Value: proto.Float64(5),
			Unit:  proto.String("gauge"),
		},
		Deployment: proto.String("deployment-name"),
		Job:        proto.String("doppler"),
		Index:      proto.String("SOME-METRIC-GUID"),
	}

	fakeFirehoseInputChan <- &events.Envelope{
		Origin:    proto.String("origin"),
		Timestamp: proto.Int64(2000000000),
		EventType: events.Envelope_ValueMetric.Enum(),
		ValueMetric: &events.ValueMetric{
			Name:  proto.String("metricName"),
			Value: proto.Float64(10),
			Unit:  proto.String("gauge"),
		},
		Deployment: proto.String("deployment-name"),
		Job:        proto.String("gorouter"),
		Index:      proto.String("SOME-METRIC-GUID-2"),
	}

	fakeFirehoseInputChan <- &events.Envelope{
		Origin:    proto.String("origin"),
		Timestamp: proto.Int64(3000000000),
		EventType: events.Envelope_CounterEvent.Enum(),
		CounterEvent: &events.CounterEvent{
			Name:  proto.String("counterName"),
			Delta: proto.Uint64(3),
			Total: proto.Uint64(15),
		},
		Deployment: proto.String("deployment-name"),
		Job:        proto.String("doppler"),
		Index:      proto.String("SOME-METRIC-GUID-3"),
	}

	close(fakeFirehoseInputChan)
}
Beispiel #16
0
func (l *logSender) emitCounters() {
	l.Lock()
	defer l.Unlock()

	l.eventEmitter.Emit(&events.ValueMetric{
		Name:  proto.String("logSenderTotalMessagesRead"),
		Value: proto.Float64(l.logMessageTotalCount),
		Unit:  proto.String("count"),
	})

	for appID, count := range l.logMessageReceiveCounts {
		l.eventEmitter.Emit(&events.ValueMetric{
			Name:  proto.String("logSenderTotalMessagesRead." + appID),
			Value: proto.Float64(count),
			Unit:  proto.String("count"),
		})
	}
}
Beispiel #17
0
func basicMessage() *events.Envelope {
	return &events.Envelope{
		EventType: events.Envelope_HttpStartStop.Enum(),
		ValueMetric: &events.ValueMetric{
			Name:  proto.String("metricName"),
			Value: proto.Float64(2.0),
			Unit:  proto.String("seconds"),
		},
	}
}
Beispiel #18
0
// generateComplexData generates more complicated InternalTimeSeriesData, where
// each contained point may have an explicit max and min.
func generateComplexData(
	startTimestamp, sampleDuration int64, dps []completeDatapoint,
) roachpb.InternalTimeSeriesData {
	result := roachpb.InternalTimeSeriesData{
		StartTimestampNanos: startTimestamp,
		SampleDurationNanos: sampleDuration,
		Samples:             make([]roachpb.InternalTimeSeriesSample, len(dps)),
	}
	for i, dp := range dps {
		result.Samples[i] = roachpb.InternalTimeSeriesSample{
			Offset: int32((dp.timestamp - startTimestamp) / sampleDuration),
			Count:  dp.count,
			Sum:    dp.sum,
			Max:    proto.Float64(dp.max),
			Min:    proto.Float64(dp.min),
		}
	}
	return result
}
func (l *logSender) emitCounters() {
	l.Lock()
	defer l.Unlock()

	l.eventEmitter.Emit(&events.ValueMetric{
		Name:  proto.String("logSenderTotalMessagesRead"),
		Value: proto.Float64(l.logMessageTotalCount),
		Unit:  proto.String("count"),
	})
}
Beispiel #20
0
func BasicValueMetricEnvelope(origin string) *events.Envelope {
	return &events.Envelope{
		Origin:    proto.String(origin),
		EventType: events.Envelope_ValueMetric.Enum(),
		ValueMetric: &events.ValueMetric{
			Name:  proto.String("fake-metric-name"),
			Value: proto.Float64(42),
			Unit:  proto.String("fake-unit"),
		},
	}
}
func createValueMessage() *events.Envelope {
	return &events.Envelope{
		Origin:    proto.String("fake-origin-2"),
		EventType: events.Envelope_ValueMetric.Enum(),
		ValueMetric: &events.ValueMetric{
			Name:  proto.String("fake-metric-name"),
			Value: proto.Float64(42),
			Unit:  proto.String("fake-unit"),
		},
	}
}
// Code taken from github.com/bigdatadev/goryman
func EventToPbEvent(event *Event) (*proto.Event, error) {
	var e proto.Event
	t := reflect.ValueOf(&e).Elem()
	s := reflect.ValueOf(event).Elem()
	typeOfEvent := s.Type()

	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		value := reflect.ValueOf(f.Interface())
		if reflect.Zero(f.Type()) != value && f.Interface() != nil {
			name := typeOfEvent.Field(i).Name
			switch name {
			case "State", "Service", "Host", "Description":
				tmp := reflect.ValueOf(pb.String(value.String()))
				t.FieldByName(name).Set(tmp)
			case "Ttl":
				tmp := reflect.ValueOf(pb.Float32(float32(value.Float())))
				t.FieldByName(name).Set(tmp)
			case "Time":
				tmp := reflect.ValueOf(pb.Int64(value.Int()))
				t.FieldByName(name).Set(tmp)
			case "Tags":
				tmp := reflect.ValueOf(value.Interface().([]string))
				t.FieldByName(name).Set(tmp)
			case "Metric":
				switch reflect.TypeOf(f.Interface()).Kind() {
				case reflect.Int:
					tmp := reflect.ValueOf(pb.Int64(int64(value.Int())))
					t.FieldByName("MetricSint64").Set(tmp)
				case reflect.Float32:
					tmp := reflect.ValueOf(pb.Float32(float32(value.Float())))
					t.FieldByName("MetricF").Set(tmp)
				case reflect.Float64:
					tmp := reflect.ValueOf(pb.Float64(value.Float()))
					t.FieldByName("MetricD").Set(tmp)
				default:
					return nil, fmt.Errorf("Metric of invalid type (type %v)",
						reflect.TypeOf(f.Interface()).Kind())
				}
			case "Attributes":
				var attrs []*proto.Attribute
				for k, v := range value.Interface().(map[string]string) {
					k_, v_ := k, v
					attrs = append(attrs, &proto.Attribute{
						Key:   &k_,
						Value: &v_,
					})
				}
				t.FieldByName(name).Set(reflect.ValueOf(attrs))
			}
		}
	}
	return &e, nil
}
Beispiel #23
0
func encodeFloatPoint(p *FloatPoint) *internal.Point {
	return &internal.Point{
		Name: proto.String(p.Name),
		Tags: proto.String(p.Tags.ID()),
		Time: proto.Int64(p.Time),
		Nil:  proto.Bool(p.Nil),
		Aux:  encodeAux(p.Aux),

		FloatValue: proto.Float64(p.Value),
	}
}
Beispiel #24
0
func (core *Core) generateResource(task *registry.Task) {
	var resources = []*mesosproto.Resource{}
	resources = append(resources, &mesosproto.Resource{
		Name:   proto.String("cpus"),
		Type:   mesosproto.Value_SCALAR.Enum(),
		Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Cpus)},
	})
	resources = append(resources, &mesosproto.Resource{
		Name:   proto.String("mem"),
		Type:   mesosproto.Value_SCALAR.Enum(),
		Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Mem)},
	})
	resources = append(resources, &mesosproto.Resource{
		Name:   proto.String("disk"),
		Type:   mesosproto.Value_SCALAR.Enum(),
		Scalar: &mesosproto.Value_Scalar{Value: proto.Float64(task.Disk)},
	})
	resources = append(resources, core.MergePorts(task.Ports))
	task.Resources = resources
}
func createValueMetric() *events.Envelope {
	return &events.Envelope{
		Origin:    proto.String(helpers.ORIGIN_NAME),
		EventType: events.Envelope_ValueMetric.Enum(),
		Timestamp: proto.Int64(time.Now().UnixNano()),
		ValueMetric: &events.ValueMetric{
			Name:  proto.String("LATs-Value"),
			Value: proto.Float64(10),
			Unit:  proto.String("test-unit"),
		},
	}
}
Beispiel #26
0
// FillPrometheusMetric fills the appropriate metric fields.
func (h *Histogram) FillPrometheusMetric(promMetric *prometheusgo.MetricFamily) {
	// TODO(mjibson): change to a Histogram once bucket counts are reasonable
	sum := &prometheusgo.Summary{}

	h.mu.Lock()
	maybeTick(h)
	merged := h.windowed.Merge()
	for _, b := range merged.CumulativeDistribution() {
		sum.Quantile = append(sum.Quantile, &prometheusgo.Quantile{
			Quantile: proto.Float64(b.Quantile),
			Value:    proto.Float64(float64(b.ValueAt)),
		})
	}
	sum.SampleCount = proto.Uint64(uint64(merged.TotalCount()))
	h.mu.Unlock()

	promMetric.Type = prometheusgo.MetricType_SUMMARY.Enum()
	promMetric.Metric = []*prometheusgo.Metric{
		{Summary: sum},
	}
}
func (s *DiegoScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	logOffers(offers)

	s.offersLock.Lock()
	defer s.offersLock.Unlock()

	if s.holdOffer {
		s.offers = append(s.offers, offers...)
	} else {
		offerIds := extractOfferIds(offers)
		driver.LaunchTasks(offerIds, nil, &mesos.Filters{RefuseSeconds: proto.Float64(30)})
	}
}
Beispiel #28
0
// decline declines an offer.
func (s *EtcdScheduler) decline(
	driver scheduler.SchedulerDriver,
	offer *mesos.Offer,
) {
	log.V(2).Infof("Declining offer %s.", offer.Id.GetValue())
	driver.DeclineOffer(
		offer.Id,
		&mesos.Filters{
			// Decline offers for 5 seconds.
			RefuseSeconds: proto.Float64(float64(5)),
		},
	)
}
Beispiel #29
0
func sendValue(name string, value float64, unit string) {
	envelope := &events.Envelope{
		EventType: events.Envelope_ValueMetric.Enum(),
		Timestamp: proto.Int64(time.Now().UnixNano()),
		Origin:    proto.String(DEFAULT_ORIGIN),
		ValueMetric: &events.ValueMetric{
			Name:  proto.String(name),
			Value: proto.Float64(value),
			Unit:  proto.String(unit),
		},
	}
	sendEnvelope(envelope)
}
func basicValueMessage() []byte {
	message, _ := proto.Marshal(&events.Envelope{
		Origin:    proto.String("fake-origin-2"),
		EventType: events.Envelope_ValueMetric.Enum(),
		ValueMetric: &events.ValueMetric{
			Name:  proto.String("fake-metric-name"),
			Value: proto.Float64(42),
			Unit:  proto.String("fake-unit"),
		},
	})

	return message
}