Пример #1
0
func TestLightLOFStaateSaveLoad(t *testing.T) {
	ctx := core.NewContext(nil)
	c := LightLOFStateCreator{}
	ls, err := c.CreateState(ctx, data.Map{
		"nearest_neighbor_algorithm":   data.String("minhash"),
		"hash_num":                     data.Int(64),
		"nearest_neighbor_num":         data.Int(10),
		"reverse_nearest_neighbor_num": data.Int(30),
	})
	if err != nil {
		t.Fatal(err)
	}
	l := ls.(*lightLOFState)

	for i := 0; i < 100; i++ {
		if err := l.Write(ctx, &core.Tuple{
			Data: data.Map{
				"feature_vector": data.Map{
					"n": data.Int(i),
				},
			},
		}); err != nil {
			t.Fatal(err)
		}
	}

	Convey("Given a trained LightLOFState", t, func() {
		Convey("when saving it", func() {
			buf := bytes.NewBuffer(nil)
			err := l.Save(ctx, buf, data.Map{})

			Convey("it should succeed.", func() {
				So(err, ShouldBeNil)

				Convey("and the loaded state should be same.", func() {
					l2, err := c.LoadState(ctx, buf, data.Map{})
					So(err, ShouldBeNil)

					m := l.lightLOF
					m2 := l2.(*lightLOFState).lightLOF
					So(m2.nn, ShouldResemble, m.nn)
					So(m2.nnNum, ShouldEqual, m.nnNum)
					So(m2.rnnNum, ShouldEqual, m.rnnNum)
					So(m2.kdists, ShouldResemble, m.kdists)
					So(m2.lrds, ShouldResemble, m.lrds)
					So(m2.maxSize, ShouldEqual, m.maxSize)
					So(m2.rg, ShouldNotBeNil)

					fv := FeatureVector(data.Map{"n": data.Int(10)})
					s, err := l.lightLOF.CalcScore(fv)
					So(err, ShouldBeNil)
					s2, err := l2.(*lightLOFState).lightLOF.CalcScore(fv)
					So(err, ShouldBeNil)
					So(s2, ShouldResemble, s)
				})
			})
		})
	})
}
Пример #2
0
func TestAROWStateSaveLoad(t *testing.T) {
	ctx := core.NewContext(nil)
	c := AROWStateCreator{}
	as, err := c.CreateState(ctx, data.Map{
		"regularization_weight": data.Float(0.001),
	})
	if err != nil {
		t.Fatal(err)
	}
	a := as.(*AROWState)

	labels := []data.String{"a", "b", "c", "d"}
	for i := 0; i < 100; i++ {
		if err := a.Write(ctx, &core.Tuple{
			Data: data.Map{
				"label": labels[i%len(labels)],
				"feature_vector": data.Map{
					"n": data.Int(i),
				},
			},
		}); err != nil {
			t.Fatal(err)
		}
	}

	Convey("Given a trained AROWState", t, func() {
		Convey("when saving it", func() {
			buf := bytes.NewBuffer(nil)
			err := a.Save(ctx, buf, data.Map{})

			Convey("it should succeed.", func() {
				So(err, ShouldBeNil)

				Convey("and the loaded state should be same.", func() {
					a2, err := c.LoadState(ctx, buf, data.Map{})
					So(err, ShouldBeNil)

					// Because AROW contains sync.RWMutex, this assertion may
					// fail if its implementation changes.
					So(a2, ShouldResemble, a)

					fv := FeatureVector(data.Map{"n": data.Int(10)})
					s, err := a.arow.Classify(fv)
					So(err, ShouldBeNil)
					s2, err := a2.(*AROWState).arow.Classify(fv)
					So(err, ShouldBeNil)
					So(s2, ShouldResemble, s)
				})
			})
		})
	})
}
func TestPassiveAggressiveStateLoad(t *testing.T) {
	ctx := core.NewContext(nil)
	c := PassiveAggressiveStateCreator{}
	pas, err := c.CreateState(ctx, data.Map{
		"regularization_weight": data.Float(3.402823e+38),
		"sensitivity":           data.Float(0.1),
	})
	if err != nil {
		t.Fatal(err)
	}
	pa := pas.(*PassiveAggressiveState)

	for i := 0; i < 100; i++ {
		if err := pa.Write(ctx, &core.Tuple{
			Data: data.Map{
				"value": data.Float(i),
				"feature_vector": data.Map{
					"n": data.Int(i),
				},
			},
		}); err != nil {
			t.Fatal(err)
		}
	}

	Convey("Given a trained PassiveAggressiveState", t, func() {
		Convey("when saving it", func() {
			buf := bytes.NewBuffer(nil)
			err := pa.Save(ctx, buf, data.Map{})

			Convey("it should succeed.", func() {
				So(err, ShouldBeNil)

				Convey("and the loaded state should be same.", func() {
					pa2, err := c.LoadState(ctx, buf, data.Map{})
					So(err, ShouldBeNil)

					So(pa2, ShouldResemble, pa)

					fv := FeatureVector{
						"n": data.Int(123),
					}
					v, err := pa.pa.Estimate(fv)
					So(err, ShouldBeNil)
					v2, err := pa2.(*PassiveAggressiveState).pa.Estimate(fv)
					So(err, ShouldBeNil)
					So(v2, ShouldResemble, v)
				})
			})
		})
	})
}
Пример #4
0
func (t *Ticker) Process(ctx *core.Context, tuple *core.Tuple, w core.Writer) error {
	var i int64
	for ; atomic.LoadInt32(&t.stopped) == 0; i++ {
		newTuple := core.NewTuple(data.Map{"tick": data.Int(i)})
		if err := w.Write(ctx, newTuple); err != nil {
			return err
		}
		time.Sleep(t.interval)
	}
	return nil
}
Пример #5
0
func (s *SourceCreator) GenerateStream(ctx *core.Context, w core.Writer) error {
	device := new(Device)
	//	devName := []string{"dev1", "dev2", "dev3", "dev4", "dev5"}
	//	devProb := []float64{0.4, 0.3, 0.15, 0.1, 0.05}
	devName := []string{"dev1", "dev2"}
	devProb := []float64{0.5, 0.5}
	pickDev := func() string {
		r := rand.Float64()
		for i, p := range devProb {
			if r < p {
				return devName[i]
			}
			r -= p
		}
		return devName[len(devName)-1]
	}

	//	device.MakeDevice(pickDev())
	device.num = 0
	temp := &device.sensorData[0]
	humid := &device.sensorData[1]

	for {
		device.ID = pickDev()
		device.num += 1
		temp.MakeData("temp", 0, 30)
		humid.MakeData("humid", 0, 100)

		t := core.NewTuple(data.Map{
			"deviceID": data.String(device.ID),
			"num":      data.Int(device.num),
			"time":     data.Float(float64(time.Now().Second()) + float64(time.Now().Nanosecond())/1e+9),
			temp.ID:    data.Float(float64(temp.value)),
			humid.ID:   data.Float(float64(humid.value)),
		})
		if err := w.Write(ctx, t); err != nil {
			return err
		}
		time.Sleep(s.interval)
	}
}