func TestLightLOFStaateSaveLoad(t *testing.T) {
ctx := core.NewContext(nil)
c := LightLOFStateCreator{}
ls, err := c.CreateState(ctx, data.Map{
"nearest_neighbor_algorithm": data.String("minhash"),
"hash_num": data.Int(64),
"nearest_neighbor_num": data.Int(10),
"reverse_nearest_neighbor_num": data.Int(30),
})
if err != nil {
t.Fatal(err)
}
l := ls.(*lightLOFState)
for i := 0; i < 100; i++ {
if err := l.Write(ctx, &core.Tuple{
Data: data.Map{
"feature_vector": data.Map{
"n": data.Int(i),
},
},
}); err != nil {
t.Fatal(err)
}
}
Convey("Given a trained LightLOFState", t, func() {
Convey("when saving it", func() {
buf := bytes.NewBuffer(nil)
err := l.Save(ctx, buf, data.Map{})
Convey("it should succeed.", func() {
So(err, ShouldBeNil)
Convey("and the loaded state should be same.", func() {
l2, err := c.LoadState(ctx, buf, data.Map{})
So(err, ShouldBeNil)
m := l.lightLOF
m2 := l2.(*lightLOFState).lightLOF
So(m2.nn, ShouldResemble, m.nn)
So(m2.nnNum, ShouldEqual, m.nnNum)
So(m2.rnnNum, ShouldEqual, m.rnnNum)
So(m2.kdists, ShouldResemble, m.kdists)
So(m2.lrds, ShouldResemble, m.lrds)
So(m2.maxSize, ShouldEqual, m.maxSize)
So(m2.rg, ShouldNotBeNil)
fv := FeatureVector(data.Map{"n": data.Int(10)})
s, err := l.lightLOF.CalcScore(fv)
So(err, ShouldBeNil)
s2, err := l2.(*lightLOFState).lightLOF.CalcScore(fv)
So(err, ShouldBeNil)
So(s2, ShouldResemble, s)
})
})
})
})
}
func TestAROWStateSaveLoad(t *testing.T) {
ctx := core.NewContext(nil)
c := AROWStateCreator{}
as, err := c.CreateState(ctx, data.Map{
"regularization_weight": data.Float(0.001),
})
if err != nil {
t.Fatal(err)
}
a := as.(*AROWState)
labels := []data.String{"a", "b", "c", "d"}
for i := 0; i < 100; i++ {
if err := a.Write(ctx, &core.Tuple{
Data: data.Map{
"label": labels[i%len(labels)],
"feature_vector": data.Map{
"n": data.Int(i),
},
},
}); err != nil {
t.Fatal(err)
}
}
Convey("Given a trained AROWState", t, func() {
Convey("when saving it", func() {
buf := bytes.NewBuffer(nil)
err := a.Save(ctx, buf, data.Map{})
Convey("it should succeed.", func() {
So(err, ShouldBeNil)
Convey("and the loaded state should be same.", func() {
a2, err := c.LoadState(ctx, buf, data.Map{})
So(err, ShouldBeNil)
// Because AROW contains sync.RWMutex, this assertion may
// fail if its implementation changes.
So(a2, ShouldResemble, a)
fv := FeatureVector(data.Map{"n": data.Int(10)})
s, err := a.arow.Classify(fv)
So(err, ShouldBeNil)
s2, err := a2.(*AROWState).arow.Classify(fv)
So(err, ShouldBeNil)
So(s2, ShouldResemble, s)
})
})
})
})
}
func TestPassiveAggressiveStateLoad(t *testing.T) {
ctx := core.NewContext(nil)
c := PassiveAggressiveStateCreator{}
pas, err := c.CreateState(ctx, data.Map{
"regularization_weight": data.Float(3.402823e+38),
"sensitivity": data.Float(0.1),
})
if err != nil {
t.Fatal(err)
}
pa := pas.(*PassiveAggressiveState)
for i := 0; i < 100; i++ {
if err := pa.Write(ctx, &core.Tuple{
Data: data.Map{
"value": data.Float(i),
"feature_vector": data.Map{
"n": data.Int(i),
},
},
}); err != nil {
t.Fatal(err)
}
}
Convey("Given a trained PassiveAggressiveState", t, func() {
Convey("when saving it", func() {
buf := bytes.NewBuffer(nil)
err := pa.Save(ctx, buf, data.Map{})
Convey("it should succeed.", func() {
So(err, ShouldBeNil)
Convey("and the loaded state should be same.", func() {
pa2, err := c.LoadState(ctx, buf, data.Map{})
So(err, ShouldBeNil)
So(pa2, ShouldResemble, pa)
fv := FeatureVector{
"n": data.Int(123),
}
v, err := pa.pa.Estimate(fv)
So(err, ShouldBeNil)
v2, err := pa2.(*PassiveAggressiveState).pa.Estimate(fv)
So(err, ShouldBeNil)
So(v2, ShouldResemble, v)
})
})
})
})
}
func (t *Ticker) Process(ctx *core.Context, tuple *core.Tuple, w core.Writer) error {
var i int64
for ; atomic.LoadInt32(&t.stopped) == 0; i++ {
newTuple := core.NewTuple(data.Map{"tick": data.Int(i)})
if err := w.Write(ctx, newTuple); err != nil {
return err
}
time.Sleep(t.interval)
}
return nil
}
func (s *SourceCreator) GenerateStream(ctx *core.Context, w core.Writer) error {
device := new(Device)
// devName := []string{"dev1", "dev2", "dev3", "dev4", "dev5"}
// devProb := []float64{0.4, 0.3, 0.15, 0.1, 0.05}
devName := []string{"dev1", "dev2"}
devProb := []float64{0.5, 0.5}
pickDev := func() string {
r := rand.Float64()
for i, p := range devProb {
if r < p {
return devName[i]
}
r -= p
}
return devName[len(devName)-1]
}
// device.MakeDevice(pickDev())
device.num = 0
temp := &device.sensorData[0]
humid := &device.sensorData[1]
for {
device.ID = pickDev()
device.num += 1
temp.MakeData("temp", 0, 30)
humid.MakeData("humid", 0, 100)
t := core.NewTuple(data.Map{
"deviceID": data.String(device.ID),
"num": data.Int(device.num),
"time": data.Float(float64(time.Now().Second()) + float64(time.Now().Nanosecond())/1e+9),
temp.ID: data.Float(float64(temp.value)),
humid.ID: data.Float(float64(humid.value)),
})
if err := w.Write(ctx, t); err != nil {
return err
}
time.Sleep(s.interval)
}
}