func createSketch(info *abstract.Info) (*SketchProxy, error) {
var sketch abstract.Sketch
var err error
manager := storage.GetManager()
err = manager.Create(info.ID)
if err != nil {
return nil, errors.New("Error creating new sketch")
}
switch info.Type {
case abstract.HLLPP:
sketch, err = hllpp.NewSketch(info)
case abstract.TopK:
sketch, err = topk.NewSketch(info)
case abstract.CML:
sketch, err = cml.NewSketch(info)
default:
return nil, errors.New("Invalid sketch type: " + info.Type)
}
if err != nil {
return nil, errors.New("Error creating new sketch")
}
sp := SketchProxy{info, sketch, sync.RWMutex{}, 0, true}
err = storage.GetManager().Create(info.ID)
if err != nil {
return nil, err
}
sp.save(true)
go sp.autosave()
return &sp, nil
}
func loadSketch(info *abstract.Info) (*SketchProxy, error) {
var sketch abstract.Sketch
data, err := storage.GetManager().LoadData(info.ID, 0, 0)
if err != nil {
return nil, fmt.Errorf("Error loading data for sketch: %s", info.ID)
}
switch info.Type {
case abstract.HLLPP:
sketch, err = hllpp.Unmarshal(info, data)
case abstract.TopK:
sketch, err = topk.Unmarshal(info, data)
case abstract.CML:
sketch, err = cml.Unmarshal(info, data)
default:
logger.Info.Println("Invalid sketch type", info.Type)
}
sp := SketchProxy{info, sketch, sync.RWMutex{}, 0, false}
if err != nil {
return nil, fmt.Errorf("Error loading data for sketch: %s", info.ID)
}
go sp.autosave()
return &sp, nil
}
func (m *ManagerStruct) dumpInfo(i *abstract.Info) {
m.info[i.ID] = i
manager := storage.GetManager()
infoData, err := json.Marshal(i)
utils.PanicOnError(err)
manager.SaveInfo(i.ID, infoData)
}
/*
NewDomainFromData ...
*/
func NewDomainFromData(info *abstract.Info) (*Domain, error) {
manager = storage.GetManager()
data, err := manager.LoadData(info.ID, 0, 0)
counter, err := hllpp.Unmarshal(data)
if err != nil {
return nil, err
}
return &Domain{info, counter, sync.RWMutex{}}, nil
}
/*
Save ...
*/
func (d *Domain) Save() error {
count := d.impl.GetCount()
d.Info.State["count"] = uint64(count)
infoData, err := json.Marshal(d.Info)
if err != nil {
return err
}
return storage.GetManager().SaveInfo(d.Info.ID, infoData)
}
func (m *ManagerStruct) dumpInfo(info *abstract.Info) {
// FIXME: Should we panic here?
m.info[info.ID] = info
manager := storage.GetManager()
infoData, err := json.Marshal(info)
utils.PanicOnError(err)
err = manager.SaveInfo(info.ID, infoData)
utils.PanicOnError(err)
}
/*
NewDomain ...
*/
func NewDomain(info *abstract.Info) (*Domain, error) {
manager = storage.GetManager()
manager.Create(info.ID)
d := Domain{info, hllpp.New(), sync.RWMutex{}}
err := d.Save()
if err != nil {
logger.Error.Println("an error has occurred while saving domain: " + err.Error())
}
return &d, nil
}
func (bs *buckets) setBucket(i uint, bckt *bucket) error {
storageManager := storage.GetManager()
data := make([]byte, bucketSize*fingerprintSize, bucketSize*fingerprintSize)
for i, fp := range bckt {
for j, v := range fp {
data[i*fingerprintSize+j] = v
}
}
storageManager.SaveData(bs.id, data, int64(i*bucketSize*fingerprintSize))
return nil
}
func (bs *buckets) getBucket(i uint) (*bucket, error) {
storageManager := storage.GetManager()
bytes, err := storageManager.LoadData(bs.id, int64(i*bucketSize*fingerprintSize), int64(bucketSize*fingerprintSize))
if err != nil {
return nil, err
}
bckt := &bucket{}
for i, b := range bytes {
bckt[i/fingerprintSize][i%fingerprintSize] = b
}
return bckt, nil
}
func newBuckets(id string, numBuckets uint) (*buckets, error) {
storageManager := storage.GetManager()
storageManager.Create(id)
bs := &buckets{id, numBuckets}
for i := uint(0); i < numBuckets; i++ {
err := bs.setBucket(i, &bucket{})
if err != nil {
return nil, err
}
}
return bs, nil
}
/*
DeleteDomain ...
*/
func (m *ManagerStruct) DeleteDomain(domainID string) error {
if _, ok := m.domains[domainID]; !ok {
return errors.New("No such domain " + domainID)
}
delete(m.domains, domainID)
delete(m.info, domainID)
manager := storage.GetManager()
err := manager.DeleteInfo(domainID)
if err != nil {
return err
}
return manager.DeleteData(domainID)
}
func (m *ManagerStruct) loadInfo() error {
manager := storage.GetManager()
var infoStruct abstract.Info
infos, err := manager.LoadAllInfo()
if err != nil {
return err
}
for _, infoData := range infos {
json.Unmarshal(infoData, &infoStruct)
m.info[infoStruct.ID] = &infoStruct
}
return nil
}
/*
NewSketch ...
*/
func NewSketch(info *abstract.Info) (*Sketch, error) {
manager = storage.GetManager()
err := manager.Create(info.ID)
if err != nil {
logger.Error.Println("an error has occurred while creating sketch: " + err.Error())
return nil, err
}
if info.Properties["capacity"] == 0 {
info.Properties["capacity"] = defaultCapacity
}
d := Sketch{info, topk.New(int(info.Properties["capacity"]))}
return &d, nil
}
/*
DeleteSketch ...
*/
func (m *ManagerStruct) DeleteSketch(sketchID string, sketchType string) error {
id := fmt.Sprintf("%s.%s", sketchID, sketchType)
if _, ok := m.sketches[id]; !ok {
return errors.New("No such sketch " + sketchID)
}
delete(m.sketches, id)
delete(m.info, id)
manager := storage.GetManager()
err := manager.DeleteInfo(id)
if err != nil {
return err
}
return manager.DeleteData(id)
}
/*
NewDomainFromData ...
*/
func NewDomainFromData(info *abstract.Info) (*Domain, error) {
manager = storage.GetManager()
data, err := manager.LoadData(info.ID, 0, 0)
if err != nil {
return nil, err
}
var network bytes.Buffer // Stand-in for a network connection
network.Write(data)
dec := gob.NewDecoder(&network) // Will read from network.
var counter topk.Stream
err = dec.Decode(&counter)
if err != nil {
return nil, err
}
return &Domain{info, &counter, sync.RWMutex{}}, nil
}
func (m *ManagerStruct) loadDomains() error {
strg := storage.GetManager()
for key, info := range m.info {
var domain abstract.Counter
var err error
switch info.Type {
case abstract.Cardinality:
domain, err = hllpp.NewDomainFromData(info)
case abstract.PurgableCardinality:
domain, err = cuckoofilter.NewDomain(info)
case abstract.TopK:
domain, err = topk.NewDomainFromData(info)
default:
logger.Info.Println("Invalid counter type", info.Type)
}
if err != nil {
errTxt := fmt.Sprint("Could not load domain ", info, ". Err: ", err)
return errors.New(errTxt)
}
m.domains[info.ID] = domain
strg.LoadData(key, 0, 0)
}
return nil
}
/*
save ...
*/
func (sp *SketchProxy) save(force bool) {
if !sp.dirty {
return
}
if sp.ops%config.GetConfig().SaveThresholdOps == 0 || force {
sp.ops++
sp.dirty = false
manager := storage.GetManager()
serialized, err := sp.sketch.Marshal()
if err != nil {
logger.Error.Println(err)
}
err = manager.SaveData(sp.Info.ID, serialized, 0)
if err != nil {
logger.Error.Println(err)
}
info, _ := json.Marshal(sp.Info)
err = manager.SaveInfo(sp.Info.ID, info)
if err != nil {
logger.Error.Println(err)
}
}
}