Example #1
0
func createSketch(info *abstract.Info) (*SketchProxy, error) {
	var sketch abstract.Sketch
	var err error
	manager := storage.GetManager()
	err = manager.Create(info.ID)
	if err != nil {
		return nil, errors.New("Error creating new sketch")
	}

	switch info.Type {
	case abstract.HLLPP:
		sketch, err = hllpp.NewSketch(info)
	case abstract.TopK:
		sketch, err = topk.NewSketch(info)
	case abstract.CML:
		sketch, err = cml.NewSketch(info)
	default:
		return nil, errors.New("Invalid sketch type: " + info.Type)
	}
	if err != nil {
		return nil, errors.New("Error creating new sketch")
	}

	sp := SketchProxy{info, sketch, sync.RWMutex{}, 0, true}
	err = storage.GetManager().Create(info.ID)
	if err != nil {
		return nil, err
	}

	sp.save(true)
	go sp.autosave()
	return &sp, nil
}
Example #2
0
func loadSketch(info *abstract.Info) (*SketchProxy, error) {
	var sketch abstract.Sketch

	data, err := storage.GetManager().LoadData(info.ID, 0, 0)
	if err != nil {
		return nil, fmt.Errorf("Error loading data for sketch: %s", info.ID)
	}

	switch info.Type {
	case abstract.HLLPP:
		sketch, err = hllpp.Unmarshal(info, data)
	case abstract.TopK:
		sketch, err = topk.Unmarshal(info, data)
	case abstract.CML:
		sketch, err = cml.Unmarshal(info, data)
	default:
		logger.Info.Println("Invalid sketch type", info.Type)
	}
	sp := SketchProxy{info, sketch, sync.RWMutex{}, 0, false}

	if err != nil {
		return nil, fmt.Errorf("Error loading data for sketch: %s", info.ID)
	}

	go sp.autosave()
	return &sp, nil
}
Example #3
0
func (m *ManagerStruct) dumpInfo(i *abstract.Info) {
	m.info[i.ID] = i
	manager := storage.GetManager()
	infoData, err := json.Marshal(i)
	utils.PanicOnError(err)
	manager.SaveInfo(i.ID, infoData)
}
Example #4
0
/*
NewDomainFromData ...
*/
func NewDomainFromData(info *abstract.Info) (*Domain, error) {
	manager = storage.GetManager()
	data, err := manager.LoadData(info.ID, 0, 0)
	counter, err := hllpp.Unmarshal(data)
	if err != nil {
		return nil, err
	}
	return &Domain{info, counter, sync.RWMutex{}}, nil
}
Example #5
0
/*
Save ...
*/
func (d *Domain) Save() error {
	count := d.impl.GetCount()
	d.Info.State["count"] = uint64(count)
	infoData, err := json.Marshal(d.Info)
	if err != nil {
		return err
	}
	return storage.GetManager().SaveInfo(d.Info.ID, infoData)
}
Example #6
0
func (m *ManagerStruct) dumpInfo(info *abstract.Info) {
	// FIXME: Should we panic here?
	m.info[info.ID] = info
	manager := storage.GetManager()
	infoData, err := json.Marshal(info)
	utils.PanicOnError(err)
	err = manager.SaveInfo(info.ID, infoData)
	utils.PanicOnError(err)
}
Example #7
0
/*
NewDomain ...
*/
func NewDomain(info *abstract.Info) (*Domain, error) {
	manager = storage.GetManager()
	manager.Create(info.ID)
	d := Domain{info, hllpp.New(), sync.RWMutex{}}
	err := d.Save()
	if err != nil {
		logger.Error.Println("an error has occurred while saving domain: " + err.Error())
	}
	return &d, nil
}
Example #8
0
func (bs *buckets) setBucket(i uint, bckt *bucket) error {
	storageManager := storage.GetManager()
	data := make([]byte, bucketSize*fingerprintSize, bucketSize*fingerprintSize)
	for i, fp := range bckt {
		for j, v := range fp {
			data[i*fingerprintSize+j] = v
		}
	}
	storageManager.SaveData(bs.id, data, int64(i*bucketSize*fingerprintSize))
	return nil
}
Example #9
0
func (bs *buckets) getBucket(i uint) (*bucket, error) {
	storageManager := storage.GetManager()
	bytes, err := storageManager.LoadData(bs.id, int64(i*bucketSize*fingerprintSize), int64(bucketSize*fingerprintSize))
	if err != nil {
		return nil, err
	}
	bckt := &bucket{}
	for i, b := range bytes {
		bckt[i/fingerprintSize][i%fingerprintSize] = b
	}
	return bckt, nil
}
Example #10
0
func newBuckets(id string, numBuckets uint) (*buckets, error) {
	storageManager := storage.GetManager()
	storageManager.Create(id)
	bs := &buckets{id, numBuckets}
	for i := uint(0); i < numBuckets; i++ {
		err := bs.setBucket(i, &bucket{})
		if err != nil {
			return nil, err
		}
	}
	return bs, nil
}
Example #11
0
/*
DeleteDomain ...
*/
func (m *ManagerStruct) DeleteDomain(domainID string) error {
	if _, ok := m.domains[domainID]; !ok {
		return errors.New("No such domain " + domainID)
	}
	delete(m.domains, domainID)
	delete(m.info, domainID)
	manager := storage.GetManager()
	err := manager.DeleteInfo(domainID)
	if err != nil {
		return err
	}
	return manager.DeleteData(domainID)
}
Example #12
0
func (m *ManagerStruct) loadInfo() error {
	manager := storage.GetManager()
	var infoStruct abstract.Info
	infos, err := manager.LoadAllInfo()
	if err != nil {
		return err
	}
	for _, infoData := range infos {
		json.Unmarshal(infoData, &infoStruct)
		m.info[infoStruct.ID] = &infoStruct
	}
	return nil
}
Example #13
0
/*
NewSketch ...
*/
func NewSketch(info *abstract.Info) (*Sketch, error) {
	manager = storage.GetManager()
	err := manager.Create(info.ID)
	if err != nil {
		logger.Error.Println("an error has occurred while creating sketch: " + err.Error())
		return nil, err
	}
	if info.Properties["capacity"] == 0 {
		info.Properties["capacity"] = defaultCapacity
	}
	d := Sketch{info, topk.New(int(info.Properties["capacity"]))}

	return &d, nil
}
Example #14
0
/*
DeleteSketch ...
*/
func (m *ManagerStruct) DeleteSketch(sketchID string, sketchType string) error {
	id := fmt.Sprintf("%s.%s", sketchID, sketchType)

	if _, ok := m.sketches[id]; !ok {
		return errors.New("No such sketch " + sketchID)
	}
	delete(m.sketches, id)
	delete(m.info, id)
	manager := storage.GetManager()
	err := manager.DeleteInfo(id)
	if err != nil {
		return err
	}
	return manager.DeleteData(id)
}
Example #15
0
/*
NewDomainFromData ...
*/
func NewDomainFromData(info *abstract.Info) (*Domain, error) {
	manager = storage.GetManager()
	data, err := manager.LoadData(info.ID, 0, 0)
	if err != nil {
		return nil, err
	}

	var network bytes.Buffer // Stand-in for a network connection
	network.Write(data)
	dec := gob.NewDecoder(&network) // Will read from network.

	var counter topk.Stream
	err = dec.Decode(&counter)
	if err != nil {
		return nil, err
	}
	return &Domain{info, &counter, sync.RWMutex{}}, nil
}
Example #16
0
func (m *ManagerStruct) loadDomains() error {
	strg := storage.GetManager()
	for key, info := range m.info {
		var domain abstract.Counter
		var err error
		switch info.Type {
		case abstract.Cardinality:
			domain, err = hllpp.NewDomainFromData(info)
		case abstract.PurgableCardinality:
			domain, err = cuckoofilter.NewDomain(info)
		case abstract.TopK:
			domain, err = topk.NewDomainFromData(info)
		default:
			logger.Info.Println("Invalid counter type", info.Type)
		}
		if err != nil {
			errTxt := fmt.Sprint("Could not load domain ", info, ". Err: ", err)
			return errors.New(errTxt)
		}
		m.domains[info.ID] = domain
		strg.LoadData(key, 0, 0)
	}
	return nil
}
Example #17
0
/*
save ...
*/
func (sp *SketchProxy) save(force bool) {
	if !sp.dirty {
		return
	}

	if sp.ops%config.GetConfig().SaveThresholdOps == 0 || force {
		sp.ops++
		sp.dirty = false
		manager := storage.GetManager()
		serialized, err := sp.sketch.Marshal()
		if err != nil {
			logger.Error.Println(err)
		}
		err = manager.SaveData(sp.Info.ID, serialized, 0)
		if err != nil {
			logger.Error.Println(err)
		}
		info, _ := json.Marshal(sp.Info)
		err = manager.SaveInfo(sp.Info.ID, info)
		if err != nil {
			logger.Error.Println(err)
		}
	}
}