Beispiel #1
0
import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"utils"

	"github.com/golang/protobuf/proto"
	"github.com/njpatel/loggo"
)

var logger = loggo.GetLogger("storage")

// AOF ...
type AOF struct {
	file     *os.File
	buffer   *bufio.ReadWriter
	lock     sync.RWMutex
	inChan   chan *Entry
	tickChan <-chan time.Time
}

// NewAOF ...
func NewAOF(path string) *AOF {
	file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600)
	utils.PanicOnError(err)
	rdr := bufio.NewReader(file)
Beispiel #2
0
info_dir = "~/.skizze"

# This is where the data is stored either as json or .count (pure bytes)
data_dir = "~/.skizze/data"

# The host interface for the server
host = "localhost"

# The port number for the server
port = 3596

# Treshold for saving a sketch to disk
save_threshold_seconds = 1
`

var logger = loggo.GetLogger("config")

// Config stores all configuration parameters for Go
type Config struct {
	InfoDir              string `toml:"info_dir"`
	DataDir              string `toml:"data_dir"`
	Host                 string `toml:"host"`
	Port                 int    `toml:"port"`
	SaveThresholdSeconds uint   `toml:"save_threshold_seconds"`
}

var config *Config

// InfoDir initialized from config file
var InfoDir string
Beispiel #3
0
package server

import (
	"datamodel"
	pb "datamodel/protobuf"
	"storage"

	"github.com/gogo/protobuf/proto"
	"github.com/njpatel/loggo"
	"golang.org/x/net/context"
)

var logger = loggo.GetLogger("server")

func (s *serverStruct) createSketch(ctx context.Context, in *pb.Sketch) (*pb.Sketch, error) {
	info := &datamodel.Info{Sketch: in}
	if err := s.manager.CreateSketch(info); err != nil {
		return nil, err
	}
	return in, nil
}

func (s *serverStruct) CreateSketch(ctx context.Context, in *pb.Sketch) (*pb.Sketch, error) {
	if err := s.storage.Append(storage.CreateSketch, in); err != nil {
		return nil, err
	}
	return s.createSketch(ctx, in)
}

func (s *serverStruct) add(ctx context.Context, in *pb.AddRequest) (*pb.AddReply, error) {
	info := datamodel.NewEmptyInfo()
Beispiel #4
0
package manager

import (
	"fmt"
	"sort"
	"strconv"

	"datamodel"
	pb "datamodel/protobuf"

	"github.com/njpatel/loggo"
)

var logger = loggo.GetLogger("manager")

func isValidType(info *datamodel.Info) bool {
	if info.Type == nil {
		return false
	}
	return len(datamodel.GetTypeString(info.GetType())) != 0
}

// Manager is responsible for manipulating the sketches and syncing to disk
type Manager struct {
	infos    *infoManager
	sketches *sketchManager
	domains  *domainManager
}

// NewManager ...
func NewManager() *Manager {
Beispiel #5
0
	_ "net/http/pprof"

	"github.com/codegangsta/cli"
	"github.com/njpatel/loggo"
	"golang.org/x/crypto/ssh/terminal"

	"config"
	"manager"
	"server"
)

var (
	datadir string
	host    string
	port    int
	logger  = loggo.GetLogger("skizze")
	version string
)

func init() {
	setupLoggers()
}

func main() {
	app := cli.NewApp()
	app.Name = "Skizze"
	app.Usage = "A sketch data store for counting and sketching using probabilistic data-structures"
	app.Version = version

	app.Flags = []cli.Flag{
		cli.StringFlag{
Beispiel #6
0
package sketches

import (
	"fmt"
	"sync"

	"github.com/njpatel/loggo"

	"datamodel"
)

var logger = loggo.GetLogger("sketches")

// SketchProxy ...
type SketchProxy struct {
	*datamodel.Info
	sketch datamodel.Sketcher
	lock   sync.RWMutex
}

// Add ...
func (sp *SketchProxy) Add(values [][]byte) (bool, error) {
	sp.lock.Lock()
	defer sp.lock.Unlock()
	return sp.sketch.Add(values)
}

// Get ...
func (sp *SketchProxy) Get(data interface{}) (interface{}, error) {
	switch datamodel.GetTypeString(sp.GetType()) {
	case datamodel.HLLPP:
Beispiel #7
0
package utils

import (
	"io"
	"os"

	"github.com/njpatel/loggo"
)

var logger = loggo.GetLogger("util")

// PanicOnError is a helper function to panic on Error
func PanicOnError(err error) {
	if err != nil {
		logger.Errorf("%v", err)
		panic(err)
	}
}

// Exists returns if path exists
func Exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return true, err
}