import (
"bufio"
"fmt"
"os"
"strconv"
"strings"
"sync"
"time"
"utils"
"github.com/golang/protobuf/proto"
"github.com/njpatel/loggo"
)
var logger = loggo.GetLogger("storage")
// AOF ...
type AOF struct {
file *os.File
buffer *bufio.ReadWriter
lock sync.RWMutex
inChan chan *Entry
tickChan <-chan time.Time
}
// NewAOF ...
func NewAOF(path string) *AOF {
file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600)
utils.PanicOnError(err)
rdr := bufio.NewReader(file)
info_dir = "~/.skizze"
# This is where the data is stored either as json or .count (pure bytes)
data_dir = "~/.skizze/data"
# The host interface for the server
host = "localhost"
# The port number for the server
port = 3596
# Treshold for saving a sketch to disk
save_threshold_seconds = 1
`
var logger = loggo.GetLogger("config")
// Config stores all configuration parameters for Go
type Config struct {
InfoDir string `toml:"info_dir"`
DataDir string `toml:"data_dir"`
Host string `toml:"host"`
Port int `toml:"port"`
SaveThresholdSeconds uint `toml:"save_threshold_seconds"`
}
var config *Config
// InfoDir initialized from config file
var InfoDir string
package server
import (
"datamodel"
pb "datamodel/protobuf"
"storage"
"github.com/gogo/protobuf/proto"
"github.com/njpatel/loggo"
"golang.org/x/net/context"
)
var logger = loggo.GetLogger("server")
func (s *serverStruct) createSketch(ctx context.Context, in *pb.Sketch) (*pb.Sketch, error) {
info := &datamodel.Info{Sketch: in}
if err := s.manager.CreateSketch(info); err != nil {
return nil, err
}
return in, nil
}
func (s *serverStruct) CreateSketch(ctx context.Context, in *pb.Sketch) (*pb.Sketch, error) {
if err := s.storage.Append(storage.CreateSketch, in); err != nil {
return nil, err
}
return s.createSketch(ctx, in)
}
func (s *serverStruct) add(ctx context.Context, in *pb.AddRequest) (*pb.AddReply, error) {
info := datamodel.NewEmptyInfo()
package manager
import (
"fmt"
"sort"
"strconv"
"datamodel"
pb "datamodel/protobuf"
"github.com/njpatel/loggo"
)
var logger = loggo.GetLogger("manager")
func isValidType(info *datamodel.Info) bool {
if info.Type == nil {
return false
}
return len(datamodel.GetTypeString(info.GetType())) != 0
}
// Manager is responsible for manipulating the sketches and syncing to disk
type Manager struct {
infos *infoManager
sketches *sketchManager
domains *domainManager
}
// NewManager ...
func NewManager() *Manager {
_ "net/http/pprof"
"github.com/codegangsta/cli"
"github.com/njpatel/loggo"
"golang.org/x/crypto/ssh/terminal"
"config"
"manager"
"server"
)
var (
datadir string
host string
port int
logger = loggo.GetLogger("skizze")
version string
)
func init() {
setupLoggers()
}
func main() {
app := cli.NewApp()
app.Name = "Skizze"
app.Usage = "A sketch data store for counting and sketching using probabilistic data-structures"
app.Version = version
app.Flags = []cli.Flag{
cli.StringFlag{
package sketches
import (
"fmt"
"sync"
"github.com/njpatel/loggo"
"datamodel"
)
var logger = loggo.GetLogger("sketches")
// SketchProxy ...
type SketchProxy struct {
*datamodel.Info
sketch datamodel.Sketcher
lock sync.RWMutex
}
// Add ...
func (sp *SketchProxy) Add(values [][]byte) (bool, error) {
sp.lock.Lock()
defer sp.lock.Unlock()
return sp.sketch.Add(values)
}
// Get ...
func (sp *SketchProxy) Get(data interface{}) (interface{}, error) {
switch datamodel.GetTypeString(sp.GetType()) {
case datamodel.HLLPP:
package utils
import (
"io"
"os"
"github.com/njpatel/loggo"
)
var logger = loggo.GetLogger("util")
// PanicOnError is a helper function to panic on Error
func PanicOnError(err error) {
if err != nil {
logger.Errorf("%v", err)
panic(err)
}
}
// Exists returns if path exists
func Exists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}