// Get a new DBManager
func New(opts Opts) *DBManager {
// Configure sqlPool
poolOpts := sqlpool.Opts{
Max: int64(opts.MaxDBs),
IdleTimeout: int64(opts.IdleTimeout),
PreInit: createDirectory,
PostInit: initializeDatabase,
}
pool := sqlpool.NewPool(poolOpts)
// Start a new DBManager
manager := DBManager{
StartTime: time.Now(),
Logger: logger.New("[DBManager]"),
Pool: pool,
}
// Handle closing connections when app is killed
go func() {
<-opts.ClosingChannel
manager.Pool.ForceClose()
opts.ClosingChannel <- true
}()
return &manager
}
func init() {
log.Info("registering heartbeat: docker")
Add("docker", func(conf map[string]string) (Plugin, error) {
host, ok := conf["host"]
if !ok {
host = os.Getenv("DOCKER_HOST")
if host == "" {
log.Info("no host information found, fallback to default: %s")
host = DEFAULT_DOCKER_HOST
}
}
log.Info("connecting to docker (%s)", host)
// TODO support for tls
docker, err := dockerclient.NewDockerClient(host, nil)
if err != nil {
return nil, err
}
return &DockerMonitor{
docker: docker,
logger: logger.New("sentinel.plugins.heartbeats.docker"),
}, nil
})
}
func init() {
log.Info("registering adapter: shell")
Add("shell", func(conf map[string]string) (Plugin, error) {
return &Shell{
logger: logger.New("sentinel.plugins.adapters.shell"),
}, nil
})
}
func init() {
log.Info("registering actuator: debugger")
Add("debug", func(adapter_ adapters.Plugin, conf map[string]string) (Plugin, error) {
return &Debug{
Adapter: adapter_,
logger: logger.New("sentinel.plugins.actuators.debug"),
}, nil
})
}
func init() {
log.Info("registering heartbeat: clock")
Add("cron", func(conf map[string]string) (Plugin, error) {
interval, ok := conf["interval"]
if !ok {
interval = DEFAULT_INTERVAL
}
return &Clock{
Interval: interval,
logger: logger.New("sentinel.plugins.heartbeats.clock"),
}, nil
})
}
func init() {
log.Info("registering actuator: ping")
Add("ping", func(adapter_ adapters.Plugin, conf map[string]string) (Plugin, error) {
endpoint, ok := conf["endpoint"]
if !ok {
return nil, fmt.Errorf("no endpoint provided")
}
return &Ping{
Endpoint: endpoint,
Adapter: adapter_,
logger: logger.New("sentinel.plugins.actuators.ping"),
}, nil
})
}
func GetGeoLite2Reader() (*maxminddb.Reader, error) {
var log = logger.New("[GeoIP]")
data, err := geolite2db.Asset("GeoLite2-Country.mmdb")
if err != nil {
log.Error("Unable to open GeoLite2-Country asset file: [%v]", err)
return nil, err
}
db, err := maxminddb.FromBytes(data)
if err != nil {
log.Error("Unable to open GeoLite2-Country database: [%v]", err)
return nil, err
}
return db, nil
}
func init() {
log.Info("registering actuator: lua")
Add("lua", func(adapter_ adapters.Plugin, conf map[string]string) (Plugin, error) {
script, ok := conf["script"]
if !ok {
script = DEFAULT_LUA_SCRIPT_PATH
// TODO check it exists
}
vm := lua.NewState()
// NOTE defer vm.Close() ?
return &LuaScript{
Script: script,
VM: vm,
Adapter: adapter_,
logger: logger.New("sentinel.plugins.actuators.ping"),
}, nil
})
}
func init() {
log.Info("registering adapter: serf")
Add("serf", func(conf map[string]string) (Plugin, error) {
// FIXME bad formatted addr (e.g. 127.0.0.1) crashes sentinel
rpcAddr := get(conf, "addr", "127.0.0.1:7373").(string)
rpcAuthKey := get(conf, "auth-key", "").(string)
coalesce := get(conf, "coalesce", true).(bool)
RPC, err := command.RPCClient(rpcAddr, rpcAuthKey)
if err != nil {
return nil, err
}
return &Serf{
RPC: RPC,
logger: logger.New("sentinel.plugins.adapters.serf"),
coalesce: coalesce,
}, nil
})
}
// Return ISOCode for an IP
func GeoIpLookup(geolite2 *maxminddb.Reader, ipStr string) (string, error) {
var log = logger.New("[GeoIP]")
// Try to split port from ipStr
host, _, err := net.SplitHostPort(ipStr)
// Found a port in ipStr, update
if err == nil {
ipStr = host
}
ip := net.ParseIP(ipStr)
result := lookupResult{}
err = geolite2.Lookup(ip, &result)
if err != nil {
log.Error("Unable to lookup for IP %s: [%v]", ipStr, err)
return "", err
}
return strings.ToLower(result.Country.ISOCode), nil
}
func NewRouter(opts RouterOpts) (http.Handler, error) {
// Create the app router
r := mux.NewRouter()
var log = logger.New("[Router]")
geolite2 := opts.Geolite2Reader
// Initiate DB driver
driver, err := sqlite.NewShardedDriver(opts.DriverOpts)
if err != nil {
return nil, err
}
/////
// Query a DB over time
/////
r.Path("/{dbName}/time").
Methods("GET").
HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// Get params from URL
vars := mux.Vars(req)
dbName := vars["dbName"]
// Parse request query
if err := req.ParseForm(); err != nil {
renderError(w, err)
return
}
// Get timeRange if provided
startTime := req.Form.Get("start")
endTime := req.Form.Get("end")
intervalStr := req.Form.Get("interval")
// Convert startTime and endTime to a TimeRange
timeRange, err := newTimeRange(startTime, endTime)
if err != nil {
renderError(w, &webErrors.InvalidTimeFormat)
return
}
// Cast interval to an integer
// Defaults to 1 day
interval := 24 * 60 * 60
if len(intervalStr) > 0 {
interval, err = strconv.Atoi(intervalStr)
if err != nil {
renderError(w, &webErrors.InvalidInterval)
return
}
}
unique := false
if strings.Compare(req.Form.Get("unique"), "true") == 0 {
unique = true
}
// Construct Params object
params := database.Params{
DBName: dbName,
Interval: interval,
TimeRange: timeRange,
Unique: unique,
URL: req.URL,
}
analytics, err := driver.Series(params)
if err != nil {
renderError(w, normalizeDriverError(err))
return
}
// Return query result
render(w, analytics, nil)
})
/////
// Count for a DB
/////
r.Path("/{dbName}/count").
Methods("GET").
HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// Get params from URL
vars := mux.Vars(req)
dbName := vars["dbName"]
// Parse request query
if err := req.ParseForm(); err != nil {
renderError(w, err)
return
}
// Get timeRange if provided
startTime := req.Form.Get("start")
endTime := req.Form.Get("end")
// Convert startTime and endTime to a TimeRange
timeRange, err := newTimeRange(startTime, endTime)
if err != nil {
renderError(w, &webErrors.InvalidTimeFormat)
return
}
unique := false
if strings.Compare(req.Form.Get("unique"), "true") == 0 {
unique = true
}
// Construct Params object
params := database.Params{
DBName: dbName,
TimeRange: timeRange,
Unique: unique,
URL: req.URL,
}
analytics, err := driver.Count(params)
if err != nil {
renderError(w, normalizeDriverError(err))
return
}
// Return query result
render(w, analytics, nil)
})
/////
// Query a DB by property
/////
r.Path("/{dbName}/{property}").
Methods("GET").
HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// Map allowed requests w/ columns names in DB schema
allowedProperties := map[string]string{
"countries": "countryCode",
"platforms": "platform",
"domains": "refererDomain",
"events": "event",
}
// Get params from URL
vars := mux.Vars(req)
dbName := vars["dbName"]
property := vars["property"]
// Check that property is allowed to be queried
property, ok := allowedProperties[property]
if !ok {
renderError(w, &webErrors.InvalidProperty)
return
}
// Parse request query
if err := req.ParseForm(); err != nil {
renderError(w, err)
return
}
// Get timeRange if provided
startTime := req.Form.Get("start")
endTime := req.Form.Get("end")
timeRange, err := newTimeRange(startTime, endTime)
if err != nil {
renderError(w, &webErrors.InvalidTimeFormat)
return
}
unique := false
if strings.Compare(req.Form.Get("unique"), "true") == 0 {
unique = true
}
// Construct Params object
params := database.Params{
DBName: dbName,
Property: property,
TimeRange: timeRange,
Unique: unique,
URL: req.URL,
}
analytics, err := driver.GroupBy(params)
if err != nil {
renderError(w, normalizeDriverError(err))
return
}
// Return query result
render(w, analytics, nil)
})
/////
// Full query a DB
/////
r.Path("/{dbName}").
Methods("GET").
HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// Get dbName from URL
vars := mux.Vars(req)
dbName := vars["dbName"]
// Parse request query
if err := req.ParseForm(); err != nil {
renderError(w, err)
return
}
// Get timeRange if provided
startTime := req.Form.Get("start")
endTime := req.Form.Get("end")
timeRange, err := newTimeRange(startTime, endTime)
if err != nil {
renderError(w, &webErrors.InvalidTimeFormat)
return
}
// Construct Params object
params := database.Params{
DBName: dbName,
TimeRange: timeRange,
URL: req.URL,
}
analytics, err := driver.Query(params)
if err != nil {
renderError(w, normalizeDriverError(err))
return
}
render(w, analytics, nil)
})
/////
// Push a list of analytics to different DBs
/////
r.Path("/bulk").
Methods("POST").
HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// Parse JSON POST data
postList := PostAnalytics{}
jsonDecoder := json.NewDecoder(req.Body)
err := jsonDecoder.Decode(&postList)
// Invalid JSON
if err != nil {
log.Error("Invalid JSON format")
log.Error("%v", err)
renderError(w, &webErrors.InvalidJSON)
return
}
// Group analytics by website
analytics := make(map[string][]database.Analytic)
for _, postData := range postList.List {
// Skip analytic if website parameter missing
if postData.Website == "" {
log.Error("Skipping analytic: website parameter missing on POST data")
continue
}
// Parse data
analytic := parseAnalytic(postData, geolite2, log)
// Add to list
analytics[postData.Website] = append(analytics[postData.Website], analytic)
}
// Insert
err = driver.BulkInsert(analytics)
if err != nil {
renderError(w, normalizeDriverError(err))
return
}
log.Info("Successfully inserted analytics: %#v", analytics)
render(w, nil, nil)
})
/////
// Push analytics to a specific DB
/////
r.Path("/{dbName}").
Methods("POST").
HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// Get dbName from URL
vars := mux.Vars(req)
dbName := vars["dbName"]
// Parse JSON POST data
postData := PostData{}
jsonDecoder := json.NewDecoder(req.Body)
err := jsonDecoder.Decode(&postData)
// Invalid JSON
if err != nil {
log.Error("Invalid JSON format")
log.Error("%v", err)
renderError(w, &webErrors.InvalidJSON)
return
}
// Create Analytic to inject in DB
analytic := database.Analytic{
Time: time.Now(),
Event: postData.Event,
Path: postData.Path,
Ip: postData.Ip,
}
// Set time from POST data if passed
if len(postData.Time) > 0 {
analytic.Time, _ = time.Parse(time.RFC3339, postData.Time)
}
analytic.Time = analytic.Time.UTC()
// Set analytic referer domain
refererHeader := getReferrer(postData.Headers)
if referrerURL, err := url.ParseRequestURI(refererHeader); err == nil {
analytic.RefererDomain = referrerURL.Host
}
// Extract analytic platform from userAgent
userAgent := getUserAgent(postData.Headers)
analytic.Platform = utils.Platform(userAgent)
// Get countryCode from GeoIp
analytic.CountryCode, err = geoip.GeoIpLookup(geolite2, postData.Ip)
// Construct Params object
params := database.Params{
DBName: dbName,
}
err = driver.Insert(params, analytic)
if err != nil {
renderError(w, normalizeDriverError(err))
return
}
log.Info("Successfully inserted analytic: %#v", analytic)
render(w, nil, nil)
})
/////
// Push a list of analytics to a specific DB
/////
r.Path("/{dbName}/bulk").
Methods("POST").
HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// Get dbName from URL
vars := mux.Vars(req)
dbName := vars["dbName"]
// Parse JSON POST data
postList := PostAnalytics{}
jsonDecoder := json.NewDecoder(req.Body)
err := jsonDecoder.Decode(&postList)
// Invalid JSON
if err != nil {
log.Error("Invalid JSON format:")
log.Error("%v", err)
renderError(w, &webErrors.InvalidJSON)
return
}
// Create map of analytics for Bulk insert
analytics := make(map[string][]database.Analytic, 1)
for _, postData := range postList.List {
// Parse data
analytic := parseAnalytic(postData, geolite2, log)
// Add analytic to list
analytics[dbName] = append(analytics[dbName], analytic)
}
// Insert
err = driver.BulkInsert(analytics)
if err != nil {
renderError(w, normalizeDriverError(err))
return
}
log.Info("Successfully inserted analytics: %#v", analytics)
render(w, nil, nil)
})
/////
// Delete a DB
/////
r.Path("/{dbName}").
Methods("DELETE").
HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// Get dbName from URL
vars := mux.Vars(req)
dbName := vars["dbName"]
// Construct Params object
params := database.Params{
DBName: dbName,
}
err := driver.Delete(params)
if err != nil {
renderError(w, normalizeDriverError(err))
return
}
render(w, nil, nil)
})
return r, nil
}
package heartbeats
import (
"github.com/azer/logger"
"github.com/hackliff/serf/command/agent"
)
var log = logger.New("sentinel.plugins.heartbeats")
// Plugin is the interface to implement report triggers. Those run in
// their own gorountines, registering sensor plugins and waiting for signals
// to execute them.
type Plugin interface {
Schedule(string, agent.EventHandler)
Stop()
}
// NOTE map[string]interface{} for better type support after casting ?
type Creator func(map[string]string) (Plugin, error)
var Plugins = map[string]Creator{}
func Add(name string, creator Creator) {
Plugins[name] = creator
}
package crud
import (
stdsql "database/sql"
"github.com/azer/crud/sql"
"github.com/azer/logger"
)
var log = logger.New("crud")
type ExecFn func(string, ...interface{}) (stdsql.Result, error)
type QueryFn func(string, ...interface{}) (*stdsql.Rows, error)
type DB struct {
Client *stdsql.DB
Driver string
URL string
}
func (db *DB) Ping() error {
return db.Client.Ping()
}
func (db *DB) Exec(sql string, params ...interface{}) (stdsql.Result, error) {
log.Info(sql)
return db.Client.Exec(sql, params...)
}
func (db *DB) Query(sql string, params ...interface{}) (*stdsql.Rows, error) {
log.Info(sql)
return db.Client.Query(sql, params...)
package actuators
import (
"github.com/azer/logger"
"github.com/hackliff/serf/serf"
"github.com/hackliff/sentinel/plugins/adapters"
)
var log = logger.New("sentinel.plugins.actuators")
// TODO make it full compatible with telegraf plugins
// NOTE inspired by telegraf project plugin style
type Plugin interface {
// NOTE String() method ?
Description() string
SampleConfig() string
// NOTE a serf agent as argument for information and event triggers ?
Gather(serf.Member, serf.Event) error
}
// NOTE map[string]interface{} for better type support after casting ?
type Creator func(adapters.Plugin, map[string]string) (Plugin, error)
var Plugins = map[string]Creator{}
func Add(name string, creator Creator) {
Plugins[name] = creator
}
func main() {
// App meta-data
app := cli.NewApp()
app.Version = "2.0.3"
app.Name = "µAnalytics"
app.Author = "Johan Preynat"
app.Email = "*****@*****.**"
app.Usage = "Fast sharded analytics database"
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "user, u",
Value: "",
Usage: "Username",
EnvVar: "MA_USER",
},
cli.StringFlag{
Name: "password, w",
Value: "",
Usage: "Password",
EnvVar: "MA_PASSWORD",
},
cli.StringFlag{
Name: "port, p",
Value: "7070",
Usage: "Port to listen on",
EnvVar: "PORT",
},
cli.StringFlag{
Name: "root, r",
Value: "./dbs/",
Usage: "Database directory",
EnvVar: "MA_ROOT",
},
cli.IntFlag{
Name: "connections, c",
Value: 1000,
Usage: "Max number of alive DB connections",
EnvVar: "MA_POOL_SIZE",
},
cli.IntFlag{
Name: "idle-timeout, i",
Value: 60,
Usage: "Idle timeout for DB connections in seconds",
EnvVar: "MA_POOL_TIMEOUT",
},
cli.StringFlag{
Name: "cache-directory, d",
Value: ".diskache",
Usage: "Cache directory",
EnvVar: "MA_CACHE_DIR",
},
}
var log = logger.New("[Main]")
// Main app code
app.Action = func(ctx *cli.Context) {
cacheDir := path.Clean(ctx.String("cache-directory"))
cacheDir = path.Join(cacheDir, strings.Split(app.Version, ".")[0])
// Set driver options
driverOpts := database.DriverOpts{
Directory: path.Clean(ctx.String("root")),
CacheDirectory: cacheDir,
MaxDBs: ctx.Int("connections"),
IdleTimeout: ctx.Int("idle-timeout"),
ClosingChannel: make(chan bool, 1),
}
// Create Analytics directory if inexistant
dirExists, err := utils.PathExists(driverOpts.Directory)
if err != nil {
log.Error("Analytics directory path error [%v]", err)
os.Exit(1)
}
if !dirExists {
log.Info("Analytics directory doesn't exist: %s", driverOpts.Directory)
log.Info("Creating Analytics directory...")
os.MkdirAll(driverOpts.Directory, os.ModePerm)
} else {
log.Info("Working with existing Analytics directory: %s", driverOpts.Directory)
}
// Initiate Geolite2 DB Reader
geolite2, err := geoip.GetGeoLite2Reader()
if err != nil {
log.Info("Error [%v] obtaining a geolite2Reader", err)
log.Info("Running without Geolite2")
}
// Handle exit by softly closing DB connections
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
signal.Notify(c, syscall.SIGTERM)
go func() {
<-c
log.Info("Closing database connections...")
driverOpts.ClosingChannel <- true
<-driverOpts.ClosingChannel
log.Info("Connections closed successfully")
log.Info("Closing Geolite2 connection...")
geolite2.Close()
log.Info("Geolite2 is now closed")
log.Info("Goodbye!")
os.Exit(1)
}()
// Authentication
auth := &web.BasicAuth{
Name: ctx.String("user"),
Pass: ctx.String("password"),
}
// Setup server
opts := ServerOpts{
Port: normalizePort(ctx.String("port")),
Version: app.Version,
DriverOpts: driverOpts,
Geolite2Reader: geolite2,
Auth: auth,
}
log.Info("Launching server with: %#v", opts)
server, err := NewServer(opts)
if err != nil {
log.Error("ServerSetup error [%v]", err)
os.Exit(1)
}
// Run server
if err := gracehttp.Serve(server); err != nil {
log.Error("ListenAndServe error [%v]", err)
os.Exit(1)
}
}
// Parse CLI args and run
app.Run(os.Args)
}
// adapters package stores various plugins aimed at bot communication with an
// operator.
package adapters
import "github.com/azer/logger"
var log = logger.New("sentinel.plugins.adapters")
type Envelope struct {
Title string
Recipient string
}
type Plugin interface {
Send(Envelope, string) error
}
// NOTE map[string]interface{} for better type support after casting ?
type Creator func(map[string]string) (Plugin, error)
var Plugins = map[string]Creator{}
func Add(name string, creator Creator) {
Plugins[name] = creator
}
package coll
import (
"github.com/azer/logger"
"github.com/syndtr/goleveldb/leveldb"
)
var log = logger.New("level-collection")
var (
Client *leveldb.DB
)
func Open(path string) error {
log.Info("Opening %s", path)
conn, err := leveldb.OpenFile(path, nil)
Client = conn
if err != nil {
return err
}
return nil
}
func Set(key, value []byte) error {
log.Info("Set", logger.Attrs{
"key": key,
})
package main
import (
"errors"
"github.com/azer/logger"
"time"
)
var log = logger.New("e-mail")
func main() {
log.Info("Sending an e-mail", logger.Attrs{
"from": "*****@*****.**",
"to": "*****@*****.**",
})
err := errors.New("Too busy")
log.Error("Failed to send e-mail. Error: %s", err, logger.Attrs{
"from": "*****@*****.**",
"to": "*****@*****.**",
})
timer := log.Timer()
time.Sleep(time.Millisecond * 500)
timer.End("Created a new %s image", "bike", logger.Attrs{
"id": 123456,
"model": "bmx",
"frame": "purple",
"year": 2014,
})
package config
import (
"github.com/azer/logger"
"github.com/hackliff/serf/command/agent"
"github.com/olebedev/config"
)
var log = logger.New("sentinel.config")
const DEFAULT_NAME string = "sentinel"
type PluginConfig struct {
Plugin string
Opts map[string]string
}
type HeartbeatConfig struct {
// NOTE create Heartbeat plugin here ?
Plugin string
Opts map[string]string
Filters []agent.EventFilter
}
type Config struct {
Name string
Actuator *PluginConfig
Heartbeat *HeartbeatConfig
Adapter *PluginConfig
}
package main
import (
"errors"
"github.com/azer/logger"
"time"
)
var app = logger.New("app")
var images = logger.New("images")
var socket = logger.New("websocket")
var users = logger.New("users")
var db = logger.New("database")
func main() {
app.Info("Starting at %d", 9088)
db.Info("Connecting to mysql://azer@localhost:9900/foobar")
images.Info("Requesting an image at foo/bar.jpg")
timer := images.Timer()
time.Sleep(time.Millisecond * 250)
timer.End("Fetched foo/bar.jpg")
db.Error("Fatal connection error.")
users.Info("%s just logged from %s", "John", "Istanbul")
socket.Info("Connecting...")
err := errors.New("Unable to connect.")
socket.Error("%v", err)
package main
import (
"github.com/azer/logger"
"time"
)
var log = logger.New("app")
func main() {
log.Info("Starting at %d", 9088)
log.Info("Requesting an image at foo/bar.jpg")
timer := log.Timer()
time.Sleep(time.Millisecond * 250)
timer.End("Fetched foo/bar.jpg")
log.Error("Failed to start, shutting down...")
}