// Init ...
func (rthink *RethinkDBImpl) Init(conntype string, dbhost string, username string, password string, port int, dbname string, log log.Logger, hosts ...string) error {
var retErr error
if conntype == "cluster" {
session, err := r.Connect(r.ConnectOpts{
Addresses: hosts,
Database: dbname,
AuthKey: password,
})
retErr = err
if err != nil {
log.Crit("Got error when connect database, the error is '%v'", err)
return retErr
}
rthink.Session = session
} else {
session, err := r.Connect(r.ConnectOpts{
Address: dbhost,
Database: dbname,
AuthKey: password,
})
retErr = err
if err != nil {
log.Crit("Got error when connect database, the error is '%v'", err)
return retErr
}
rthink.Session = session
}
fmt.Println("Rethink Connected")
return nil
}
func Logger(inner http.Handler, name string, logger log.Logger) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
inner.ServeHTTP(w, r)
//fmt.Printf("%s\t%s\t%s\t%s", r.Method, r.RequestURI, name, time.Since(start))
logger.Info("Request", "method", r.Method, "uri", r.RequestURI, "name", name, "duration", time.Since(start))
})
}
func loggerFn(handler http.Handler, logger log.Logger, clientIP string, rw *hh.ResponseWriter, req *http.Request) {
start := time.Now()
handler.ServeHTTP(rw, req)
switch rw.Status() {
case 200, 307, 0: // 0 == 200 OK
default:
logger.Info("request completed", "method", req.Method, "path", req.URL.Path, "client_ip", clientIP, "status", rw.Status(), "duration", time.Since(start))
}
}
// matchQuery takes a MongoDB-like query object and returns wether or not
// the given document match the query.
// The document will be flattened before the checks, to handle the dot-notation.
func matchQuery(qLogger log.Logger, query, odoc map[string]interface{}) bool {
logger := qLogger.New("subquery", query, "doc", odoc)
ok := true
// Flatten the map to handle dot-notation handling
doc := flattenMap(odoc, "", ".")
for key, eval := range query {
switch key {
case "$or":
res := false
for _, iexpr := range eval.([]interface{}) {
expr := iexpr.(map[string]interface{})
res = res || matchQuery(qLogger, expr, doc)
}
if !res {
ok = false
}
case "$and":
res := true
for _, iexpr := range eval.([]interface{}) {
expr := iexpr.(map[string]interface{})
res = res && matchQuery(qLogger, expr, doc)
}
ok = res
default:
// TODO(ts) make this part cleaner
// (check orignal doc VS flattend doc)
val, check := doc[key]
oval, err := getPath(key, odoc)
if !check && err != nil {
logger.Debug("key not found")
return false
}
// `val` (the value of the queried doc) must be:
// - a standard type: nil, int, float64, string, bool
// - a []interface[}
// It can't ba `map[string]interface{}` since maps are flattened
switch vval := oval.(type) {
// If it's an array, the doc is returned if a lest one of the doc match the query
case []interface{}:
res := false
for _, li := range vval {
res = res || matchQueryValue(eval, li)
}
if res {
ok = ok && true
}
default:
ok = ok && matchQueryValue(eval, val)
}
}
// logger.Debug("subquery res", "ok", ok, "key", key, "eval", eval)
}
return ok
}
func NewProxy(c net.Conn, root log.Logger) *Proxy {
logger := root.New(
"remoteAddr", c.RemoteAddr(),
"localAddr", c.LocalAddr(),
)
ret := &Proxy{
remoteConn: c,
logger: logger,
errChan: make(chan struct{}),
}
return ret
}
func setFilterHandler(level string, logger log.Logger, handler log.Handler) error {
if level == "none" {
logger.SetHandler(log.DiscardHandler())
return nil
}
lvl, err := log.LvlFromString(level)
if err != nil {
return fmt.Errorf("Bad log level: %v", err)
}
logger.SetHandler(log.LvlFilterHandler(lvl, handler))
return nil
}
func New(logger log.Logger, kvStore client.KvStorer, blobStore client.BlobStorer) *DocStoreExt {
indexPath := filepath.Join(pathutil.VarDir(), "docstore.bleve")
index, err := openIndex(indexPath)
if err != nil {
// TODO(tsileo): returns an error instead
panic(err)
}
logger.Debug("Bleve index init", "index-path", indexPath)
return &DocStoreExt{
kvStore: kvStore,
blobStore: blobStore,
index: index,
logger: logger,
}
}
func LogForRequest(l log.Logger, r *http.Request) log.Logger {
ctx := log.Ctx{
"method": r.Method,
"host": r.Host,
"url": r.URL,
"requesturi": r.RequestURI,
"remote": r.RemoteAddr,
"proto": r.Proto,
// "header": r.Header,
}
if val, ok := r.Header["X-Forwarded-For"]; ok {
ctx["x_forward_for"] = val
}
if val, ok := r.Header["X-Authenticated-User"]; ok {
ctx["x_authenticate_user"] = val
}
return l.New(ctx)
}
func newMailer(conf ini.File, logger log.Logger) (Mailer, error) {
mailConf := conf.Section("mail")
if len(mailConf) == 0 {
return nil, nil
}
smtpAddr, ok := mailConf["smtp_server"]
if !ok {
return nil, errors.New("Missing mail -- smtp_server")
}
smtpPort, _ := mailConf["port"]
if smtpPort == "" {
smtpPort = "587"
}
fromAddr, ok := mailConf["from_address"]
if !ok {
return nil, errors.New("Missing mail -- from_address")
}
rootURL, ok := mailConf["root_url"]
if !ok {
return nil, errors.New("Missing mail -- root_url")
}
username, _ := mailConf["username"]
password, _ := mailConf["password"]
auth := smtp.PlainAuth("", username, password, smtpAddr)
logger = logger.New("module", "mail")
mailer := &SMTPMailer{
ServerAddr: smtpAddr + ":" + smtpPort,
Auth: auth,
From: fromAddr,
rootURL: rootURL,
logger: logger,
}
return mailer, nil
}
// Init ...
func (rdbms *RDBMSImpl) Init(dbtype string, dbhost string, username string, password string, port int, dbname string, log log.Logger) error {
var err error
connString := fmt.Sprintf("user=%s port=%d password=%s dbname=%s sslmode=disable", username, port, password, dbname)
rdbms.DB, err = gorm.Open(dbtype, connString)
if err != nil {
log.Crit("Got error when connect database, the error is '%v'", err)
return err
}
rdbms.DB.LogMode(true)
fmt.Println("Connected")
return nil
}
// InitDataStore ...
func (datastore *DataStore) InitDataStore(storeType string, connDetails map[string]interface{}, log log.Logger) error {
var err error
datastore.StoreType = storeType
if storeType == "rdbms" {
rdbms := RDBMSImpl{}
err = rdbms.Init(connDetails["dbtype"].(string), connDetails["host"].(string), connDetails["username"].(string), connDetails["password"].(string), int(connDetails["port"].(float64)), connDetails["database"].(string), log)
if err != nil {
log.Crit("Got error when connect database, the error is '%v'", err)
}
datastore.RDBMS = rdbms
}
return err
}
func (lua *LuaExt) exec(reqLogger log.Logger, app *LuaApp, appID, reqId, script string, w http.ResponseWriter, r *http.Request) int {
// FIXME(tsileo) a debug mode, with a defer/recover
// also parse the Lu error and show the bugging line!
start := time.Now()
httpClient := &http.Client{}
// Initialize internal Lua module written in Go
logger := loggerModule.New(reqLogger.New("ctx", "Lua"), start, reqId)
response := responseModule.New()
request := requestModule.New(r, reqId, lua.authFunc)
blobstore := blobstoreModule.New(lua.blobStore)
kvstore := kvstoreModule.New(lua.kvStore)
bewit := bewitModule.New(reqLogger.New("ctx", "Lua bewit module"), r)
template := templateModule.New()
// Initialize Lua state
L := luamod.NewState()
defer L.Close()
setCustomGlobals(L)
L.PreloadModule("request", request.Loader)
L.PreloadModule("response", response.Loader)
L.PreloadModule("logger", logger.Loader)
L.PreloadModule("blobstore", blobstore.Loader)
L.PreloadModule("kvstore", kvstore.Loader)
L.PreloadModule("bewit", bewit.Loader)
L.PreloadModule("template", template.Loader)
// TODO(tsileo) docstore module
// TODO(tsileo) cookies module
// TODO(tsileo) lru module
// TODO(tsileo) cache module => to cache response
// TODO(tsileo) load module from github directly?
// TODO(tsileo) ETag support
// 3rd party module
luajson.Preload(L)
L.PreloadModule("http", gluahttp.NewHttpModule(httpClient).Loader)
// Set some global variables
L.SetGlobal("reqID", luamod.LString(reqId))
L.SetGlobal("appID", luamod.LString(appID))
// Execute the code
if err := L.DoString(script); err != nil {
// FIXME better error, with debug mode?
panic(err)
}
// Apply the Response object to the actual response
response.WriteTo(w)
// TODO save the logRecords in the AppStats and find a way to serve them over Server-Sent Events
// keep them in memory with the ability to dump them in bulk as blob for later query
// logRecords := logger.Records()
for _, logRecord := range logger.Records() {
app.logs = append(app.logs, logRecord)
}
reqLogger.Info("Script executed", "response", response, "duration", time.Since(start))
return response.Status()
}
func newRouter(fallback ldap.Backend, logger log.Logger) *ldap.RouteMux {
defaults := &DefaultsBackend{
Log: logger.New(log.Ctx{"type": "backend", "backend": "defaults"}),
}
//Create routes bindings
routes := ldap.NewRouteMux(logger)
// buildins
routes.Search(defaults).
BaseDn("").
Scope(ldap.SearchRequestScopeBaseObject).
Filter("(objectclass=*)").
Label("Search - ROOT DSE")
routes.Search(defaults).
BaseDn("o=Pronoc, c=Net").
Scope(ldap.SearchRequestScopeBaseObject).
Label("Search - Company Root")
routes.Extended(defaults).
RequestName(ldap.NoticeOfStartTLS).Label("StartTLS")
//default routes
routes.NotFound(fallback)
routes.Abandon(fallback)
routes.Compare(fallback)
routes.Delete(fallback)
routes.Modify(fallback)
routes.Extended(fallback).
RequestName(ldap.NoticeOfWhoAmI).Label("Ext - WhoAmI")
routes.Extended(fallback).Label("Ext - Generic")
routes.Add(fallback).Label("Default Add")
routes.Bind(fallback).Label("Default Bind")
routes.Search(fallback).Label("Default Search")
return routes
}
func updateIssueReviewLabels(client *github.Client, log log15.Logger, review ReviewRequest) error {
oldLabels := []string{}
newLabels := []string{review.CalculateAppropriateStatus()}
foundReviewLabel, incorrectReviewLabel := false, false
for _, l := range review.issue.Labels {
oldLabels = append(oldLabels, *l.Name)
switch *l.Name {
case WIPLabel, CakedLabel, AwaitingCakeLabel:
foundReviewLabel = true
if *l.Name != newLabels[0] {
incorrectReviewLabel = true
}
continue
default:
newLabels = append(newLabels, *l.Name)
}
}
var labelsNeedUpdating bool
switch {
case !foundReviewLabel:
labelsNeedUpdating = true
log.Info("could not find review label", "old_labels", oldLabels, "new_labels", newLabels)
case incorrectReviewLabel:
labelsNeedUpdating = true
log.Info("review label is incorrect", "old_labels", oldLabels, "new_labels", newLabels)
default:
log.Info("review label does not need updating", "labels", oldLabels)
}
if labelsNeedUpdating {
_, _, err := client.Issues.ReplaceLabelsForIssue(*review.repo.Owner.Login, *review.repo.Name, review.Number(), newLabels)
if err != nil {
log.Error("unable to update issue review label", "err", err)
return err
}
}
return nil
}
func newPool(conf ini.File, logger log.Logger) (*pgx.ConnPool, error) {
logger = logger.New("module", "pgx")
if level, ok := conf.Get("log", "pgx_level"); ok {
setFilterHandler(level, logger, log.StdoutHandler)
}
connConfig := pgx.ConnConfig{Logger: logger}
connConfig.Host, _ = conf.Get("database", "host")
if connConfig.Host == "" {
return nil, errors.New("Config must contain database.host but it does not")
}
if p, ok := conf.Get("database", "port"); ok {
n, err := strconv.ParseUint(p, 10, 16)
connConfig.Port = uint16(n)
if err != nil {
return nil, err
}
}
var ok bool
if connConfig.Database, ok = conf.Get("database", "database"); !ok {
return nil, errors.New("Config must contain database.database but it does not")
}
connConfig.User, _ = conf.Get("database", "user")
connConfig.Password, _ = conf.Get("database", "password")
poolConfig := pgx.ConnPoolConfig{
ConnConfig: connConfig,
MaxConnections: 10,
}
return pgx.NewConnPool(poolConfig)
}
func (fsm *LogFSM) ParentLogger(parent log15.Logger) {
fsm.logger = parent.New("module", "fsm")
}
func defaultLoggerFn(handler http.Handler, logger log.Logger, clientIP string, rw *ResponseWriter, req *http.Request) {
start := time.Now()
logger.Info("request started", "method", req.Method, "path", req.URL.Path, "client_ip", clientIP)
handler.ServeHTTP(rw, req)
logger.Info("request completed", "status", rw.Status(), "duration", time.Since(start))
}
//NewServer return a LDAP Server
func NewServer(logger log.Logger) *Server {
return &Server{
chDone: make(chan bool),
log: logger.New(log.Ctx{"type": "ldap"}),
}
}
// NewRaft returns a new raft struct based on the provided configuration
func NewRaft(config *Config, logger log.Logger) (*Raft, error) {
rand.Seed(time.Now().Unix())
if config.StateMachine == nil {
return nil, fmt.Errorf("No state machine configured.")
}
inbox := make(chan *channelCmd, 4096)
// set advertise address, if set
advertise := config.Bind
if len(config.Advertise) > 0 {
advertise = config.Advertise
}
logger = logger.New("module", "raft", "server", advertise)
// cleanup old directories
if len(config.Peers) != 0 {
err := os.RemoveAll(config.BaseDir)
if err != nil {
return nil, err
}
}
// create basic directories
err := os.MkdirAll(config.BaseDir, os.FileMode(0700))
err = createDirs(config.BaseDir)
if err != nil {
logger.Error("Can not create directories", "error", err.Error())
return nil, err
}
// create stable
stable, err := newBoltDBStable(fmt.Sprintf("%s/logs", config.BaseDir), logger)
raftRPCImpl := newRaftRPC(logger, inbox)
// create raft rpc server
rpcServer, err := multiserver.NewRPCServer(config.Bind, codecFunc, raftRPCImpl, config.TLSConfig, logger)
if err != nil {
logger.Error("Can not create rpc server", "error", err.Error())
return nil, err
}
client := NewClient(config.ClientTLSConfig)
// create peer list without itself
// at the moment the config is static
// and needs restart for changes
peers := make(map[string]*peer)
for _, server := range config.Peers {
// check if this instance is in the list
if server != advertise {
p := &peer{
addr: server,
timeoutCh: make(chan bool),
syncing: false,
}
peers[server] = p
}
}
chMap := &channelMap{
responseWaiter: make(map[uint64]chan interface{}),
responseResult: make(map[uint64]interface{}),
}
vRaft := &Raft{
peers: peers,
state: Shutdown,
client: client,
appendResCh: make(chan *ResponseAppendEntry, 4096),
commitCh: make(chan uint64, 4096),
inbox: inbox,
config: config,
channelMap: chMap,
stable: stable,
logger: logger,
advertise: advertise,
rpcServer: rpcServer,
bind: config.Bind,
fsm: config.StateMachine,
voteRespCh: make(chan *ResponseVote, len(peers)),
commandCh: make(chan *channelCmd, 4096),
leaderCh: make(chan string, 5),
raftRPC: raftRPCImpl,
}
vRaft.workingSnapshot = &fsmSnapshot{
LastIndex: 0,
LastTerm: 0,
}
return vRaft, nil
}
// NewRouteMux returns a new *RouteMux
// RouteMux implements ldapserver.Handler
func NewRouteMux(logger log.Logger) *RouteMux {
return &RouteMux{
Log: logger.New(log.Ctx{"type": "router"}),
}
}
/*
InitializeLogger creates the logger for the server module
*/
func InitializeLogger(parentLogger log.Logger) {
logger = parentLogger.New("module", "server")
}