// Opens a connection to the datasource. See Session().
func (self *Source) Open() error {
var err error
connURL := &url.URL{Scheme: "mongodb"}
if self.config.Port == 0 {
self.config.Port = 27017
}
if self.config.Host == "" {
self.config.Host = "127.0.0.1"
}
connURL.Host = fmt.Sprintf("%s:%d", self.config.Host, self.config.Port)
if self.config.User != "" {
connURL.User = url.UserPassword(self.config.User, self.config.Password)
}
if self.config.Database != "" {
connURL.Path = "/" + self.config.Database
}
self.session, err = mgo.DialWithTimeout(connURL.String(), 5*time.Second)
if err != nil {
return err
}
if self.config.Database != "" {
self.Use(self.config.Database)
}
return nil
}
// Connects to the previously specified datasource. See Session().
func (m *MongoDataSource) Open() error {
var err error
connURL := &url.URL{Scheme: "mongodb"}
if m.config.Port == 0 {
m.config.Port = 27017
}
if m.config.Host == "" {
m.config.Host = "127.0.0.1"
}
connURL.Host = fmt.Sprintf("%s:%d", m.config.Host, m.config.Port)
if m.config.User != "" {
connURL.User = url.UserPassword(m.config.User, m.config.Password)
}
m.session, err = mgo.DialWithTimeout(connURL.String(), 5*time.Second)
if err != nil {
return fmt.Errorf("Could not connect to %v.", m.config.Host)
}
if m.config.Database != "" {
m.Use(m.config.Database)
}
return nil
}
// Attempts to connect to a database using the stored settings.
func (self *Source) Open() error {
var err error
connURL := &url.URL{Scheme: `mongodb`}
if self.config.Port == 0 {
self.config.Port = 27017
}
if self.config.Host == "" {
self.config.Host = `127.0.0.1`
}
connURL.Host = fmt.Sprintf(`%s:%d`, self.config.Host, self.config.Port)
if self.config.User != "" {
connURL.User = url.UserPassword(self.config.User, self.config.Password)
}
if self.config.Database != "" {
connURL.Path = "/" + self.config.Database
}
if self.config.Database == "" {
return db.ErrMissingDatabaseName
}
if self.session, err = mgo.DialWithTimeout(connURL.String(), connTimeout); err != nil {
return err
}
self.Use(self.config.Database)
return nil
}
//private to this file
func _createSession() (*mgo.Session, error) {
if _cfg == nil {
_cfg = &mongoConfig{}
framework.Config.ReadInto("mongo", &_cfg)
}
var timeout = 10 * time.Second
if _cfg.ConnectionString != "" {
return mgo.DialWithTimeout(_cfg.ConnectionString, timeout)
} else {
dialInfo := mgo.DialInfo{}
dialInfo.Database = _cfg.Database
dialInfo.Username = _cfg.User
dialInfo.Password = _cfg.Password
dialInfo.Addrs = []string{fmt.Sprintf("%s:%s", _cfg.Host, _cfg.Port)}
dialInfo.Timeout = timeout
fmt.Println("Logging to", dialInfo.Addrs[0])
return mgo.DialWithInfo(&dialInfo)
}
}
func DBConnect() (d *db, err error) {
session, err := mgo.DialWithTimeout(conf.MONGODB, DbTimeout)
if err != nil {
return
}
d = &db{User: session.DB("ShockDB").C("Users"), Session: session}
return
}
func DBConnect() (d *db, err error) {
session, err := mgo.DialWithTimeout(conf.MONGODB, DbTimeout)
if err != nil {
return
}
d = &db{Jobs: session.DB("AWEDB").C("Jobs"), Session: session}
return
}
func DBConnect() (d *db, err error) {
session, err := mgo.DialWithTimeout(conf.MONGODB, DbTimeout)
if err != nil {
return
}
d = &db{Nodes: session.DB("ShockDB").C("Nodes"), PreAuth: session.DB("ShockDB").C("PreAuth"), Session: session}
return
}
func connectMongo(c *C) *mgo.Session {
session, err := mgo.DialWithTimeout(testMongoHost, 500*time.Millisecond)
if err != nil {
c.Fatalf(err.Error())
}
session.SetMode(mgo.Monotonic, true)
session.SetSyncTimeout(200 * time.Millisecond)
session.SetSocketTimeout(200 * time.Millisecond)
return session
}
func main() {
host := flag.String("host", "nair.local", "mongodb host name") // "nair.local"
dbname := flag.String("db", "enron", "database name") //"enron"
colname := flag.String("col", "messages", "collection name") //"messages"
keyword := flag.String("kw", "Sunday", "keyword to search") //"Sunday"
flag.Parse()
timeout := time.Duration(1) * time.Second
session, _ := mgo.DialWithTimeout(*host, timeout)
operator := NewOperator(session.DB(*dbname).C(*colname))
operator.FindKeyword(*keyword)
}
func getMgoSession() *mgo.Session {
if mgoSession == nil {
var err error
mgoSession, err = mgo.DialWithTimeout(mgoURL, 5*time.Second)
if err != nil {
panic(err)
}
// Set timeout to suitably small value by default.
mgoSession.SetSyncTimeout(5 * time.Second)
}
return mgoSession.Copy()
}
func (b *Backend) connectDB() error {
var err error
b.session, err = mgo.DialWithTimeout(fmt.Sprintf("%s:%d/%s", b.dbConfig.IP, b.dbConfig.Port, b.dbConfig.Name), time.Second*3)
if err != nil {
return err
}
b.session.SetMode(mgo.Monotonic, true)
return b.session.Ping()
}
func getSession() *mgo.Session {
if mgoSession == nil {
var err error
mgoSession, err = mgo.DialWithTimeout(databaseServer, 60*time.Second)
mgoSession.SetMode(mgo.Monotonic, true)
mgoSession.SetMode(mgo.Strong, true)
if err != nil {
panic(err) // no, not really
}
}
return mgoSession.Clone()
}
// ping won't work with old (1.2) mongo servers.
func (ins *instance) ping(timeout time.Duration) bool {
session, err := mgo.DialWithTimeout(ins.url(), timeout)
if err != nil {
return false
}
defer session.Close()
session.SetSyncTimeout(timeout)
if err = session.Ping(); err != nil {
return false
}
return true
}
func Initialize() (err error) {
c := connection{}
s, err := mgo.DialWithTimeout(conf.MONGODB_HOST, DbTimeout)
if err != nil {
e := errors.New(fmt.Sprintf("no reachable mongodb server(s) at %s", conf.MONGODB_HOST))
return e
}
c.Session = s
c.DB = c.Session.DB(conf.MONGODB_DATABASE)
if conf.MONGODB_USER != "" && conf.MONGODB_PASSWD != "" {
c.DB.Login(conf.MONGODB_USER, conf.MONGODB_PASSWD)
}
Connection = c
return
}
func open(addr, dbname string) (*Storage, error) {
sess, err := mgo.DialWithTimeout(addr, 1e9)
if err != nil {
return nil, err
}
copy := sess.Copy()
go func(t time.Duration) {
time.Sleep(t)
copy.Close()
}(maxIdleTime)
storage := &Storage{session: copy, dbname: dbname}
mut.Lock()
conn[addr] = &session{s: sess, used: time.Now()}
mut.Unlock()
return storage, nil
}
func Initialize() (err error) {
c := connection{}
s, err := mgo.DialWithTimeout(conf.Conf["mongodb-hosts"], DbTimeout)
if err != nil {
e := errors.New(fmt.Sprintf("no reachable mongodb server(s) at %s", conf.Conf["mongodb-hosts"]))
fmt.Fprintln(os.Stderr, "Error: "+e.Error())
return e
}
c.Session = s
c.DB = c.Session.DB(conf.Conf["mongodb-database"])
if conf.Conf["mongodb-user"] != "" && conf.Conf["mongodb-password"] != "" {
c.DB.Login(conf.Conf["mongodb-user"], conf.Conf["mongodb-password"])
}
Connection = c
return
}
func (this *Client) getConn(url string) (*mgo.Session, error) {
sess, ok := this.getFreeConn(url)
if ok {
return sess, nil
}
// create session on demand
sess, err := mgo.DialWithTimeout(url, this.connectTimeout)
if err != nil {
return nil, err
}
sess.SetSocketTimeout(this.ioTimeout)
return sess, nil
}
func (self *SessionFactory) GetSession() (*mgo.Session, *mgo.Database, error) {
// init the master session if necessary
if self.masterSession == nil {
var err error
self.masterSession, err = mgo.DialWithTimeout(self.Url,
self.DialTimeout)
if err != nil {
return nil, nil, err
}
}
// copy the master session
sessionCopy := self.masterSession.Copy()
return sessionCopy, sessionCopy.DB(self.DBName), nil
}
// GetDB returns a database connection.
func GetDB() (*mgo.Database, error) {
if db == nil && session == nil {
goserv.Logger.Info("Creating new session: ", conf.Config.MongoURI)
ses, err := mgo.DialWithTimeout(conf.Config.MongoURI, 3*time.Second)
if err != nil {
goserv.Logger.Error("Could not connect to MongoDB: ", err)
return nil, err
}
session := *ses
newdb := session.DB(conf.Config.MongoDatabase)
goserv.Logger.Infof("Connection made %s", newdb.Name)
if err != nil {
goserv.Logger.Fatal(err)
}
db = newdb
}
return db, nil
}
func (s *S) TestDialWithTimeout(c *C) {
if *fast {
c.Skip("-fast")
}
timeout := 2 * time.Second
started := time.Now()
// 40009 isn't used by the test servers.
session, err := mgo.DialWithTimeout("localhost:40009", timeout)
if session != nil {
session.Close()
}
c.Assert(err, ErrorMatches, "no reachable servers")
c.Assert(session, IsNil)
c.Assert(started.Before(time.Now().Add(-timeout)), Equals, true)
c.Assert(started.After(time.Now().Add(-timeout*2)), Equals, true)
}
func (MongoPlugin) Perform(req *api.StatusRequest, ch chan bool) {
url := req.Address + ":" + req.Port
if req.Verbose {
log.Printf("Connecting to MongoDB (%s) \n", url)
}
session, err := mgo.DialWithTimeout(url, api.Timeout)
if err != nil {
ch <- false
return
}
session.SetMode(mgo.Monotonic, true)
defer session.Close()
// Use 'test' database by default. If you want to specify another
// database, then pass it as an argument to the 'status' command
//
database := "test"
if len(req.Args) > 1 {
database = req.Args[1]
}
// Execute 'db.stats' command on selected database
//
retval := &bson.M{}
if err := session.DB(database).Run("dbStats", &retval); err != nil {
if req.Verbose {
log.Printf("Failed to query MongoDB: %s\n", err)
}
ch <- false
return
}
if (*retval)["ok"] != 1 {
ch <- true
} else {
if req.Verbose {
log.Printf("The database is not ready yet: %q", *retval)
}
ch <- false
}
}
func (s *S) TestSocketTimeoutOnDial(c *C) {
if *fast {
c.Skip("-fast")
}
timeout := 1 * time.Second
defer mgo.HackSyncSocketTimeout(timeout)()
s.Freeze("localhost:40001")
started := time.Now()
session, err := mgo.DialWithTimeout("localhost:40001", timeout)
c.Assert(err, ErrorMatches, "no reachable servers")
c.Assert(session, IsNil)
c.Assert(started.Before(time.Now().Add(-timeout)), Equals, true)
c.Assert(started.After(time.Now().Add(-20*time.Second)), Equals, true)
}
// Connect establishes a database connection.
func (m *Mongo) Connect(hosts string) {
log.Infof("connecting session to hosts:%s", hosts)
for {
session, err := mgo.DialWithTimeout(m.Hosts, 10*time.Second)
if err != nil {
log.Error(err)
time.Sleep(time.Second)
continue
}
session.SetMode(mgo.Strong, true)
//mgo.SetDebug(true)
//mgo.SetLogger(lg.New(os.Stderr, "", lg.LstdFlags))
session.SetSocketTimeout(120 * time.Second)
m.Session = session
go m.Pinger()
return
}
}
func main() {
flag.Parse()
config, err := ConfigOpen(*configPath)
if err != nil {
log.Fatalln(err)
}
mgoDatabase = config.Mongo.DB
// Set session timeout to fail early and avoid long response times.
mgoSession, err = mgo.DialWithTimeout(config.Mongo.URL, 5*time.Second)
if err != nil {
log.Fatalln("[MongoDB]", err)
}
defer mgoSession.Close()
addr := fmt.Sprintf("%s:%d", config.Http.Host, config.Http.Port)
log.Printf("serving at %s\n", addr)
err = http.ListenAndServe(addr, APIHandler())
if err != nil {
log.Fatal(err)
}
}
func (this *Client) dial(uri string) (*mgo.Session, error) {
if this.breakers[uri].Open() {
return nil, ErrCircuitOpen
}
this.throttleConns[uri] <- true
defer func() {
// release throttle
<-this.throttleConns[uri]
}()
sess, err := mgo.DialWithTimeout(uri, this.conf.ConnectTimeout)
if err != nil {
this.breakers[uri].Fail()
return nil, err
}
this.breakers[uri].Succeed()
sess.SetSocketTimeout(this.conf.IoTimeout)
sess.SetMode(mgo.Monotonic, true)
return sess, nil
}
func (s *Server) Run() {
//initialize mongoclient
timeout := time.Duration(s.cfg.Db.ConnTimeout) * time.Second
s.mclient, _ = mgo.DialWithTimeout(s.cfg.Db.Connstr, timeout)
//TODO add logger
}
func main() {
if *numCpu > 0 {
runtime.GOMAXPROCS(*numCpu)
} else {
runtime.GOMAXPROCS(runtime.NumCPU())
}
flag.Parse()
log.SetFlags(log.Lshortfile | log.LstdFlags)
// Enable http server for debug endpoint
go func() {
if *debugAddr != "" {
log.Println(http.ListenAndServe(*debugAddr, nil))
}
}()
interrupt := make(chan os.Signal)
signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
mgoSession, err := mgo.DialWithTimeout(*mongoServer+"?connect=direct", time.Duration(*mongoTimeout)*time.Minute)
if err != nil {
log.Fatal(err)
}
defer mgoSession.Close()
mongoc := make(chan *mongodb.Operation)
mongoErr := make(chan error)
exit := make(chan bool)
go func() {
mongoErr <- mongodb.Tail(mgoSession, *ns, *mongoInitial, lastEsSeen, mongoc, exit)
}()
esc := make(chan elasticsearch.Transaction)
esDone := make(chan bool)
go func() {
// Boot up our slurpers.
// The client will have the transport configured to allow the same amount of connections
// as go routines towards ES, each connection may be re-used between slurpers.
client := elasticsearch.NewClient(fmt.Sprintf("%s/_bulk", *esServer), *esConcurrency)
var slurpers sync.WaitGroup
slurpers.Add(*esConcurrency)
for n := 0; n < *esConcurrency; n++ {
go func() {
elasticsearch.Slurp(client, esc)
slurpers.Done()
}()
}
slurpers.Wait()
close(esDone)
}()
tailDone := make(chan bool)
go func() {
// Map mongo collections to es index
indexes := map[string]string{
strings.Split(*ns, ".")[0]: *esIndex,
}
for op := range mongoc {
// Wrap all mongo operations to comply with ES interface, then send them off to the slurper.
esOp := mongodb.NewEsOperation(indexes, nil, op)
select {
case esc <- esOp:
lastEsSeenC <- &op.Timestamp
// Abort delivering any pending EsOperations we might block for
case <-exit:
break
}
}
// If mongoc closed, tailer has stopped
close(tailDone)
}()
select {
// Get more operations from mongo tail
case <-tailDone:
log.Println("MongoDB tailer returned")
// ES client closed
case <-esDone:
log.Println("ES slurper returned")
// An interrupt signal was catched
case <-interrupt:
log.Println("Closing down...")
}
close(exit)
// MongoDB tailer shutdown
if err := <-mongoErr; err != nil {
log.Println(err)
} else {
log.Println("No errors occured in mongo tail")
}
// Elasticsearch indexer shutdown
log.Println("Waiting for EsOperation tail to stop")
<-tailDone
log.Println("Waiting for ES to return")
// We are the producer for this channel, close it down and wait for ES slurpers to return
close(esc)
<-esDone
log.Println("Bye!")
}
