// initSubscribers performs one-time initialization of the Postgres listener and // goroutine for event delivery, termination and subscription management. func (n *Notifier) initSubscribers(dbUrl string) error { n.l = pq.NewListener(dbUrl, 20*time.Millisecond, time.Hour, nil) go func() { for { select { case <-n.terminated: return case <-n.exit: n.l.UnlistenAll() n.l.Close() n.db.Close() for _, localL := range n.subscribers { close(localL) } close(n.terminated) case cmd := <-n.ch: cmd.fun() close(cmd.ack) case pgn := <-n.l.Notify: if pgn != nil { localN, err := n.makeNotification(pgn) if err != nil { log.Printf("Error parsing inbound notification %v: %v", pgn, err) } else { for _, sub := range n.subscribers { sub <- localN } } } } } }() return nil }
func waitForNotification(dbcluter DBCluster, parition string) { conninfo := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=require", "read", dbcluter.Password, dbcluter.Parition[parition].Write.IP, "regentmarkets") listener := pq.NewListener(conninfo, 5*time.Second, 10*time.Second, nil) db, _ := sql.Open("postgres", conninfo) err := db.Ping() if err != nil { log.Fatal(err) } err = listener.Listen("transactions_watcher") checkErr(err) fmt.Println("Listing to", parition) var redisdb redis.Conn var notification *pq.Notification for { select { case notification = <-listener.Notify: if notification != nil { redisconn(&redisdb) publish(redisdb, regexp.MustCompile(",").Split(notification.Extra, -1)) } case <-time.After(60 * time.Second): fmt.Println("no notifications for 60 seconds...") } } }
func (cmd *activityCmd) run(ctx scope.Context, args []string) error { // Get cluster in order to load config. _, err := getCluster(ctx) if err != nil { return fmt.Errorf("cluster error: %s", err) } listener := pq.NewListener(backend.Config.DB.DSN, 200*time.Millisecond, 5*time.Second, nil) if err := listener.Listen("broadcast"); err != nil { return fmt.Errorf("pq listen error: %s", err) } defer func() { ctx.Cancel() ctx.WaitGroup().Wait() }() // Start metrics server. ctx.WaitGroup().Add(1) go activity.Serve(ctx, cmd.addr) // Start scanner. ctx.WaitGroup().Add(1) activity.ScanLoop(ctx, listener) return nil }
func main() { conninfo := "dbname=gotraining user=root password=root port=7705 sslmode=disable" _, err := sql.Open("postgres", conninfo) if err != nil { panic(err) } reportProblem := func(ev pq.ListenerEventType, err error) { if err != nil { fmt.Println(err.Error()) } } listener := pq.NewListener(conninfo, 10*time.Second, time.Minute, reportProblem) err = listener.Listen("events") if err != nil { panic(err) } fmt.Println("Start monitoring PostgreSQL...") for { waitForNotification(listener) } }
func Init() { var err error db, err = sql.Open("postgres", *dataSource) if err != nil { debug.Fatal(err) } create := func(name, content string) { if err != nil { debug.Fatal(err) } err = createTable(name, content) } // primary tables create("user_", createUserSQL) create("project", createProjectSQL) create("task", createTaskSQL) create("milestone", createMilestoneSQL) create("member", createMemberSQL) create("worker", createWorkerSQL) create("friend", createFriendSQL) create("chat", createChatSQL) // secondary tables create("featured_project", createFeaturedProjectSQL) create("user_tag", createUserTagSQL) create("task_tag", createTaskTagSQL) create("tag", createTagSQL) // setup listener if *listenerEnabled { listener = pq.NewListener(*dataSource, 1*time.Second, time.Minute, func(ev pq.ListenerEventType, err error) { if err != nil { log.Fatal(err) } }) if err := listener.Listen("chat"); err != nil { log.Fatal(err) } go func() { for { select { case notification := <-listener.Notify: if Notify != nil { Notify(notification.Channel, notification.Extra) } } } }() } else { debug.Warn("PostgreSQL listener is disabled") } }
func listenMessages() { revel.TRACE.Println("Setting up db listening") var Spec, _ = revel.Config.String("db.spec") var listener = pq.NewListener(Spec, 10*time.Second, time.Minute, listenerEventsHandler) var err = listener.Listen("new_message") if err != nil { panic(err) } waitForNotification(listener) }
func (repo *postgresRepository) ensureListener() { repo.m.Lock() defer repo.m.Unlock() if repo.listener == nil { repo.listener = pq.NewListener(repo.connectionString, time.Second, time.Minute, nil) kit.SafeGo(repo.listenDaemon) } }
func run(config Config) { listener := pq.NewListener(config.PostgresURL, 10*time.Second, time.Minute, errorReporter) err := listener.Listen("urlwork") if err != nil { log.Fatal(err) } rabbitchannel := make(chan string, 100) go func() { cfg := new(tls.Config) cfg.InsecureSkipVerify = true conn, err := amqp.DialTLS(config.RabbitMQURL, cfg) if err != nil { log.Fatal(err) } defer conn.Close() ch, err := conn.Channel() if err != nil { log.Fatal(err) } defer ch.Close() for { payload := <-rabbitchannel log.Println(payload) err := ch.Publish("urlwork", "todo", false, false, amqp.Publishing{ ContentType: "text/plain", Body: []byte(payload), }) if err != nil { log.Fatal(err) } } }() for { select { case notification := <-listener.Notify: rabbitchannel <- notification.Extra case <-time.After(90 * time.Second): go func() { err := listener.Ping() if err != nil { log.Fatal(err) } }() } } }
// newPgListener creates and returns the pglistener from the pq package. func newPgListener(pgconninfo string) (*pq.Listener, error) { // create a callback function to monitor connection state changes pgEventCallback := func(ev pq.ListenerEventType, err error) { if err != nil { fmt.Println("pgbroadcast: ", err.Error()) } } // create the listener l := pq.NewListener(pgconninfo, 10*time.Second, time.Minute, pgEventCallback) return l, nil }
func run(config Config) { purl := fmt.Sprintf("user=%s password=%s host=%s port=%s dbname=%s sslmode=%s", config.DbUser, config.DbPassword, config.DbHost, config.DbPort, config.DbName, config.SslMode) listener := pq.NewListener(purl, 10*time.Second, time.Minute, errorReporter) err := listener.Listen("usertrigger") if err != nil { log.Fatal(err) } rabbitchannel := make(chan string, 100) //Code for STOMP go func() { rabbitHost := fmt.Sprintf("%s:%s", config.RabbitHost, config.RabbitPort) conn, err := stomp.Dial("tcp", rabbitHost, stomp.ConnOpt.Login(config.RabbitUser, config.RabbitPassword), stomp.ConnOpt.AcceptVersion(stomp.V11), stomp.ConnOpt.AcceptVersion(stomp.V12), stomp.ConnOpt.Host(config.RabbitVHost), stomp.ConnOpt.Header("nonce", "B256B26D320A")) if err != nil { log.Fatal(err) } defer conn.Disconnect() for { payload := <-rabbitchannel log.Println(payload) err = conn.Send(config.RabbitQueue, "text/plain", []byte(payload)) if err != nil { log.Fatal(err) } } }() for { select { case notification := <-listener.Notify: rabbitchannel <- notification.Extra case <-time.After(90 * time.Second): go func() { err := listener.Ping() if err != nil { log.Fatal(err) } }() } } }
func (jql *jobQueueListener) background(wg *sync.WaitGroup) { ctx := jql.Backend.ctx.Fork() logger := jql.Backend.logger defer ctx.WaitGroup().Done() listener := pq.NewListener(jql.Backend.dsn, 200*time.Millisecond, 5*time.Second, nil) if err := listener.Listen("job_item"); err != nil { // TODO: manage this more nicely panic("job listen: " + err.Error()) } logger.Printf("job listener started") // Signal to constructor that we're ready to handle operations. wg.Done() keepalive := time.NewTicker(30 * time.Second) defer keepalive.Stop() for { select { case <-ctx.Done(): return case <-keepalive.C: // Ping to make sure the database connection is still live. if err := listener.Ping(); err != nil { logger.Printf("job listener ping: %s\n", err) jql.Backend.ctx.Terminate(fmt.Errorf("job listener ping: %s", err)) return } case notice := <-listener.Notify: if notice == nil { logger.Printf("job listener: received nil notification") // A nil notice indicates a loss of connection. // For now it's easier to just shut down and force job // processor to restart. jql.Backend.ctx.Terminate(ErrPsqlConnectionLost) return } jql.m.Lock() if c, ok := jql.cs[notice.Extra]; ok { c.Signal() } jql.m.Unlock() } } }
func (cmd *ATCCommand) constructDB(logger lager.Logger) (*db.SQLDB, db.PipelineDBFactory, error) { dbConn, err := migrations.LockDBAndMigrate(logger.Session("db.migrations"), "postgres", cmd.PostgresDataSource) if err != nil { return nil, nil, fmt.Errorf("failed to migrate database: %s", err) } listener := pq.NewListener(cmd.PostgresDataSource, time.Second, time.Minute, nil) bus := db.NewNotificationsBus(listener, dbConn) explainDBConn := db.Explain(logger, dbConn, clock.NewClock(), 500*time.Millisecond) sqlDB := db.NewSQL(logger.Session("db"), explainDBConn, bus) pipelineDBFactory := db.NewPipelineDBFactory(logger.Session("db"), explainDBConn, bus, sqlDB) return sqlDB, pipelineDBFactory, err }
func waitForNotification(dbcluter DBCluster, parition string) { conninfo := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=require", "read", dbcluter.Password, dbcluter.Parition[parition].Write.IP, "regentmarkets") listener := pq.NewListener(conninfo, 5*time.Second, 10*time.Second, nil) _ = listener.Listen("transactions_watcher") redisdb, _ := redis.DialURL(os.Getenv("REDIS_URL")) var notification *pq.Notification for { select { case notification = <-listener.Notify: if notification != nil { publish(redisdb, regexp.MustCompile(",").Split(notification.Extra, -1)) } } } }
// Listen executes `LISTEN channel`. Uses f to handle received notifications on chanel. // On error logs error messages (if a logs exists) func (db *Database) Listen(channel string, f func(payload ...string)) error { // Create a new listener only if Listen is called for the first time if db.listener == nil { db.listenerCallbacks = make(map[string]func(...string)) reportProblem := func(ev pq.ListenerEventType, err error) { if err != nil && db.logger != nil { db.printLog(err.Error()) } } db.listener = pq.NewListener(db.connectionString, 10*time.Second, time.Minute, reportProblem) if db.listener == nil { return errors.New("Unable to create a new listener") } // detach event handler go func() { for { select { case notification := <-db.listener.Notify: go db.listenerCallbacks[notification.Channel](notification.Extra) case <-time.After(90 * time.Second): go func() { if db.listener.Ping() != nil { db.printLog(fmt.Sprintf("Error checking server connection for channel %s\n", channel)) return } }() } } }() } if _, alreadyIn := db.listenerCallbacks[channel]; alreadyIn { return errors.New("Already subscribed to channel " + channel) } db.listenerCallbacks[channel] = f if err := db.listener.Listen(channel); err != nil { return err } return nil }
// pgstr - строка для коннекта к постгресу func NewEventService(pgstr string) (event *EventService) { event = &EventService{Channels: map[string]Channel{}} reportProblem := func(ev pq.ListenerEventType, err error) { if err != nil { event.handleErrorAll(err) } } event.Listener = pq.NewListener(pgstr, 10*time.Second, time.Minute, reportProblem) go func() { var msg Message for { select { case n := <-event.Listener.Notify: if ch, ok := event.Channels[n.Channel]; ok { ch.MessageCount++ err := json.Unmarshal([]byte(n.Extra), &msg) // не распарсили сообщение if err != nil { ch.handleError(err) continue } // распарсили и нашли клиента if client, ok := ch.Clients[msg.Target]; ok { go client.EventHandler(msg.Body) } } continue case <-time.After(60 * time.Second): go func() { err := event.Listener.Ping() if err != nil { event.handleErrorAll(err) } }() continue } } }() return }
func waitForNotification(clientdb ClientDB, company string) { conninfo := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=require", "postgres", clientdb.Password, clientdb.Company[company].Write.IP, "test") listener := pq.NewListener(conninfo, 10*time.Second, time.Minute, nil) err := listener.Listen("getwork") checkErr(err) redisdb, err := redis.DialURL(os.Getenv("REDIS_URL")) checkErr(err) var notification *pq.Notification for { select { case notification = <-listener.Notify: if notification != nil { publish(redisdb, regexp.MustCompile(",").Split(notification.Extra, -1)) } case <-time.After(60 * time.Second): fmt.Println("no notifications for 60 seconds...") } } }
func NewDispatcher(conf DBConf) (d *Dispatcher, err error) { c, ok := conf[env()] if !ok { err = errors.New("Configuration for \"" + env() + "\" environment not found.") return } d = &Dispatcher{ done: make(chan bool), waitGroup: &sync.WaitGroup{}, listener: pq.NewListener(c.Conn, 10*time.Second, time.Minute, nil), clients: NewClientsPool(), tasks: make(map[int64]autogram.BackgroundTask), } d.db, err = sql.Open(c.Driver, c.Conn) if err != nil { return } return }
func main() { config_dsn := os.Getenv("CP_DSN") if config_dsn == "" { config_dsn = "postgres://*****:*****@10.10.42.23:5432/cachepusher?sslmode=disable" } config_redis := os.Getenv("CP_REDIS") if config_redis == "" { config_redis = "10.10.42.23:6379" } config_cacheprefix := os.Getenv("CP_PREFIX") if config_cacheprefix == "" { config_cacheprefix = "customer" } db, err := sqlx.Connect("postgres", config_dsn) if err != nil { log.Print("[Postgresql] ", err) return } r := redis.NewClient(&redis.Options{ Addr: config_redis, }) _, err = r.Ping().Result() if err != nil { log.Print("[Redis] ", err) return } log.Print("Clearing cache") keys, err := r.Keys(fmt.Sprintf("%s:*", config_cacheprefix)).Result() if err != nil { log.Print(err) } r.Pipelined(func(r *redis.Pipeline) error { for _, key := range keys { err = r.Del(key).Err() if err != nil { log.Print(err) } } return nil }) listener := pq.NewListener(config_dsn, 10*time.Second, time.Minute, func(ev pq.ListenerEventType, err error) { if err != nil { log.Print(err) } }) err = listener.Listen("customer_updated") if err != nil { log.Print(err) return } err = listener.Listen("customer_deleted") if err != nil { log.Print(err) return } /** Wait until we have set up the listener to get notifications before we trigger the warmup **/ log.Print("Triggering cache warmup") _, err = db.Exec("select customer_warmup()") if err != nil { log.Print(err) } log.Printf("Listening for updates") for { select { case n := <-listener.Notify: cachekey := fmt.Sprintf("%s:%s", config_cacheprefix, n.Extra) switch n.Channel { case "customer_deleted": log.Printf("DEL %s", cachekey) err = r.Del(cachekey).Err() if err != nil { log.Print(err) } case "customer_updated": var customerJson string err = db.Get(&customerJson, "select customer_get($1)", n.Extra) if err != nil { log.Print(err) } log.Printf("SET %s %s", cachekey, customerJson) err = r.Set(cachekey, customerJson, 0).Err() if err != nil { log.Print(err) } } // Make sure our connection stays up case <-time.After(90 * time.Second): log.Print("LISTEN PING") go func() { err = listener.Ping() if err != nil { log.Print(err) } }() } } }
func main() { flag.Parse() if !*dev && (*httpUsername == "" || (*httpHashedPassword == "" && *httpPassword == "")) { fatal(errors.New("must specify -httpUsername and -httpPassword or -httpHashedPassword or turn on dev mode")) } if _, err := os.Stat(*templatesDir); err != nil { fatal(errors.New("directory specified via -templates does not exist")) } if _, err := os.Stat(*publicDir); err != nil { fatal(errors.New("directory specified via -public does not exist")) } logger := lager.NewLogger("atc") logLevel := lager.INFO if *dev { logLevel = lager.DEBUG } sink := lager.NewReconfigurableSink(lager.NewWriterSink(os.Stdout, lager.DEBUG), logLevel) logger.RegisterSink(sink) var err error var dbConn Db.Conn for { dbConn, err = migration.Open(*sqlDriver, *sqlDataSource, migrations.Migrations) if err != nil { if strings.Contains(err.Error(), " dial ") { logger.Error("failed-to-open-db", err) time.Sleep(5 * time.Second) continue } fatal(err) } break } dbConn = Db.Explain(logger, dbConn, 500*time.Millisecond) listener := pq.NewListener(*sqlDataSource, time.Second, time.Minute, nil) bus := Db.NewNotificationsBus(listener) db := Db.NewSQL(logger.Session("db"), dbConn, bus) pipelineDBFactory := Db.NewPipelineDBFactory(logger.Session("db"), dbConn, bus, db) var configDB Db.ConfigDB configDB = Db.PlanConvertingConfigDB{db} var resourceTypesNG []atc.WorkerResourceType err = json.Unmarshal([]byte(*resourceTypes), &resourceTypesNG) if err != nil { logger.Fatal("invalid-resource-types", err) } var workerClient worker.Client if *gardenAddr != "" { workerClient = worker.NewGardenWorker( gclient.New(gconn.NewWithLogger( *gardenNetwork, *gardenAddr, logger.Session("garden-connection"), )), clock.NewClock(), -1, resourceTypesNG, "linux", []string{}, ) } else { workerClient = worker.NewPool(worker.NewDBWorkerProvider(db, logger)) } resourceTracker := resource.NewTracker(workerClient) gardenFactory := exec.NewGardenFactory(workerClient, resourceTracker, func() string { guid, err := uuid.NewV4() if err != nil { panic("not enough entropy to generate guid: " + err.Error()) } return guid.String() }) execEngine := engine.NewExecEngine(gardenFactory, engine.NewBuildDelegateFactory(db), db) engine := engine.NewDBEngine(engine.Engines{execEngine}, db, db) var webValidator auth.Validator if *httpUsername != "" && *httpHashedPassword != "" { webValidator = auth.BasicAuthHashedValidator{ Username: *httpUsername, HashedPassword: *httpHashedPassword, } } else if *httpUsername != "" && *httpPassword != "" { webValidator = auth.BasicAuthValidator{ Username: *httpUsername, Password: *httpPassword, } } else { webValidator = auth.NoopValidator{} } callbacksURL, err := url.Parse(*callbacksURLString) if err != nil { fatal(err) } drain := make(chan struct{}) apiHandler, err := api.NewHandler( logger, // logger lager.Logger, webValidator, // validator auth.Validator, pipelineDBFactory, // pipelineDBFactory db.PipelineDBFactory, configDB, // configDB db.ConfigDB, db, // buildsDB buildserver.BuildsDB, db, // workerDB workerserver.WorkerDB, db, // pipeDB pipes.PipeDB, db, // pipelinesDB db.PipelinesDB, config.ValidateConfig, // configValidator configserver.ConfigValidator, callbacksURL.String(), // peerURL string, buildserver.NewEventHandler, // eventHandlerFactory buildserver.EventHandlerFactory, drain, // drain <-chan struct{}, engine, // engine engine.Engine, workerClient, // workerClient worker.Client, sink, // sink *lager.ReconfigurableSink, *cliDownloadsDir, // cliDownloadsDir string, ) if err != nil { fatal(err) } radarSchedulerFactory := pipelines.NewRadarSchedulerFactory( resourceTracker, *checkInterval, db, engine, db, ) webHandler, err := web.NewHandler( logger, webValidator, radarSchedulerFactory, db, pipelineDBFactory, configDB, *templatesDir, *publicDir, engine, ) if err != nil { fatal(err) } webMux := http.NewServeMux() webMux.Handle("/api/v1/", apiHandler) webMux.Handle("/", webHandler) var httpHandler http.Handler httpHandler = webMux if !*publiclyViewable { httpHandler = auth.Handler{ Handler: httpHandler, Validator: webValidator, } } // copy Authorization header as ATC-Authorization cookie for websocket auth httpHandler = auth.CookieSetHandler{ Handler: httpHandler, } httpHandler = httpmetrics.Wrap(httpHandler) webListenAddr := fmt.Sprintf("%s:%d", *webListenAddress, *webListenPort) debugListenAddr := fmt.Sprintf("%s:%d", *debugListenAddress, *debugListenPort) syncer := pipelines.NewSyncer( logger.Session("syncer"), db, pipelineDBFactory, func(pipelineDB Db.PipelineDB) ifrit.Runner { return grouper.NewParallel(os.Interrupt, grouper.Members{ { pipelineDB.ScopedName("radar"), rdr.NewRunner( logger.Session(pipelineDB.ScopedName("radar")), *noop, db, radarSchedulerFactory.BuildRadar(pipelineDB), pipelineDB, 1*time.Minute, ), }, { pipelineDB.ScopedName("scheduler"), &sched.Runner{ Logger: logger.Session(pipelineDB.ScopedName("scheduler")), Locker: db, DB: pipelineDB, Scheduler: radarSchedulerFactory.BuildScheduler(pipelineDB), Noop: *noop, Interval: 10 * time.Second, }, }, }) }, ) buildTracker := builds.NewTracker( logger.Session("build-tracker"), db, engine, ) memberGrouper := []grouper.Member{ {"web", http_server.New(webListenAddr, httpHandler)}, {"debug", http_server.New(debugListenAddr, http.DefaultServeMux)}, {"drainer", ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error { close(ready) <-signals close(drain) return nil })}, {"pipelines", pipelines.SyncRunner{ Syncer: syncer, Interval: 10 * time.Second, Clock: clock.NewClock(), }}, {"builds", builds.TrackerRunner{ Tracker: buildTracker, Interval: 10 * time.Second, Clock: clock.NewClock(), }}, } group := grouper.NewParallel(os.Interrupt, memberGrouper) running := ifrit.Envoke(sigmon.New(group)) logger.Info("listening", lager.Data{ "web": webListenAddr, "debug": debugListenAddr, }) err = <-running.Wait() if err != nil { logger.Error("exited-with-failure", err) os.Exit(1) } }
func main() { InitErrorLog(os.Stderr) if len(os.Args) != 2 { printUsage() os.Exit(1) } else if os.Args[1] == "--help" { printUsage() os.Exit(1) } err := readConfigFile(os.Args[1]) if err != nil { elog.Fatalf("error while reading configuration file: %s", err) } if len(Config.Databases) == 0 { elog.Fatalf("at least one database must be configured") } listenAddr := net.JoinHostPort(Config.ListenAddress, strconv.Itoa(Config.ListenPort)) tcpAddr, err := net.ResolveTCPAddr("tcp", listenAddr) if err != nil { elog.Fatalf("could not resolve listen address: %s", err) } l, err := net.ListenTCP("tcp", tcpAddr) if err != nil { elog.Fatalf("could not open listen socket: %s", err) } var m sync.Mutex var connStatusNotifier chan struct{} listenerStateChange := func(ev pq.ListenerEventType, err error) { switch ev { case pq.ListenerEventConnectionAttemptFailed: elog.Warningf("Listener: could not connect to the database: %s", err.Error()) case pq.ListenerEventDisconnected: elog.Warningf("Listener: lost connection to the database: %s", err.Error()) m.Lock() close(connStatusNotifier) connStatusNotifier = nil m.Unlock() case pq.ListenerEventReconnected, pq.ListenerEventConnected: elog.Logf("Listener: connected to the database") m.Lock() connStatusNotifier = make(chan struct{}) m.Unlock() } } // make sure pq.Listener doesn't pick up any env variables os.Clearenv() clientConnectionString := fmt.Sprintf("fallback_application_name=allas %s", Config.ClientConnInfo) listener := pq.NewListener(clientConnectionString, 250*time.Millisecond, 5*time.Minute, listenerStateChange) nd := notifydispatcher.NewNotifyDispatcher(listener) nd.SetBroadcastOnConnectionLoss(false) nd.SetSlowReaderEliminationStrategy(notifydispatcher.NeglectSlowReaders) for { c, err := l.Accept() if err != nil { panic(err) } var myConnStatusNotifier chan struct{} m.Lock() if connStatusNotifier == nil { m.Unlock() go RejectFrontendConnection(c) continue } else { myConnStatusNotifier = connStatusNotifier } m.Unlock() newConn := NewFrontendConnection(c, nd, myConnStatusNotifier) go newConn.mainLoop(Config.StartupParameters, Config.Databases) } }
func (b *Backend) background(wg *sync.WaitGroup) { ctx := b.ctx.Fork() logger := b.logger defer ctx.WaitGroup().Done() listener := pq.NewListener(b.dsn, 200*time.Millisecond, 5*time.Second, nil) if err := listener.Listen("broadcast"); err != nil { // TODO: manage this more nicely panic("pq listen: " + err.Error()) } logger.Printf("pq listener started") peerWatcher := b.cluster.Watch() keepalive := time.NewTicker(3 * cluster.TTL / 4) defer keepalive.Stop() // Signal to constructor that we're ready to handle client connections. wg.Done() for { select { case <-ctx.Done(): return case <-keepalive.C: if b.desc != nil { if err := b.cluster.Update(b.desc); err != nil { logger.Printf("cluster: keepalive error: %s", err) } } // Ping to make sure the database connection is still live. if err := listener.Ping(); err != nil { logger.Printf("pq ping: %s\n", err) b.ctx.Terminate(fmt.Errorf("pq ping: %s", err)) return } case event := <-peerWatcher: b.Lock() switch e := event.(type) { case *cluster.PeerJoinedEvent: logger.Printf("cluster: peer %s joining with era %s", e.ID, e.Era) b.peers[e.ID] = e.Era case *cluster.PeerAliveEvent: if prevEra := b.peers[e.ID]; prevEra != e.Era { b.invalidatePeer(ctx, e.ID, prevEra) logger.Printf("cluster: peer %s changing era from %s to %s", e.ID, prevEra, e.Era) } b.peers[e.ID] = e.Era case *cluster.PeerLostEvent: logger.Printf("cluster: peer %s departing", e.ID) if era, ok := b.peers[e.ID]; ok { b.invalidatePeer(ctx, e.ID, era) delete(b.peers, e.ID) } } b.Unlock() case notice := <-listener.Notify: if notice == nil { logger.Printf("pq listen: received nil notification") // A nil notice indicates a loss of connection. We could // re-snapshot for all connected clients, but for now it's // easier to just shut down and force everyone to reconnect. b.ctx.Terminate(ErrPsqlConnectionLost) return } var msg BroadcastMessage if err := json.Unmarshal([]byte(notice.Extra), &msg); err != nil { logger.Printf("error: pq listen: invalid broadcast: %s", err) logger.Printf(" payload: %#v", notice.Extra) continue } // Check for UserID- if so, notify user instead of room if msg.UserID != "" { for _, lm := range b.listeners { if err := lm.NotifyUser(ctx, msg.UserID, msg.Event, msg.Exclude...); err != nil { logger.Printf("error: pq listen: notify user error on userID %s: %s", msg.Room, err) } } continue } // Check for global ban, which is a special-case broadcast. if msg.Room == "" && msg.Event.Type == proto.BounceEventType { for _, lm := range b.listeners { if err := lm.Broadcast(ctx, msg.Event, msg.Exclude...); err != nil { logger.Printf("error: pq listen: bounce broadcast error on %s: %s", msg.Room, err) } } continue } // TODO: if room name is empty, broadcast globally if lm, ok := b.listeners[msg.Room]; ok { logger.Printf("broadcasting %s to %s", msg.Event.Type, msg.Room) if err := lm.Broadcast(ctx, msg.Event, msg.Exclude...); err != nil { logger.Printf("error: pq listen: broadcast error on %s: %s", msg.Room, err) } } if msg.Event.Type == proto.PartEventType { payload, err := msg.Event.Payload() if err != nil { continue } if presence, ok := payload.(*proto.PresenceEvent); ok { if c, ok := b.partWaiters[presence.SessionID]; ok { c <- struct{}{} } } } } } }
func (a *app) listen() { listener := pq.NewListener(a.config.Connect.DSN(), minReconnectInterval, maxReconnectInterval, func(event pq.ListenerEventType, err error) { if err != nil { log.Errorf("Postgres listen: %v", err) return } log.Debugf("Postgres notify send event: %v", event) }) listener.Listen(channelTasks) listener.Listen(channelStopContainer) var ( events = listener.NotificationChannel() checkTasks = time.Tick(time.Minute) checkContainers = time.Tick(time.Minute * 10) ) for { select { case event := <-events: if event == nil { continue } log.Debugf("Received from [%s] playload: %s", event.Channel, event.Extra) switch event.Channel { case channelTasks: var task Task if err := json.Unmarshal([]byte(event.Extra), &task); err != nil { log.Errorf("Could not unmarshal notify playload: %v", err) continue } var accept bool if err := a.connect.Get(&accept, "SELECT accept FROM build.accept($1)", task.BuildID); err == nil { if accept { a.tasks <- task } } else { log.Debugf("Error when accepting a task: %v", err) } case channelStopContainer: var container struct { ContainerID string `json:"container_id"` CreatedAt time.Time `json:"created_at"` } if err := json.Unmarshal([]byte(event.Extra), &container); err != nil { log.Errorf("Could not unmarshal notify playload: %v", err) continue } log.Warnf("Remove container because build stopped by timeout: %s", container.ContainerID) a.docker.RemoveContainer(container.ContainerID) } case <-checkTasks: log.Debug("Checking for new tasks") for { var task Task if err := a.connect.Get(&task, "SELECT build_id, created_at FROM build.fetch()"); err != nil { if err != sql.ErrNoRows { log.Errorf("Could not fetch new task: %v", err) } break } a.tasks <- task } if _, err := a.connect.Exec("SELECT build.gc()"); err != nil { log.Errorf("Error when checking the lost builds: %v", err) } case <-checkContainers: if containers, err := a.docker.ListContainers(); err == nil { for _, container := range containers { if strings.HasPrefix(container.Name, "/pci-seq-") && container.CreatedAt.Add(containerZombieTTL).Before(time.Now()) { log.Warnf("Container was running too long time, destroy: %s", container.Name) container.Destroy() } } } } } }
// RegisterScanListener "subscribes" to the notifications published to the scan_listener notifier. // It has as input the usual db attributes and returns an int64 channel which can be consumed // for newly created scan id's. func (db *DB) RegisterScanListener(dbname, user, password, hostport, sslmode string) <-chan int64 { log := logger.GetLogger() reportProblem := func(ev pq.ListenerEventType, err error) { if err != nil { log.WithFields(logrus.Fields{ "error": err.Error(), }).Error("Listener Error") } } listenerChan := make(chan int64) listenerName := "scan_listener" connInfo := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", user, password, hostport, dbname, sslmode) go func() { listener := pq.NewListener(connInfo, 100*time.Millisecond, 10*time.Second, reportProblem) err := listener.Listen(listenerName) if err != nil { log.WithFields(logrus.Fields{ "listener": listenerName, "error": err.Error(), }).Error("could not listen for notification") close(listenerChan) return } for m := range listener.Notify { sid := m.Extra if !db.acquireScan(sid) { // skip this scan if we didn't acquire it continue } // scan was acquired, inform the scanner to launch it id, err := strconv.ParseInt(string(sid), 10, 64) if err != nil { log.WithFields(logrus.Fields{ "scan_id": sid, "error": err.Error(), }).Error("could not decode acquired notification") } listenerChan <- id log.WithFields(logrus.Fields{ "scan_id": id, }).Debug("Acquired notification.") } }() // Launch a goroutine that relaunches scans that have not yet been processed go func() { for { // don't requeue scans more than 3 times _, err := db.Exec(`UPDATE scans SET ack = false, timestamp = NOW() WHERE completion_perc = 0 AND attempts < 3 AND ack = true AND timestamp < NOW() - INTERVAL '10 minute'`) if err != nil { log.WithFields(logrus.Fields{ "error": err, }).Error("Could not run zero completion update query") } _, err = db.Exec(fmt.Sprintf(`SELECT pg_notify('%s', ''||id ) FROM scans WHERE ack=false ORDER BY id ASC LIMIT 1000`, listenerName)) if err != nil { log.WithFields(logrus.Fields{ "error": err, }).Error("Could not run unacknowledged scans periodic check.") } time.Sleep(3 * time.Minute) } }() return listenerChan }
"github.com/concourse/atc" "github.com/concourse/atc/db" "github.com/concourse/atc/event" ) var _ = Describe("One-off Builds", func() { var atcProcess ifrit.Process var dbListener *pq.Listener var atcPort uint16 var pipelineDBFactory db.PipelineDBFactory BeforeEach(func() { dbLogger := lagertest.NewTestLogger("test") postgresRunner.Truncate() dbConn = postgresRunner.Open() dbListener = pq.NewListener(postgresRunner.DataSourceName(), time.Second, time.Minute, nil) bus := db.NewNotificationsBus(dbListener, dbConn) sqlDB = db.NewSQL(dbLogger, dbConn, bus) pipelineDBFactory = db.NewPipelineDBFactory(dbLogger, dbConn, bus, sqlDB) atcProcess, atcPort = startATC(atcBin, 1) }) AfterEach(func() { ginkgomon.Interrupt(atcProcess) Expect(dbConn.Close()).To(Succeed()) Expect(dbListener.Close()).To(Succeed()) }) Describe("viewing a list of builds", func() {
func setupDatabaseListener() { if !checkDatabaseHandleValid(db) { return } //create/replace function for notifyPhoneNumber() if _, err := db.Exec(`CREATE or REPLACE FUNCTION notifyPhoneNumber() RETURNS trigger AS $$ BEGIN IF TG_OP='DELETE' THEN EXECUTE FORMAT('NOTIFY notifyphonenumber, ''%s''', OLD.PhoneNumber); ELSE EXECUTE FORMAT('NOTIFY notifyphonenumber, ''%s''', NEW.PhoneNumber); END IF; RETURN NULL; END; $$ LANGUAGE plpgsql;`); err != nil { log.Println(err) } else { log.Println("Successfully create/replace function notifyPhoneNumber()") } //check for trigger existence var triggerExist bool if err := db.QueryRow(`SELECT EXISTS( SELECT 1 FROM pg_trigger WHERE tgname='inprogresschange')`).Scan(&triggerExist); err != nil { log.Println(err) } if !triggerExist { if _, err := db.Exec(`CREATE TRIGGER inprogresschange AFTER INSERT OR UPDATE OR DELETE ON inprogress FOR EACH ROW EXECUTE PROCEDURE notifyPhoneNumber();`); err != nil { log.Println(err) } else { log.Println("Successfully create trigger inprogresschange") } } else { log.Println("Trigger inprogresschange exists.") } //Create handler for logging listener errors reportProblem := func(ev pq.ListenerEventType, err error) { if err != nil { fmt.Println(err.Error()) } } //Listen for table updates var listenerObj *pq.Listener listenerObj = pq.NewListener(os.Getenv("DATABASE_URL"), 10*time.Second, time.Minute, reportProblem) err := listenerObj.Listen("notifyphonenumber") if err != nil { log.Println(err) } //Find our session PID so we can ignore notifications from ourselves var pid int fmt.Println("Getting session PID...") if rows, err := db.Query(`SELECT * FROM pg_stat_activity WHERE pid = pg_backend_pid();`); err != nil { log.Println(err) } else { var i interface{} //empty interface to read unneeded columns into for rows.Next() { if err := rows.Scan(&i, &i, &pid, &i, &i, &i, &i, &i, &i, &i, &i, &i, &i, &i, &i, &i, &i, &i); err != nil { log.Println(err) } else { fmt.Println("Session PID:", pid) } } } //Monitor for noitification in background go func() { for { var notificationObj *pq.Notification notificationObj = <-listenerObj.Notify fmt.Printf("Backend PID %v\nChannel %v\nPayload %v\n", notificationObj.BePid, notificationObj.Channel, notificationObj.Extra) //Get updated row from database if the notifying PID is not this instance's PID if pid != notificationObj.BePid { if updatedRows := selectRowsFromTableByPhoneNumber("inprogress", notificationObj.Extra); updatedRows == nil { log.Println(err) } else { //We handle the possibility of deleted rows in laod pickups rows into memory since we can only enumerate over rows object once loadPickupRowsIntoMemory(&pickups, updatedRows, notificationObj) } } } }() }