예제 #1
0
파일: analyser.go 프로젝트: the42/ogdat
func heartbeat(interval int) chan bool {
	retchan := make(chan bool)
	go func() {
		for {
			dbconn, err := database.GetDatabaseConnection()
			if err != nil {
				logger.Panicln(err)
			}
			db := &database.DBConn{DBer: dbconn, Appid: AppID}
			if err := db.HeartBeat(); err != nil {
				logger.Panicln(err)
			}
			dbconn.Close()
			logger.Printf("Watchdog beating every %d minute\n", interval)
			retchan <- true
			time.Sleep(time.Duration(interval) * time.Minute)
		}
	}()
	return retchan
}
예제 #2
0
파일: analyser.go 프로젝트: the42/ogdat
func main() {

	dbcon, err := database.GetDatabaseConnection()
	if err != nil {
		logger.Panicln(err)
	}
	defer dbcon.Close()
	analyser := NewAnalyser(dbcon, redis.NewPool(func() (redis.Conn, error) { return database.GetRedisConnection(getredisconnect()) }, 10))

	restful.DefaultResponseMimeType = restful.MIME_JSON
	restful.DefaultContainer.EnableContentEncoding(true)
	restful.Add(NewAnalyseOGDATRESTService(analyser))

	config := swagger.Config{
		WebServicesUrl:  discoveryhost(),
		ApiPath:         "/swaggerdoc",
		SwaggerPath:     "/doc/",
		SwaggerFilePath: "swagger-ui/dist/",
		WebServices:     restful.RegisteredWebServices()}
	swagger.InstallSwaggerService(config)

	logger.Printf("analyser (%s) listening on port %s\n", AppID, portbinding())
	go func() {
		logger.Fatal(http.ListenAndServe(":"+portbinding(), nil))
	}()

	var datachange, urlchange chan []byte
	var heartbeatchannel chan bool

	populatedatasetinfo := func() {
		if err := analyser.populatedatasetinfo(); err != nil {
			logger.Panicln(err)
		}
	}

	if !isonlyweb() {
		heartbeatchannel = heartbeat(getheartbeatinterval())
		<-heartbeatchannel // Wait for the first heartbeat, so the logging in the database is properly set up

		populatedatasetinfo()

		datachange = analyser.listenredischannel(watcherappid + ":DataChange")
		urlchange = analyser.listenredischannel(watcherappid + ":UrlChange")
	}

	for {
		select {
		case <-urlchange:
			// = REMARK =
			// Naive approach here. If a URLChange or DataChange event is triggered,
			// the whole analytic database will be recreated. It would be better to trace
			// only the affected datasets and only create the relevant statistic.
			// In future, urlchange/datachange might contain a JSON-encoded []byte which contains
			// the affected IDs
			logger.Println("Received URL change notice, re-generating database analysis")
			populatedatasetinfo()
		case <-datachange:
			logger.Println("Received Data change notice, re-generating database analysis")
			populatedatasetinfo()
		case <-heartbeatchannel:
			logger.Println("Idle")
		}
	}
}
예제 #3
0
파일: ogdatwatcher.go 프로젝트: the42/ogdat
func mymain() int {

	if flag.NFlag() == 0 {
		fmt.Println("No command line flags given. Usage:")
		flag.PrintDefaults()
		logger.Panicln("Fatal: No command line flags given")
	}

	dbconnection, err := database.GetDatabaseConnection()
	if err != nil {
		logger.Panicln(err)
	}

	watcherdatabase = &watcherdb{DBConn: database.DBConn{DBer: dbconnection, Appid: AppID}}
	defer dbconnection.Close()

	if *resettdb {
		resetdb()
		logger.Println("Info: Earyl exit due to maintainance switches")
		return 1
	}

	if *servetdb {

		portal = ckan.NewDataPortalAPIEndpoint(getckanurl(), "2/")
		heartbeatinterval := getheartbeatinterval()
		heartbeatchannel := heartbeat(heartbeatinterval)

		loc, err := time.LoadLocation(gettimezone())
		if err != nil {
			logger.Panicln(err)
		}

		logger.Printf("Processing relative to timezone %s\n", loc)

		whenurlcheck := urlchecktime(loc)
		whendatacheck := datachecktime(loc)

		datacheckchan := time.After(0) // assign a ticker of 0 to immediately trigger a data check
		urlcheckchan := time.After(whenurlcheck.Sub(time.Now().In(loc)))

		for {
			select {
			case <-urlcheckchan:
				anz, err := checkurls(dbconnection)
				if err != nil {
					logger.Panicln(err)
				}
				if anz > 0 {
					if err := redispublishint("UrlChange", anz); err != nil {
						logger.Printf("Cannot publish url change to redis: %s\n", err)
					}
				}
				whenurlcheck = urlchecktime(loc)
				urlcheckchan = time.After(whenurlcheck.Sub(time.Now().In(loc)))
			case <-datacheckchan:
				anz, err := checkdata(dbconnection)
				if err != nil {
					logger.Panicln(err)
				}
				if anz > 0 {
					if err := redispublishint("DataChange", anz); err != nil {
						logger.Printf("Cannot publish data change to redis: %s\n", err)
					}
				}
				whendatacheck = datachecktime(loc)
				datacheckchan = time.After(whendatacheck.Sub(time.Now().In(loc)))
			case <-time.After(time.Duration(heartbeatinterval) * time.Minute):
			}

			now := time.Now().In(loc)
			logger.Printf("%v: Nothing to do\n", now)

			datacheckdiff := whendatacheck.Sub(now)
			urlcheckdiff := whenurlcheck.Sub(now)

			logger.Printf("Next Data check in %v\n", datacheckdiff)
			logger.Printf("Next Url check in %v\n", urlcheckdiff)

			// drain the heartbeat channel; without draining, the heartbeat won't get written to the database
			select {
			case <-heartbeatchannel:
			}

			if sdidle != nil && *sdidle > 0 {
				if datacheckdiff > *sdidle && urlcheckdiff > *sdidle {
					logger.Printf("Next activity is more than %v ahead, terminating\n", *sdidle)
					return 0
				}
			}
		}
	}
	return 0
}