func main() {
// add handlers to help us track memory usage - they don't track memory until they're told to
profiler.AddMemoryProfilingHandlers()
// Uncomment if you want to start profiling automatically
profiler.StartProfiling()
// listen on port 6060 (pick a port)
http.ListenAndServe(":6060", nil)
}
func main() {
u := universe.NewCache(universe.Config{
BerksOnly: config.Opts.BerksOnly,
})
// add handlers to help us track memory usage - they don't track memory until they're told to
profiler.AddMemoryProfilingHandlers()
profiler.RegisterExtraServiceInfoRetriever(u.ServiceInfo)
profiler.StartProfiling()
ghsConfig := gh.Config{
Token: config.Opts.Token,
Org: config.Opts.Org,
SyncInterval: config.Opts.SyncInterval,
MaxVersions: config.Opts.MaxGithubVersions,
}
ghs := gh.New(ghsConfig, u)
var wg sync.WaitGroup
wg.Add(1) // we want to do some stuff after github is done, this is so we can block on the gh repo processing before we do something like add in other sources
// Add supermarket to the universe after github
d, _ := time.ParseDuration("6h") // not exposed as a tunable yet
conf := supermarket.Config{SyncInterval: d}
sm := supermarket.New(conf, u)
// ye yea hate the ! no sync, but default behavior should be to sync
if !config.Opts.NoSync {
go func() {
if err := ghs.Sync(); err != nil {
log.Printf("Error syncing github org %s %s", ghs.Conf.Org, err.Error())
} else {
wg.Done()
}
}()
}
go periodicSync(ghs)
go periodicSync(sm)
// create our custom http server/router
router := httprouter.New()
server := &http.Server{
Addr: fmt.Sprintf("%s:%d", config.Opts.Bind, config.Opts.Port),
Handler: Log(router),
}
ghook = hook.NewServer()
ghook.Secret = config.Opts.Secret
ghook.Path = "/hook"
router.GET("/", rootHandler)
// can see a post or refresh or purge going here as well
router.GET("/cookbook/:cook", u.CookBookHandler)
router.HandlerFunc("GET", "/profiler/info.html", profiler.MemStatsHTMLHandler)
router.HandlerFunc("GET", "/profiler/info", profiler.ProfilingInfoJSONHandler)
router.HandlerFunc("GET", "/profiler/start", profiler.StartProfilingHandler)
router.HandlerFunc("GET", "/profiler/stop", profiler.StopProfilingHandler)
// github rate info
router.HandlerFunc("GET", "/rate", ghs.RateHandler)
// throw these standard handlers in the router
router.HandlerFunc("GET", "/universe", u.Handler)
router.HandlerFunc("PUT", "/hook", ghook.ServeHTTP)
router.HandlerFunc("POST", "/hook", ghook.ServeHTTP)
// TODO:(jnelson) the EventProcessor is tied to a gh source, but really it should be not a global event stream, but configerd per-org since the gh sources can be multi-org.
// Potentially we should setup a WebHook on the Source interface, so each source could have a configured hook/hook path.. Sun Sep 20 03:25:05 2015
go ghs.EventProcessor(ghook.Events)
if !config.Opts.NoSync {
wg.Wait() // make sure we have processed everything from github before other sources
if err := sm.Sync(); err != nil { // sync supermarket
log.Println("Error syncing supermarket ", err)
}
}
log.Printf("starting server on %s:%d\n", config.Opts.Bind, config.Opts.Port)
if config.Opts.TLS {
log.Fatal(server.ListenAndServeTLS(config.Opts.Cert, config.Opts.Key))
} else {
log.Fatal(server.ListenAndServe())
}
}
func main() {
flagWorker := flag.Int("workers", 8, "Number of concurrent workers")
flagMaxSize := flag.String("max", "10MB", "Max size of files to scan")
flagWait := flag.Int("wait", 5, "Wait delay before completion")
//flagConfig := flag.String("config", "config.cfg", "Config file")
flagDebug := flag.Bool("debug", false, "Debugging")
flag.Parse()
if flag.NArg() != 1 {
myUsage()
os.Exit(1)
}
startDir := filepath.FromSlash(flag.Arg(0))
if strings.HasSuffix(startDir, string(filepath.Separator)) == false {
startDir += string(filepath.Separator)
}
if *flagDebug {
profiler.AddMemoryProfilingHandlers()
profiler.StartProfiling()
go http.ListenAndServe(":6060", nil)
}
//loadConfig(*flagConfig)
var err error
config.MaxFileSize, err = ToBytes(*flagMaxSize)
testErrDie("Error parsing max size", err)
config.MaxThreads = *flagWorker
config.WaitDelay = *flagWait
loadRegexps("regexps.cfg")
reportChan := make(chan FileReport)
var reportWg sync.WaitGroup
reportWg.Add(1)
go createReport(reportChan, &reportWg)
workChannel := make(chan string)
dispChan := make(chan string, 3)
var workerWg sync.WaitGroup
workerWg.Add(config.MaxThreads)
now := time.Now()
for i := 0; i < config.MaxThreads; i++ {
go worker(workChannel, reportChan, dispChan, &workerWg)
}
go dispatcher(dispChan, workChannel)
dispChan <- startDir
workerWg.Wait()
fmt.Println("Dirs:", nrOfDirs.nr)
fmt.Println("Files:", nrOfFiles.nr)
close(reportChan)
reportWg.Wait()
fmt.Println(time.Since(now))
}