func main() {
var help = flag.Bool("help", false, "print help")
var port = flag.Int("port", -1, "port to listen")
var cacheTime = flag.Int("cachetime", -1, "time to keep resolved records in cache (in seconds)")
var gracefulRestart = graceful.SetFlag()
flag.Parse()
if *help || *port == -1 || *cacheTime == -1 {
flag.PrintDefaults()
return
}
srv := rpc.NewServer()
srv.Register(&dns.ResolverServer{dns.NewResolver(time.Duration(*cacheTime) * time.Second)})
log.Println(srv)
server := gjsonrpc.NewServer(srv)
graceful.SetSighup(server)
if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
log.Fatal(errors.NewErr(err))
}
if err := graceful.Restart(server); err != nil {
log.Fatal(err)
}
}
func main() {
var help = flag.Bool("help", false, "print help")
var port = flag.Int("port", -1, "port to listen")
var gracefulRestart = graceful.SetFlag()
flag.Parse()
if *help || *port == -1 {
flag.PrintDefaults()
return
}
srv := rpc.NewServer()
srv.Register(&downloader.DownloaderServer{&downloader.Downloader{}})
server := gjsonrpc.NewServer(srv)
graceful.SetSighup(server)
if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
log.Fatal(errors.NewErr(err))
}
if err := graceful.Restart(server); err != nil {
log.Fatal(err)
}
}
func main() {
var help = flag.Bool("help", false, "print help")
var port = flag.Int("port", -1, "port to listen")
var dir = flag.String("dir", "", "data directory")
var maxFileSize = flag.Int("max-size", 10*1024*1024, "maximum file size")
var maxTime = flag.Int("max-time", 1*60, "maximum time between sync calls (in seconds)")
var gracefulRestart = graceful.SetFlag()
flag.Parse()
if *help || *port == -1 {
flag.PrintDefaults()
return
}
gk, err := gatekeeper.NewGatekeeper(*dir, uint64(*maxFileSize), time.Duration(*maxTime)*time.Second)
if err != nil {
log.Fatal(err)
}
srv := rpc.NewServer()
srv.Register(&gatekeeper.GatekeeperServer{gk})
server := gjsonrpc.NewServer(srv)
graceful.SetSighup(server)
if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
log.Fatal(errors.NewErr(err))
}
if err := graceful.Restart(server); err != nil {
log.Fatal(err)
}
}
func main() {
var help = flag.Bool("help", false, "print help")
var port = flag.Int("port", -1, "port to listen")
var dlerArrd = flag.String("dl", "", "downloader address")
var dnsAddr = flag.String("dns", "", "dns resolver address")
var defaultMaxCount = flag.Int("maxcount", 1, "default maximum count urls for specific host to download at once")
var defaultTimeout = flag.Int("timeout", 0, "default timeout between downloads for specific host (im ms)")
var pullTimeout = flag.Int("pull-timeout", 100, "pull check timeout (im ms)")
var workTimeout = flag.Int("work-timeout", 1000, "pull check timeout (im ms)")
var gracefulRestart = graceful.SetFlag()
flag.Parse()
if *help || *port == -1 || *dlerArrd == "" {
flag.PrintDefaults()
return
}
ct, err := caregiver.NewCaregiver(
*dlerArrd,
*dnsAddr,
uint(*defaultMaxCount),
time.Duration(*defaultTimeout)*time.Millisecond,
time.Duration(*pullTimeout)*time.Millisecond,
time.Duration(*workTimeout)*time.Millisecond,
)
if err != nil {
log.Fatal(err)
}
srv := rpc.NewServer()
srv.Register(&caregiver.CaregiverServer{ct})
server := gjsonrpc.NewServer(srv)
graceful.SetSighup(server)
go func() {
for {
if err := ct.Start(); err != nil {
log.Errorln(err)
time.Sleep(ct.WorkTimeout)
continue
}
break
}
}()
if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
log.Fatal(errors.NewErr(err))
}
if err := graceful.Restart(server); err != nil {
log.Fatal(err)
}
}
func main() {
var help = flag.Bool("help", false, "print help")
var port = flag.Int("port", -1, "port to listen")
var gkArrd = flag.String("gatekeeper", "", "gatekeeper address")
var cgAddr = flag.String("caregiver", "", "caregiver address")
var vint = flag.Int("interval", 1, "sleep interval")
var pushCnt = flag.Int("push-cnt", 10, "urls to push to the caregiver at a time")
var gracefulRestart = graceful.SetFlag()
flag.Parse()
if *help || *port == -1 || *gkArrd == "" || *cgAddr == "" {
flag.PrintDefaults()
return
}
sp, err := spider.NewSpider(*gkArrd, *cgAddr, time.Duration(*vint)*time.Second, uint(*pushCnt))
if err != nil {
log.Fatal(err)
}
srv := rpc.NewServer()
srv.Register(&spider.SpiderServer{sp})
server := gjsonrpc.NewServer(srv)
graceful.SetSighup(server)
go func() {
for {
if err := sp.RunPusher(); err != nil {
log.Errorln(err)
}
}
}()
go func() {
for {
if err := sp.RunPuller(); err != nil {
log.Errorln(err)
}
}
}()
if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
log.Fatal(errors.NewErr(err))
}
if err := graceful.Restart(server); err != nil {
log.Fatal(err)
}
}