func main() {
var help = flag.Bool("help", false, "print help")
var addr = flag.String("addr", "", "address to dial")
var method = flag.String("method", "", "rpc method")
var parg = flag.String("arg", "", "method argument")
flag.Parse()
if *help || *addr == "" || *method == "" || *parg == "" {
flag.PrintDefaults()
return
}
client, err := util.JsonRpcDial(*addr)
if err != nil {
log.Fatal(err)
}
var arg interface{}
if err := json.Unmarshal([]byte(*parg), &arg); err != nil {
log.Fatal(errors.NewErr(err))
}
var res interface{}
if err := client.Call(*method, arg, &res); err != nil {
log.Fatal(errors.NewErr(err))
}
log.Printf("%+v\n", res)
}
func (self *Gatekeeper) Write(url, key string, data []byte) (Value, error) {
log.Printf("Gatekeeper.Write(%v, %v)\n", url, key)
if self.file.file == nil {
f, err := os.Create(self.dir + "/" + strconv.Itoa(int(self.fNum)))
if err != nil {
return Value{}, errors.NewErr(err)
}
self.file = gkFile{
file: f,
offset: 0,
end: time.Now().Add(self.maxTime),
}
} else if self.file.offset >= self.maxFileSize {
if err := self.nextFile(); err != nil {
return Value{}, err
}
} else if self.file.end.Before(time.Now()) {
if err := self.file.file.Sync(); err != nil {
return Value{}, errors.NewErr(err)
}
self.file.end = time.Now().Add(self.maxTime)
}
offset := self.file.offset
cnt := 0
n, err := self.file.WriteLenval([]byte(url))
if err != nil {
return Value{}, err
}
cnt += n
n, err = self.file.WriteLenval(data)
if err != nil {
return Value{}, err
}
cnt += n
self.file.offset += uint64(cnt)
res := Value{
FNum: self.fNum,
Offset: offset,
Len: uint64(cnt),
}
self.trie.Add([]byte(key), res)
// TODO: remove
self.file.file.Sync()
log.Printf("Gatekeeper.Write(%v, %v) OK (%+v)\n", url, key, res)
return res, nil
}
func (self *gkFile) WriteLenval(b []byte) (n int, err error) {
buf := make([]byte, 8)
num := binary.PutUvarint(buf, uint64(len(b)))
n1, err := self.file.Write(buf[:num])
if err != nil {
return n1, errors.NewErr(err)
}
n2, err := self.file.Write(b)
return n1 + n2, errors.NewErr(err)
}
func DoIterTsvRequest(url string, it iter.Iterator) (*http.Response, error) {
req, err := http.NewRequest("GET", url, iter.ReadDelim(it, []byte("\t")))
if err != nil {
return nil, errors.NewErr(err)
}
req.ContentLength = -1
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, errors.NewErr(err)
}
return resp, nil
}
func (self *Balanser) Request(w http.ResponseWriter, r *http.Request) ([]error, error) {
if r.Method != "GET" {
return nil, util.ClientError(errors.New("Can only process GET requests!"))
}
url := *r.URL
url.Scheme = "http"
var resErrors []error = nil
failed := map[string]struct{}{}
for {
if len(failed) == self.count {
return resErrors, errors.New("All backends are broken!")
}
backend, err := self.router.Choose()
if err != nil {
return resErrors, err
}
if _, ok := failed[backend]; ok {
continue
}
url.Host = backend
nreq, err := http.NewRequest("GET", url.String(), r.Body)
if err != nil {
return resErrors, errors.NewErr(err)
}
nreq.Header = r.Header
resp, err := (&http.Client{}).Do(nreq)
if err != nil {
failed[backend] = struct{}{}
resErrors = append(resErrors, errors.NewErr(err))
continue
}
defer resp.Body.Close()
for k, hs := range resp.Header {
for _, val := range hs {
w.Header().Add(k, val)
}
}
w.WriteHeader(resp.StatusCode)
_, err = io.Copy(w, resp.Body)
return resErrors, errors.NewErr(err)
}
}
func (self *Balanser) Request(client *net.TCPConn) error {
defer client.Close()
failed := map[string]struct{}{}
for {
if len(failed) == self.count {
return errors.New("All backends are broken!")
}
backend, err := self.router.Choose()
if err != nil {
return err
}
if _, ok := failed[backend]; ok {
continue
}
conn, err := net.Dial("tcp", backend)
if err != nil {
failed[backend] = struct{}{}
continue
}
server := conn.(*net.TCPConn)
clientClosed := make(chan error, 1)
go func() {
_, err := io.Copy(server, client)
clientClosed <- errors.NewErr(err)
}()
serverClosed := make(chan error, 1)
go func() {
_, err := io.Copy(client, server)
serverClosed <- errors.NewErr(err)
}()
select {
case err = <-clientClosed:
server.SetLinger(0)
server.CloseRead()
case err = <-serverClosed:
}
return err
}
}
func SendJson(w http.ResponseWriter, data interface{}) error {
w.Header().Set("Content-type", "application/json")
if err := json.NewEncoder(w).Encode(data); err != nil {
return errors.NewErr(err)
}
return nil
}
func (self *Downloader) Download(url string) (string, error) {
log.Printf("Downloader.Download(%s)\n", url)
resp, err := http.Get(url)
if err != nil {
return "", errors.NewErr(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", errors.NewErr(err)
}
log.Printf("Downloader.Download(%s) OK!\n", url)
return string(body), nil
}
func (self *FileReader) Write(b []byte) (n int, err error) {
n, err = self.file.Write(b)
if err == io.EOF {
return n, err
}
return n, errors.NewErr(err)
}
func main() {
var help = flag.Bool("help", false, "print help")
var port = flag.Int("port", -1, "port to listen")
var gracefulRestart = graceful.SetFlag()
flag.Parse()
if *help || *port == -1 {
flag.PrintDefaults()
return
}
srv := rpc.NewServer()
srv.Register(&downloader.DownloaderServer{&downloader.Downloader{}})
server := gjsonrpc.NewServer(srv)
graceful.SetSighup(server)
if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
log.Fatal(errors.NewErr(err))
}
if err := graceful.Restart(server); err != nil {
log.Fatal(err)
}
}
func main() {
var help = flag.Bool("help", false, "print help")
var port = flag.Int("port", -1, "port to listen")
var dir = flag.String("dir", "", "data directory")
var maxFileSize = flag.Int("max-size", 10*1024*1024, "maximum file size")
var maxTime = flag.Int("max-time", 1*60, "maximum time between sync calls (in seconds)")
var gracefulRestart = graceful.SetFlag()
flag.Parse()
if *help || *port == -1 {
flag.PrintDefaults()
return
}
gk, err := gatekeeper.NewGatekeeper(*dir, uint64(*maxFileSize), time.Duration(*maxTime)*time.Second)
if err != nil {
log.Fatal(err)
}
srv := rpc.NewServer()
srv.Register(&gatekeeper.GatekeeperServer{gk})
server := gjsonrpc.NewServer(srv)
graceful.SetSighup(server)
if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
log.Fatal(errors.NewErr(err))
}
if err := graceful.Restart(server); err != nil {
log.Fatal(err)
}
}
func (self *Caregiver) PushUrls(urls []string) error {
log.Printf("Caregiver.PushUrls(%#v)\n", urls)
data := map[string][]string{}
for _, u := range urls {
url, err := url.Parse(u)
if err != nil {
return errors.NewErr(err)
}
path := url.Path
if len(path) != 0 && path[0] == '/' {
path = path[1:]
}
if url.RawQuery != "" {
path += "?"
path += url.RawQuery
}
data[url.Host] = append(data[url.Host], path)
}
for host, urls := range data {
self.mutex.Lock()
hosts, ok := self.hosts[host]
if !ok {
hosts = self.getData(host)
self.hosts[host] = hosts
}
self.mutex.Unlock()
hosts.urls.EnqueueAll(urls...)
}
log.Printf("Caregiver.PushUrls(%#v) OK\n", urls)
return nil
}
func main() {
var help = flag.Bool("help", false, "print help")
var port = flag.Int("port", -1, "port to listen")
var cacheTime = flag.Int("cachetime", -1, "time to keep resolved records in cache (in seconds)")
var gracefulRestart = graceful.SetFlag()
flag.Parse()
if *help || *port == -1 || *cacheTime == -1 {
flag.PrintDefaults()
return
}
srv := rpc.NewServer()
srv.Register(&dns.ResolverServer{dns.NewResolver(time.Duration(*cacheTime) * time.Second)})
log.Println(srv)
server := gjsonrpc.NewServer(srv)
graceful.SetSighup(server)
if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
log.Fatal(errors.NewErr(err))
}
if err := graceful.Restart(server); err != nil {
log.Fatal(err)
}
}
func NewGatekeeper(dir string, maxFileSize uint64, maxTime time.Duration) (*Gatekeeper, error) {
self := &Gatekeeper{
dir: dir,
maxTime: maxTime,
maxFileSize: maxFileSize,
fNum: 0,
}
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil, errors.NewErr(err)
}
arr := make(valTArr, len(files))
for i, f := range files {
num, err := strconv.Atoi(f.Name())
if err != nil {
return nil, errors.NewErr(err)
}
arr[i] = valT{
num: uint(num),
file: f,
}
}
sort.Sort(arr)
counts := map[uint]uint{}
for _, f := range arr {
counts[f.num] = 0
self.fNum = f.num + 1
err := self.load(self.dir+"/"+f.file.Name(), f.num, counts)
if err != nil {
return nil, err
}
}
for k, v := range counts {
if v == 0 {
if err := os.Remove(self.dir + "/" + strconv.Itoa(int(k))); err != nil {
return nil, errors.NewErr(err)
}
}
}
return self, nil
}
func (self *CaregiverClient) PullUrls() (map[string]string, error) {
var res map[string]string
if err := self.Call("CaregiverServer.PullUrls", struct{}{}, &res); err != nil {
return nil, errors.NewErr(err)
}
return res, nil
}
func (self *GracefulListener) File() (*os.File, error) {
f, err := self.base.File()
if err != nil {
return nil, errors.NewErr(err)
}
return f, nil
}
func JsonRpcDial(addr string) (*rpc.Client, error) {
conn, err := net.Dial("tcp", addr)
if err != nil {
return nil, errors.NewErr(err)
}
return jsonrpc.NewClient(conn), nil
}
func (self *FileReader) Seek(offset int64, whence int) (ret int64, err error) {
ret, err = self.file.Seek(offset, whence)
if err == io.EOF {
return ret, err
}
return ret, errors.NewErr(err)
}
func (self *DownloaderClient) DownloadAll(urls []string) ([]string, error) {
var res []string
if err := self.Call("DownloaderServer.DownloadAll", ArgsAll{Urls: urls}, &res); err != nil {
return nil, errors.NewErr(err)
}
return res, nil
}
func (self *DownloaderClient) Download(url string) (string, error) {
var res string
if err := self.Call("DownloaderServer.Download", Args{Url: url}, &res); err != nil {
return "", errors.NewErr(err)
}
return string(res), nil
}
func (self *ResolverClient) ResolveAll(hosts []string) ([][]string, error) {
var res [][]string
if err := self.Call("ResolverServer.ResolveAll", ArgsAll{Hosts: hosts}, &res); err != nil {
return nil, errors.NewErr(err)
}
return res, nil
}
func NewDownloaderClient(addr string) (DownloaderClient, error) {
c, err := util.JsonRpcDial(addr)
if err != nil {
return DownloaderClient{}, errors.NewErr(err)
}
return DownloaderClient{c}, nil
}
func NewResolverClient(addr string) (ResolverClient, error) {
c, err := util.JsonRpcDial(addr)
if err != nil {
return ResolverClient{}, errors.NewErr(err)
}
return ResolverClient{c}, nil
}
func GetListener(addr string, graceful bool) (*net.TCPListener, error) {
var listener net.Listener = nil
var err error = nil
if graceful {
file := os.NewFile(3, "")
listener, err = net.FileListener(file)
if err != nil {
file.Close()
return nil, errors.NewErr(err)
}
} else {
listener, err = net.Listen("tcp", addr)
if err != nil {
return nil, errors.NewErr(err)
}
}
return listener.(*net.TCPListener), nil
}
func (self *Server) Serve(l *net.TCPListener) error {
self.listener = graceful.NewGracefulListener(l, self)
err := self.base.Serve(self.listener)
self.restart.Wait()
if err == graceful.NeedRestart {
return nil
}
return errors.NewErr(err)
}
func main() {
var help = flag.Bool("help", false, "print help")
var port = flag.Int("port", -1, "port to listen")
var dlerArrd = flag.String("dl", "", "downloader address")
var dnsAddr = flag.String("dns", "", "dns resolver address")
var defaultMaxCount = flag.Int("maxcount", 1, "default maximum count urls for specific host to download at once")
var defaultTimeout = flag.Int("timeout", 0, "default timeout between downloads for specific host (im ms)")
var pullTimeout = flag.Int("pull-timeout", 100, "pull check timeout (im ms)")
var workTimeout = flag.Int("work-timeout", 1000, "pull check timeout (im ms)")
var gracefulRestart = graceful.SetFlag()
flag.Parse()
if *help || *port == -1 || *dlerArrd == "" {
flag.PrintDefaults()
return
}
ct, err := caregiver.NewCaregiver(
*dlerArrd,
*dnsAddr,
uint(*defaultMaxCount),
time.Duration(*defaultTimeout)*time.Millisecond,
time.Duration(*pullTimeout)*time.Millisecond,
time.Duration(*workTimeout)*time.Millisecond,
)
if err != nil {
log.Fatal(err)
}
srv := rpc.NewServer()
srv.Register(&caregiver.CaregiverServer{ct})
server := gjsonrpc.NewServer(srv)
graceful.SetSighup(server)
go func() {
for {
if err := ct.Start(); err != nil {
log.Errorln(err)
time.Sleep(ct.WorkTimeout)
continue
}
break
}
}()
if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
log.Fatal(errors.NewErr(err))
}
if err := graceful.Restart(server); err != nil {
log.Fatal(err)
}
}
func Restart(srv GServer) error {
f, err := srv.Listener().File()
if err != nil {
return errors.NewErr(err)
}
newargs := make([]string, len(os.Args)+1)
for i, v := range os.Args {
newargs[i] = v
}
newargs[len(os.Args)] = "-graceful"
_, err = syscall.ForkExec(os.Args[0], newargs, &syscall.ProcAttr{
Env: os.Environ(),
Files: []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd(), f.Fd()},
})
if err != nil {
return errors.NewErr(err)
}
return nil
}
func Open(file string) (*FileReader, error) {
f, err := os.Open(file)
if err != nil {
return nil, errors.NewErr(err)
}
return &FileReader{
file: f,
offset: 0,
buf: []byte{0},
}, nil
}
func GetParamInt(r *http.Request, name string) (int, error) {
sval, err := GetParam(r, name)
if err != nil {
return -1, err
}
val, err := strconv.Atoi(sval)
if err != nil {
return -1, errors.NewErr(err)
}
return val, nil
}
func UrlTransform(key string) (string, error) {
u, err := url.Parse(string(key))
if err != nil {
return "", errors.NewErr(err)
}
arr := strings.Split(u.Host, ".")
for i, j := 0, len(arr)-1; i < j; i, j = i+1, j-1 {
arr[i], arr[j] = arr[j], arr[i]
}
u.Host = strings.Join(arr, ".")
return u.String(), nil
}