Beispiel #1
0
func main() {
	var help = flag.Bool("help", false, "print help")
	var addr = flag.String("addr", "", "address to dial")
	var method = flag.String("method", "", "rpc method")
	var parg = flag.String("arg", "", "method argument")
	flag.Parse()

	if *help || *addr == "" || *method == "" || *parg == "" {
		flag.PrintDefaults()
		return
	}

	client, err := util.JsonRpcDial(*addr)
	if err != nil {
		log.Fatal(err)
	}

	var arg interface{}
	if err := json.Unmarshal([]byte(*parg), &arg); err != nil {
		log.Fatal(errors.NewErr(err))
	}

	var res interface{}
	if err := client.Call(*method, arg, &res); err != nil {
		log.Fatal(errors.NewErr(err))
	}

	log.Printf("%+v\n", res)
}
Beispiel #2
0
func (self *Gatekeeper) Write(url, key string, data []byte) (Value, error) {
	log.Printf("Gatekeeper.Write(%v, %v)\n", url, key)

	if self.file.file == nil {
		f, err := os.Create(self.dir + "/" + strconv.Itoa(int(self.fNum)))
		if err != nil {
			return Value{}, errors.NewErr(err)
		}

		self.file = gkFile{
			file:   f,
			offset: 0,
			end:    time.Now().Add(self.maxTime),
		}
	} else if self.file.offset >= self.maxFileSize {
		if err := self.nextFile(); err != nil {
			return Value{}, err
		}
	} else if self.file.end.Before(time.Now()) {
		if err := self.file.file.Sync(); err != nil {
			return Value{}, errors.NewErr(err)
		}
		self.file.end = time.Now().Add(self.maxTime)
	}

	offset := self.file.offset

	cnt := 0
	n, err := self.file.WriteLenval([]byte(url))
	if err != nil {
		return Value{}, err
	}
	cnt += n

	n, err = self.file.WriteLenval(data)
	if err != nil {
		return Value{}, err
	}
	cnt += n

	self.file.offset += uint64(cnt)

	res := Value{
		FNum:   self.fNum,
		Offset: offset,
		Len:    uint64(cnt),
	}

	self.trie.Add([]byte(key), res)

	// TODO: remove
	self.file.file.Sync()

	log.Printf("Gatekeeper.Write(%v, %v) OK (%+v)\n", url, key, res)
	return res, nil
}
Beispiel #3
0
func (self *gkFile) WriteLenval(b []byte) (n int, err error) {
	buf := make([]byte, 8)
	num := binary.PutUvarint(buf, uint64(len(b)))
	n1, err := self.file.Write(buf[:num])
	if err != nil {
		return n1, errors.NewErr(err)
	}

	n2, err := self.file.Write(b)
	return n1 + n2, errors.NewErr(err)
}
Beispiel #4
0
func DoIterTsvRequest(url string, it iter.Iterator) (*http.Response, error) {
	req, err := http.NewRequest("GET", url, iter.ReadDelim(it, []byte("\t")))
	if err != nil {
		return nil, errors.NewErr(err)
	}
	req.ContentLength = -1

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, errors.NewErr(err)
	}

	return resp, nil
}
Beispiel #5
0
func (self *Balanser) Request(w http.ResponseWriter, r *http.Request) ([]error, error) {
	if r.Method != "GET" {
		return nil, util.ClientError(errors.New("Can only process GET requests!"))
	}

	url := *r.URL
	url.Scheme = "http"

	var resErrors []error = nil
	failed := map[string]struct{}{}
	for {
		if len(failed) == self.count {
			return resErrors, errors.New("All backends are broken!")
		}

		backend, err := self.router.Choose()
		if err != nil {
			return resErrors, err
		}

		if _, ok := failed[backend]; ok {
			continue
		}

		url.Host = backend
		nreq, err := http.NewRequest("GET", url.String(), r.Body)
		if err != nil {
			return resErrors, errors.NewErr(err)
		}

		nreq.Header = r.Header
		resp, err := (&http.Client{}).Do(nreq)
		if err != nil {
			failed[backend] = struct{}{}
			resErrors = append(resErrors, errors.NewErr(err))
			continue
		}
		defer resp.Body.Close()

		for k, hs := range resp.Header {
			for _, val := range hs {
				w.Header().Add(k, val)
			}
		}
		w.WriteHeader(resp.StatusCode)
		_, err = io.Copy(w, resp.Body)
		return resErrors, errors.NewErr(err)
	}
}
Beispiel #6
0
func (self *Balanser) Request(client *net.TCPConn) error {
	defer client.Close()

	failed := map[string]struct{}{}
	for {
		if len(failed) == self.count {
			return errors.New("All backends are broken!")
		}

		backend, err := self.router.Choose()
		if err != nil {
			return err
		}

		if _, ok := failed[backend]; ok {
			continue
		}

		conn, err := net.Dial("tcp", backend)
		if err != nil {
			failed[backend] = struct{}{}
			continue
		}

		server := conn.(*net.TCPConn)

		clientClosed := make(chan error, 1)
		go func() {
			_, err := io.Copy(server, client)
			clientClosed <- errors.NewErr(err)
		}()

		serverClosed := make(chan error, 1)
		go func() {
			_, err := io.Copy(client, server)
			serverClosed <- errors.NewErr(err)
		}()

		select {
		case err = <-clientClosed:
			server.SetLinger(0)
			server.CloseRead()
		case err = <-serverClosed:
		}

		return err
	}
}
Beispiel #7
0
func SendJson(w http.ResponseWriter, data interface{}) error {
	w.Header().Set("Content-type", "application/json")
	if err := json.NewEncoder(w).Encode(data); err != nil {
		return errors.NewErr(err)
	}
	return nil
}
Beispiel #8
0
func (self *Downloader) Download(url string) (string, error) {
	log.Printf("Downloader.Download(%s)\n", url)
	resp, err := http.Get(url)
	if err != nil {
		return "", errors.NewErr(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", errors.NewErr(err)
	}

	log.Printf("Downloader.Download(%s) OK!\n", url)
	return string(body), nil
}
Beispiel #9
0
func (self *FileReader) Write(b []byte) (n int, err error) {
	n, err = self.file.Write(b)
	if err == io.EOF {
		return n, err
	}
	return n, errors.NewErr(err)
}
Beispiel #10
0
func main() {
	var help = flag.Bool("help", false, "print help")
	var port = flag.Int("port", -1, "port to listen")
	var gracefulRestart = graceful.SetFlag()
	flag.Parse()

	if *help || *port == -1 {
		flag.PrintDefaults()
		return
	}

	srv := rpc.NewServer()
	srv.Register(&downloader.DownloaderServer{&downloader.Downloader{}})

	server := gjsonrpc.NewServer(srv)
	graceful.SetSighup(server)

	if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
		log.Fatal(errors.NewErr(err))
	}

	if err := graceful.Restart(server); err != nil {
		log.Fatal(err)
	}
}
Beispiel #11
0
func main() {
	var help = flag.Bool("help", false, "print help")
	var port = flag.Int("port", -1, "port to listen")
	var dir = flag.String("dir", "", "data directory")
	var maxFileSize = flag.Int("max-size", 10*1024*1024, "maximum file size")
	var maxTime = flag.Int("max-time", 1*60, "maximum time between sync calls (in seconds)")
	var gracefulRestart = graceful.SetFlag()
	flag.Parse()

	if *help || *port == -1 {
		flag.PrintDefaults()
		return
	}

	gk, err := gatekeeper.NewGatekeeper(*dir, uint64(*maxFileSize), time.Duration(*maxTime)*time.Second)
	if err != nil {
		log.Fatal(err)
	}

	srv := rpc.NewServer()
	srv.Register(&gatekeeper.GatekeeperServer{gk})

	server := gjsonrpc.NewServer(srv)
	graceful.SetSighup(server)

	if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
		log.Fatal(errors.NewErr(err))
	}

	if err := graceful.Restart(server); err != nil {
		log.Fatal(err)
	}
}
Beispiel #12
0
func (self *Caregiver) PushUrls(urls []string) error {
	log.Printf("Caregiver.PushUrls(%#v)\n", urls)
	data := map[string][]string{}
	for _, u := range urls {
		url, err := url.Parse(u)
		if err != nil {
			return errors.NewErr(err)
		}

		path := url.Path
		if len(path) != 0 && path[0] == '/' {
			path = path[1:]
		}
		if url.RawQuery != "" {
			path += "?"
			path += url.RawQuery
		}
		data[url.Host] = append(data[url.Host], path)
	}

	for host, urls := range data {
		self.mutex.Lock()
		hosts, ok := self.hosts[host]
		if !ok {
			hosts = self.getData(host)
			self.hosts[host] = hosts
		}
		self.mutex.Unlock()
		hosts.urls.EnqueueAll(urls...)
	}
	log.Printf("Caregiver.PushUrls(%#v) OK\n", urls)
	return nil
}
Beispiel #13
0
func main() {
	var help = flag.Bool("help", false, "print help")
	var port = flag.Int("port", -1, "port to listen")
	var cacheTime = flag.Int("cachetime", -1, "time to keep resolved records in cache (in seconds)")
	var gracefulRestart = graceful.SetFlag()
	flag.Parse()

	if *help || *port == -1 || *cacheTime == -1 {
		flag.PrintDefaults()
		return
	}

	srv := rpc.NewServer()
	srv.Register(&dns.ResolverServer{dns.NewResolver(time.Duration(*cacheTime) * time.Second)})
	log.Println(srv)

	server := gjsonrpc.NewServer(srv)
	graceful.SetSighup(server)

	if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
		log.Fatal(errors.NewErr(err))
	}

	if err := graceful.Restart(server); err != nil {
		log.Fatal(err)
	}
}
Beispiel #14
0
func NewGatekeeper(dir string, maxFileSize uint64, maxTime time.Duration) (*Gatekeeper, error) {
	self := &Gatekeeper{
		dir:         dir,
		maxTime:     maxTime,
		maxFileSize: maxFileSize,
		fNum:        0,
	}

	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, errors.NewErr(err)
	}

	arr := make(valTArr, len(files))
	for i, f := range files {
		num, err := strconv.Atoi(f.Name())
		if err != nil {
			return nil, errors.NewErr(err)
		}

		arr[i] = valT{
			num:  uint(num),
			file: f,
		}
	}

	sort.Sort(arr)
	counts := map[uint]uint{}
	for _, f := range arr {
		counts[f.num] = 0
		self.fNum = f.num + 1
		err := self.load(self.dir+"/"+f.file.Name(), f.num, counts)
		if err != nil {
			return nil, err
		}
	}

	for k, v := range counts {
		if v == 0 {
			if err := os.Remove(self.dir + "/" + strconv.Itoa(int(k))); err != nil {
				return nil, errors.NewErr(err)
			}
		}
	}

	return self, nil
}
Beispiel #15
0
func (self *CaregiverClient) PullUrls() (map[string]string, error) {
	var res map[string]string
	if err := self.Call("CaregiverServer.PullUrls", struct{}{}, &res); err != nil {
		return nil, errors.NewErr(err)
	}

	return res, nil
}
Beispiel #16
0
func (self *GracefulListener) File() (*os.File, error) {
	f, err := self.base.File()
	if err != nil {
		return nil, errors.NewErr(err)
	}

	return f, nil
}
Beispiel #17
0
func JsonRpcDial(addr string) (*rpc.Client, error) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, errors.NewErr(err)
	}

	return jsonrpc.NewClient(conn), nil
}
Beispiel #18
0
func (self *FileReader) Seek(offset int64, whence int) (ret int64, err error) {
	ret, err = self.file.Seek(offset, whence)
	if err == io.EOF {
		return ret, err
	}

	return ret, errors.NewErr(err)
}
Beispiel #19
0
func (self *DownloaderClient) DownloadAll(urls []string) ([]string, error) {
	var res []string
	if err := self.Call("DownloaderServer.DownloadAll", ArgsAll{Urls: urls}, &res); err != nil {
		return nil, errors.NewErr(err)
	}

	return res, nil
}
Beispiel #20
0
func (self *DownloaderClient) Download(url string) (string, error) {
	var res string
	if err := self.Call("DownloaderServer.Download", Args{Url: url}, &res); err != nil {
		return "", errors.NewErr(err)
	}

	return string(res), nil
}
Beispiel #21
0
func (self *ResolverClient) ResolveAll(hosts []string) ([][]string, error) {
	var res [][]string
	if err := self.Call("ResolverServer.ResolveAll", ArgsAll{Hosts: hosts}, &res); err != nil {
		return nil, errors.NewErr(err)
	}

	return res, nil
}
Beispiel #22
0
func NewDownloaderClient(addr string) (DownloaderClient, error) {
	c, err := util.JsonRpcDial(addr)
	if err != nil {
		return DownloaderClient{}, errors.NewErr(err)
	}

	return DownloaderClient{c}, nil
}
Beispiel #23
0
func NewResolverClient(addr string) (ResolverClient, error) {
	c, err := util.JsonRpcDial(addr)
	if err != nil {
		return ResolverClient{}, errors.NewErr(err)
	}

	return ResolverClient{c}, nil
}
Beispiel #24
0
func GetListener(addr string, graceful bool) (*net.TCPListener, error) {
	var listener net.Listener = nil
	var err error = nil
	if graceful {
		file := os.NewFile(3, "")
		listener, err = net.FileListener(file)
		if err != nil {
			file.Close()
			return nil, errors.NewErr(err)
		}
	} else {
		listener, err = net.Listen("tcp", addr)
		if err != nil {
			return nil, errors.NewErr(err)
		}
	}
	return listener.(*net.TCPListener), nil
}
Beispiel #25
0
func (self *Server) Serve(l *net.TCPListener) error {
	self.listener = graceful.NewGracefulListener(l, self)
	err := self.base.Serve(self.listener)
	self.restart.Wait()
	if err == graceful.NeedRestart {
		return nil
	}
	return errors.NewErr(err)
}
Beispiel #26
0
func main() {
	var help = flag.Bool("help", false, "print help")
	var port = flag.Int("port", -1, "port to listen")
	var dlerArrd = flag.String("dl", "", "downloader address")
	var dnsAddr = flag.String("dns", "", "dns resolver address")
	var defaultMaxCount = flag.Int("maxcount", 1, "default maximum count urls for specific host to download at once")
	var defaultTimeout = flag.Int("timeout", 0, "default timeout between downloads for specific host (im ms)")
	var pullTimeout = flag.Int("pull-timeout", 100, "pull check timeout (im ms)")
	var workTimeout = flag.Int("work-timeout", 1000, "pull check timeout (im ms)")
	var gracefulRestart = graceful.SetFlag()
	flag.Parse()

	if *help || *port == -1 || *dlerArrd == "" {
		flag.PrintDefaults()
		return
	}

	ct, err := caregiver.NewCaregiver(
		*dlerArrd,
		*dnsAddr,
		uint(*defaultMaxCount),
		time.Duration(*defaultTimeout)*time.Millisecond,
		time.Duration(*pullTimeout)*time.Millisecond,
		time.Duration(*workTimeout)*time.Millisecond,
	)
	if err != nil {
		log.Fatal(err)
	}

	srv := rpc.NewServer()
	srv.Register(&caregiver.CaregiverServer{ct})

	server := gjsonrpc.NewServer(srv)
	graceful.SetSighup(server)

	go func() {
		for {
			if err := ct.Start(); err != nil {
				log.Errorln(err)
				time.Sleep(ct.WorkTimeout)
				continue
			}
			break
		}
	}()

	if err := server.ListenAndServe(":"+strconv.Itoa(*port), *gracefulRestart); err != nil {
		log.Fatal(errors.NewErr(err))
	}

	if err := graceful.Restart(server); err != nil {
		log.Fatal(err)
	}
}
Beispiel #27
0
func Restart(srv GServer) error {
	f, err := srv.Listener().File()
	if err != nil {
		return errors.NewErr(err)
	}

	newargs := make([]string, len(os.Args)+1)
	for i, v := range os.Args {
		newargs[i] = v
	}
	newargs[len(os.Args)] = "-graceful"

	_, err = syscall.ForkExec(os.Args[0], newargs, &syscall.ProcAttr{
		Env:   os.Environ(),
		Files: []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd(), f.Fd()},
	})
	if err != nil {
		return errors.NewErr(err)
	}
	return nil
}
Beispiel #28
0
func Open(file string) (*FileReader, error) {
	f, err := os.Open(file)
	if err != nil {
		return nil, errors.NewErr(err)
	}

	return &FileReader{
		file:   f,
		offset: 0,
		buf:    []byte{0},
	}, nil
}
Beispiel #29
0
func GetParamInt(r *http.Request, name string) (int, error) {
	sval, err := GetParam(r, name)
	if err != nil {
		return -1, err
	}

	val, err := strconv.Atoi(sval)
	if err != nil {
		return -1, errors.NewErr(err)
	}

	return val, nil
}
Beispiel #30
0
func UrlTransform(key string) (string, error) {
	u, err := url.Parse(string(key))
	if err != nil {
		return "", errors.NewErr(err)
	}

	arr := strings.Split(u.Host, ".")
	for i, j := 0, len(arr)-1; i < j; i, j = i+1, j-1 {
		arr[i], arr[j] = arr[j], arr[i]
	}

	u.Host = strings.Join(arr, ".")
	return u.String(), nil
}