예제 #1
0
파일: ring.go 프로젝트: dadgar/onecache
// newConsistentRing returns a constinent ring and takes the name of this node
// and the number of its replicas.
func newConsistentRing(name string, replicas int) *consistentRing {
	return &consistentRing{
		self:     name,
		ring:     consistent.New(),
		replicas: replicas,
	}
}
예제 #2
0
func main() {
	var bindAddress string
	var port int

	flag.IntVar(&port, "port", 9125, "Port to listen on")
	flag.IntVar(&port, "p", 9125, "Port to listen on")

	flag.StringVar(&bindAddress, "bind", "0.0.0.0", "IP Address to listen on")
	flag.StringVar(&bindAddress, "b", "0.0.0.0", "IP Address to listen on")

	flag.StringVar(&prefix, "prefix", "statsrelay", "The prefix to use with self generated stats")

	flag.BoolVar(&verbose, "verbose", false, "Verbose output")
	flag.BoolVar(&verbose, "v", false, "Verbose output")

	flag.Parse()

	if len(flag.Args()) == 0 {
		log.Fatalf("One or most host specifications are needed to locate statsd daemons.\n")
	}

	hashRing = consistent.New()
	hashRing.NumberOfReplicas = 1

	for _, v := range flag.Args() {
		var addr *net.UDPAddr
		var err error
		host := strings.Split(v, ":")

		switch len(host) {
		case 1:
			log.Printf("Invalid statsd location: %s\n", v)
			log.Fatalf("Must be of the form HOST:PORT or HOST:PORT:INSTANCE\n")
		case 2:
			addr, err = net.ResolveUDPAddr("udp", v)
			if err != nil {
				log.Printf("Error parsing HOST:PORT \"%s\"\n", v)
				log.Fatalf("%s\n", err.Error())
			}
		case 3:
			addr, err = net.ResolveUDPAddr("udp", host[0]+":"+host[1])
			if err != nil {
				log.Printf("Error parsing HOST:PORT:INSTANCE \"%s\"\n", v)
				log.Fatalf("%s\n", err.Error())
			}
		default:
			log.Fatalf("Unrecongnized host specification: %s\n", v)
		}

		if addr != nil {
			udpAddr[v] = addr
			hashRing.Add(v)
		}
	}

	epochTime = time.Now().Unix()
	runServer(bindAddress, port)

	log.Printf("Normal shutdown.\n")
}
예제 #3
0
func ExampleAdd(rw http.ResponseWriter, req *http.Request, p httprouter.Params) {
	c := consistent.New()
	c.Add("http://localhost:3000/keys")
	c.Add("http://localhost:3001/keys")
	c.Add("http://localhost:3002/keys")
	//users := []string{"user_mcnulty", "user_bunk", "user_omar", "user_bunny", "user_stringer"}
	var vars Cache
	vars.Key = p.ByName("name")
	vars.Value = p.ByName("names")
	//fmt.Println("initial state [A, B, C]")
	//	for _, u := range vars {
	server, err := c.Get(vars.Key)
	if err != nil {
		log.Fatal(err)
	}

	//fmt.Printf("%s => %s\n", vars.Key, server)
	var Url string
	Url = server + "/" + vars.Key + "/" + vars.Value
	//	fmt.Println(Url)
	req, err = http.NewRequest("PUT", Url, nil)
	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		fmt.Println("Error Server Unavailable")
		//	log.Fatal(err)
	}

	rw.WriteHeader(http.StatusOK)

}
예제 #4
0
파일: backend.go 프로젝트: yubo/falcon
func (p *Backend) Init() error {
	glog.V(3).Infof(MODULE_NAME+"%s Init()", p.Params.Name)
	// core

	//cache
	p.cacheInit()

	// rpc
	p.rpcConnects = connList{list: list.New()}

	// http
	p.httpMux = http.NewServeMux()
	p.httpRoutes()

	// rrdtool/sync_disk/migrate
	p.storageNetTaskCh = make(map[string]chan *netTask)
	p.storageMigrateClients = make(map[string][]*rpc.Client)
	p.storageMigrateConsistent = consistent.New()

	// store
	size := CACHE_TIME / FLUSH_DISK_STEP
	if size < 0 {
		glog.Fatalf(MODULE_NAME+"store.init, bad size %d\n", size)
	}
	p.status = specs.APP_STATUS_INIT
	return nil

}
예제 #5
0
파일: scheduler.go 프로젝트: yubo/falcon
func newSchedConsistent() *schedConsistent {
	sched := &schedConsistent{
		name:    "consistent",
		consist: consistent.New(),
		chans:   make(map[string]chan *specs.MetaData),
	}
	sched.consist.NumberOfReplicas = specs.REPLICAS
	return sched
}
예제 #6
0
파일: proxy.go 프로젝트: shawnsi/cacheproxy
func New(replicas *int) *CacheProxy {
	c := new(CacheProxy)
	c.backends = consistent.New()
	c.replicas = replicas

	// Metric registry
	c.registry = metrics.NewRegistry()

	// Meters
	c.hits = metrics.NewMeter()
	c.misses = metrics.NewMeter()
	c.requests = metrics.NewMeter()
	c.registry.Register("hits", c.hits)
	c.registry.Register("misses", c.misses)
	c.registry.Register("requests", c.requests)

	// Backend response timer
	c.timer = metrics.NewTimer()
	c.registry.Register("backend", c.timer)

	// RESTful interface for cache members
	c.manager = http.NewServeMux()
	c.manager.HandleFunc("/members/", func(w http.ResponseWriter, req *http.Request) {
		backend := path.Base(req.URL.Path)
		switch {
		case req.Method == "PUT":
			// Adds "member" as a cache member
			log.Println("Adding backend: " + backend)
			c.backends.Add(backend)
		case req.Method == "DELETE":
			// Deletes "member" from cache ring
			log.Println("Removing backend: " + backend)
			c.backends.Remove(backend)
		}
		// Always returns a list of current cache members
		io.WriteString(w, strings.Join(c.backends.Members(), ","))
	})

	// RESTful interface for metrics
	c.manager.HandleFunc("/metrics/", func(w http.ResponseWriter, req *http.Request) {
		metrics.WriteOnce(c.registry, w)
	})

	// Reverse proxy that selects the backend by nearest match to the request URL
	// on the consistent hash ring.
	c.proxy = &httputil.ReverseProxy{
		Director: func(req *http.Request) {
			req.URL.Scheme = "http"
			req.URL.Host = c.router
			req.Header.Set("X-Backends", strings.Join(c.Backends(req), ","))
		},
		Transport: &MeteredTransport{cacheproxy: c},
	}

	return c
}
예제 #7
0
func PutUser(w http.ResponseWriter, r *http.Request, p httprouter.Params) {

	keyid := p.ByName("key_id")
	Val := p.ByName("value")

	Keyidint, _ := strconv.Atoi(keyid)
	MClient[Keyidint] = Val

	cons := consistent.New()

	cons.Add("127.0.0.1:3000")
	cons.Add("127.0.0.1:3001")
	cons.Add("127.0.0.1:3002")

	fmt.Println("Servers added")

	//Find a cache server for a user:

	server, err := cons.Get(Val)
	if err != nil {
	}
	fmt.Println("server:", server)

	s := []string{}
	s = append(s, "http://")
	s = append(s, server)
	s = append(s, "/")
	s = append(s, "keys")
	s = append(s, "/")
	s = append(s, keyid)
	s = append(s, "/")
	s = append(s, Val)

	url = s[0] + s[1] + s[2] + s[3] + s[4] + s[5] + s[6] + s[7]

	var urlString = []byte(url)

	req, err := http.NewRequest("PUT", url, bytes.NewBuffer(urlString))

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	fmt.Println("response Status:", resp.Status)
	fmt.Println("response Headers:", resp.Header)
	body, _ := ioutil.ReadAll(resp.Body)
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(201)
	fmt.Fprintf(w, "%s", string(body))

}
예제 #8
0
func initBackendSvrs(svrs []string) {
	pConsisthash = consistent.New()
	pBackendSvrs = make(map[string]*BackendSvr)

	for _, svr := range svrs {
		pConsisthash.Add(svr)
		pBackendSvrs[svr] = &BackendSvr{
			svrStr:    svr,
			isUp:      true,
			failTimes: 0,
		}
	}
	go checkBackendSvrs()
}
예제 #9
0
파일: ring.go 프로젝트: dadgar/onecache
// ringCopy returns a copy of the consistentRing.
func (c *consistentRing) ringCopy() ring {
	c.lock.Lock()
	defer c.lock.Unlock()

	ring := &consistentRing{
		self:     c.self,
		ring:     consistent.New(),
		replicas: c.replicas,
	}

	m := c.ring.Members()
	ring.ring.Set(m)
	return ring
}
예제 #10
0
func ExampleGet(rw http.ResponseWriter, req *http.Request, p httprouter.Params) {
	c := consistent.New()
	c.Add("http://localhost:3000/keys")
	c.Add("http://localhost:3001/keys")
	c.Add("http://localhost:3002/keys")
	//users := []string{"user_mcnulty", "user_bunk", "user_omar", "user_bunny", "user_stringer"}
	var vars Cache
	vars.Key = p.ByName("name")
	//vars.Value = p.ByName("names")
	//fmt.Println("initial state [A, B, C]")
	//	for _, u := range vars {
	server, err := c.Get(vars.Key)
	if err != nil {
		log.Fatal(err)
	}

	//fmt.Printf("%s => %s\n", vars.Key, server)
	var Url string
	Url = server + "/" + vars.Key
	//	fmt.Println(Url)
	req, err = http.NewRequest("GET", Url, nil)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Error in Request")
		//	log.Fatal(err)
	}

	// var dat map[string]interface{}
	var Result Cache
	var Results Caches

	//  var m map[string]interface{}
	body, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()

	if err := json.Unmarshal(body, &Results); err != nil {
		panic(err)
	}
	rw.WriteHeader(http.StatusOK)
	Result.Key = strconv.Itoa(Results.Key)
	Result.Value = Results.Value

	mapB, _ := json.Marshal(Result)
	fmt.Fprintf(rw, string(mapB))

}
func main() {
	Init()
	//	hostname,_ =os.Hostname()
	//	servers=make(map[string]string)
	//	servers["clive"]=ip
	//	servers["clive0"]="7.7.7.1"
	//	servers["clive1"]="7.7.7.7"
	//	servers["clive2"]="7.7.7.9"
	//	for k,_ :=range servers{
	//		c.Add(k)
	//	}
	client.ConsistentHash = consistent.New()
	client.ConsistentHash.Add("7.7.7.1")
	client.ConsistentHash.Add("7.7.7.7")
	conn1, _ := net.Dial("tcp", "7.7.7.1:7777")
	conn2, _ := net.Dial("tcp", "7.7.7.7:7777")
	client.Conns["7.7.7.1"] = conn1
	client.Conns["7.7.7.7"] = conn2
	client.Conns["localhost"] = client.Conns[ip]
	fmt.Println(flag.Args())
	if flag.Args()[1] == "server" {
		if flag.Args()[0] == "start" {
			StartServer()
		} else if flag.Args()[0] == "stop" {
			//quit <- 0
		}
	}

	//client.TestClient()
	//load Graph, why so slow
	if flag.Args()[1] == "loadGraph" {
		time.Sleep(100 * time.Millisecond)
		fmt.Println("loadGraph start: " + time.Now().Format("2006-01-02 15:04:05"))
		client.LoadWebGraph()
		fmt.Println("loadGraph end: " + time.Now().Format("2006-01-02 15:04:05"))
	}
	if flag.Args()[1] == "pageRank" {
		fmt.Println("pagerank start: " + time.Now().Format("2006-01-02 15:04:05"))
		//	client.PageRank()
		fmt.Println("pagerank end: " + time.Now().Format("2006-01-02 15:04:05"))
	}
	if flag.Args()[1] == "test" {
		client.TestClientProto()
	}
}
예제 #12
0
func main() {
	Init()
	client.ConsistentHash = consistent.New()
	client.ConsistentHash.Add("7.7.7.1")
	client.ConsistentHash.Add("7.7.7.7")
	conn1, _ := net.Dial("tcp", "7.7.7.1:7777")
	conn2, _ := net.Dial("tcp", "7.7.7.7:7777")
	client.Conns["7.7.7.1"] = conn1
	client.Conns["7.7.7.7"] = conn2
	client.Conns["localhost"] = client.Conns[ip]

	fmt.Println(flag.Args())
	if flag.Args()[1] == "master" {
		StartServer()
	}
	if flag.Args()[1] == "worker" {
		StartWorker()
	}

}
// serviceName is like "/services/busybox"
// endPoints is an array of "http://<etcd client ip:port>"
func (service *ConsistentService) Connect(serviceName string, endPoints []string) error {
	if service.connected {
		log.Printf("Can't connected twice")
		return errors.New("math: square root of negative number")
	}

	service.consis = consistent.New()

	cfg := client.Config{
		Endpoints:               endPoints,
		Transport:               client.DefaultTransport,
		HeaderTimeoutPerRequest: time.Second,
	}

	var err error
	service.etcdClient, err = client.New(cfg)
	if err != nil {
		return err
	}
	kapi := client.NewKeysAPI(service.etcdClient)

	resp, err := kapi.Get(context.Background(), serviceName, nil)
	if err != nil {
		return err
	} else {
		if resp.Node.Dir {
			for _, peer := range resp.Node.Nodes {
				n := peer.Value
				service.consis.Add(n)
			}
		}
	}

	watcher := kapi.Watcher(serviceName, &client.WatcherOptions{Recursive: true})
	go service.watch(watcher)
	service.connected = true
	return nil
}
예제 #14
0
func New(localNode string, allNodes []string) (conrd *consistentRoad, err error) {
	conrd = &consistentRoad{allNodes: make(map[string]*node)}

	conrd.hashRing = consistent.New()
	conrd.hashRing.NumberOfReplicas = NumRingReplicas

	conrd.localNode, err = newNode(localNode)
	if err != nil {
		return nil, err
	}
	conrd.hashRing.Add(conrd.localNode.hashId)

	for _, s := range allNodes {
		n, err := newNode(s)
		if err != nil {
			return nil, err
		}
		conrd.hashRing.Add(n.hashId)
		conrd.allNodes[n.hashId] = n
	}

	conrd.httpClient = &http.Client{
		Timeout: NodeReqTimeout,
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   NodeConnTimeout,
				KeepAlive: NodeConnKeepAlive,
			}).Dial,
			TLSHandshakeTimeout: NodeReqTimeout,
			MaxIdleConnsPerHost: 30,
			DisableCompression:  true,
			DisableKeepAlives:   false,
		},
	}

	return conrd, nil
}
예제 #15
0
		"9575CD": &color.RGBA{149, 117, 205, 255},
		"B0855E": &color.RGBA{176, 133, 94, 255},
		"F06292": &color.RGBA{240, 98, 146, 255},
		"A3D36C": &color.RGBA{163, 211, 108, 255},
		"7986CB": &color.RGBA{121, 134, 203, 255},
		"F1B91D": &color.RGBA{241, 185, 29, 255},
	}

	defaultColorKey = "45BDF3"

	// ErrUnsupportChar is returned when the character is not supported
	ErrUnsupportChar = errors.New("unsupported character")

	// ErrUnsupportedEncoding is returned when the given image encoding is not supported
	ErrUnsupportedEncoding = errors.New("avatar: Unsuppored encoding")
	c                      = consistent.New()
)

// InitialsAvatar represents an initials avatar.
type InitialsAvatar struct {
	drawer *drawer
	cache  *lru.Cache
}

// New creates an instance of InitialsAvatar
func New(fontFile string) *InitialsAvatar {
	avatar := NewWithConfig(Config{
		MaxItems: 1024, // default to 1024 items.
		FontFile: fontFile,
	})
	return avatar
예제 #16
0
파일: migrate.go 프로젝트: modeyang/graph
func init() {
	Consistent = consistent.New()
	Net_task_ch = make(map[string]chan *Net_task_t)
	clients = make(map[string][]*rpc.Client)
}
예제 #17
0
func newConsistentHashNodesRing(numberOfReplicas int, nodes []string) *ConsistentHashNodeRing {
	ret := &ConsistentHashNodeRing{ring: consistent.New()}
	ret.SetNumberOfReplicas(numberOfReplicas)
	ret.SetNodes(nodes)
	return ret
}