// newConsistentRing returns a constinent ring and takes the name of this node
// and the number of its replicas.
func newConsistentRing(name string, replicas int) *consistentRing {
return &consistentRing{
self: name,
ring: consistent.New(),
replicas: replicas,
}
}
func main() {
var bindAddress string
var port int
flag.IntVar(&port, "port", 9125, "Port to listen on")
flag.IntVar(&port, "p", 9125, "Port to listen on")
flag.StringVar(&bindAddress, "bind", "0.0.0.0", "IP Address to listen on")
flag.StringVar(&bindAddress, "b", "0.0.0.0", "IP Address to listen on")
flag.StringVar(&prefix, "prefix", "statsrelay", "The prefix to use with self generated stats")
flag.BoolVar(&verbose, "verbose", false, "Verbose output")
flag.BoolVar(&verbose, "v", false, "Verbose output")
flag.Parse()
if len(flag.Args()) == 0 {
log.Fatalf("One or most host specifications are needed to locate statsd daemons.\n")
}
hashRing = consistent.New()
hashRing.NumberOfReplicas = 1
for _, v := range flag.Args() {
var addr *net.UDPAddr
var err error
host := strings.Split(v, ":")
switch len(host) {
case 1:
log.Printf("Invalid statsd location: %s\n", v)
log.Fatalf("Must be of the form HOST:PORT or HOST:PORT:INSTANCE\n")
case 2:
addr, err = net.ResolveUDPAddr("udp", v)
if err != nil {
log.Printf("Error parsing HOST:PORT \"%s\"\n", v)
log.Fatalf("%s\n", err.Error())
}
case 3:
addr, err = net.ResolveUDPAddr("udp", host[0]+":"+host[1])
if err != nil {
log.Printf("Error parsing HOST:PORT:INSTANCE \"%s\"\n", v)
log.Fatalf("%s\n", err.Error())
}
default:
log.Fatalf("Unrecongnized host specification: %s\n", v)
}
if addr != nil {
udpAddr[v] = addr
hashRing.Add(v)
}
}
epochTime = time.Now().Unix()
runServer(bindAddress, port)
log.Printf("Normal shutdown.\n")
}
func ExampleAdd(rw http.ResponseWriter, req *http.Request, p httprouter.Params) {
c := consistent.New()
c.Add("http://localhost:3000/keys")
c.Add("http://localhost:3001/keys")
c.Add("http://localhost:3002/keys")
//users := []string{"user_mcnulty", "user_bunk", "user_omar", "user_bunny", "user_stringer"}
var vars Cache
vars.Key = p.ByName("name")
vars.Value = p.ByName("names")
//fmt.Println("initial state [A, B, C]")
// for _, u := range vars {
server, err := c.Get(vars.Key)
if err != nil {
log.Fatal(err)
}
//fmt.Printf("%s => %s\n", vars.Key, server)
var Url string
Url = server + "/" + vars.Key + "/" + vars.Value
// fmt.Println(Url)
req, err = http.NewRequest("PUT", Url, nil)
client := &http.Client{}
_, err = client.Do(req)
if err != nil {
fmt.Println("Error Server Unavailable")
// log.Fatal(err)
}
rw.WriteHeader(http.StatusOK)
}
func (p *Backend) Init() error {
glog.V(3).Infof(MODULE_NAME+"%s Init()", p.Params.Name)
// core
//cache
p.cacheInit()
// rpc
p.rpcConnects = connList{list: list.New()}
// http
p.httpMux = http.NewServeMux()
p.httpRoutes()
// rrdtool/sync_disk/migrate
p.storageNetTaskCh = make(map[string]chan *netTask)
p.storageMigrateClients = make(map[string][]*rpc.Client)
p.storageMigrateConsistent = consistent.New()
// store
size := CACHE_TIME / FLUSH_DISK_STEP
if size < 0 {
glog.Fatalf(MODULE_NAME+"store.init, bad size %d\n", size)
}
p.status = specs.APP_STATUS_INIT
return nil
}
func newSchedConsistent() *schedConsistent {
sched := &schedConsistent{
name: "consistent",
consist: consistent.New(),
chans: make(map[string]chan *specs.MetaData),
}
sched.consist.NumberOfReplicas = specs.REPLICAS
return sched
}
func New(replicas *int) *CacheProxy {
c := new(CacheProxy)
c.backends = consistent.New()
c.replicas = replicas
// Metric registry
c.registry = metrics.NewRegistry()
// Meters
c.hits = metrics.NewMeter()
c.misses = metrics.NewMeter()
c.requests = metrics.NewMeter()
c.registry.Register("hits", c.hits)
c.registry.Register("misses", c.misses)
c.registry.Register("requests", c.requests)
// Backend response timer
c.timer = metrics.NewTimer()
c.registry.Register("backend", c.timer)
// RESTful interface for cache members
c.manager = http.NewServeMux()
c.manager.HandleFunc("/members/", func(w http.ResponseWriter, req *http.Request) {
backend := path.Base(req.URL.Path)
switch {
case req.Method == "PUT":
// Adds "member" as a cache member
log.Println("Adding backend: " + backend)
c.backends.Add(backend)
case req.Method == "DELETE":
// Deletes "member" from cache ring
log.Println("Removing backend: " + backend)
c.backends.Remove(backend)
}
// Always returns a list of current cache members
io.WriteString(w, strings.Join(c.backends.Members(), ","))
})
// RESTful interface for metrics
c.manager.HandleFunc("/metrics/", func(w http.ResponseWriter, req *http.Request) {
metrics.WriteOnce(c.registry, w)
})
// Reverse proxy that selects the backend by nearest match to the request URL
// on the consistent hash ring.
c.proxy = &httputil.ReverseProxy{
Director: func(req *http.Request) {
req.URL.Scheme = "http"
req.URL.Host = c.router
req.Header.Set("X-Backends", strings.Join(c.Backends(req), ","))
},
Transport: &MeteredTransport{cacheproxy: c},
}
return c
}
func PutUser(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
keyid := p.ByName("key_id")
Val := p.ByName("value")
Keyidint, _ := strconv.Atoi(keyid)
MClient[Keyidint] = Val
cons := consistent.New()
cons.Add("127.0.0.1:3000")
cons.Add("127.0.0.1:3001")
cons.Add("127.0.0.1:3002")
fmt.Println("Servers added")
//Find a cache server for a user:
server, err := cons.Get(Val)
if err != nil {
}
fmt.Println("server:", server)
s := []string{}
s = append(s, "http://")
s = append(s, server)
s = append(s, "/")
s = append(s, "keys")
s = append(s, "/")
s = append(s, keyid)
s = append(s, "/")
s = append(s, Val)
url = s[0] + s[1] + s[2] + s[3] + s[4] + s[5] + s[6] + s[7]
var urlString = []byte(url)
req, err := http.NewRequest("PUT", url, bytes.NewBuffer(urlString))
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
fmt.Println("response Status:", resp.Status)
fmt.Println("response Headers:", resp.Header)
body, _ := ioutil.ReadAll(resp.Body)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(201)
fmt.Fprintf(w, "%s", string(body))
}
func initBackendSvrs(svrs []string) {
pConsisthash = consistent.New()
pBackendSvrs = make(map[string]*BackendSvr)
for _, svr := range svrs {
pConsisthash.Add(svr)
pBackendSvrs[svr] = &BackendSvr{
svrStr: svr,
isUp: true,
failTimes: 0,
}
}
go checkBackendSvrs()
}
// ringCopy returns a copy of the consistentRing.
func (c *consistentRing) ringCopy() ring {
c.lock.Lock()
defer c.lock.Unlock()
ring := &consistentRing{
self: c.self,
ring: consistent.New(),
replicas: c.replicas,
}
m := c.ring.Members()
ring.ring.Set(m)
return ring
}
func ExampleGet(rw http.ResponseWriter, req *http.Request, p httprouter.Params) {
c := consistent.New()
c.Add("http://localhost:3000/keys")
c.Add("http://localhost:3001/keys")
c.Add("http://localhost:3002/keys")
//users := []string{"user_mcnulty", "user_bunk", "user_omar", "user_bunny", "user_stringer"}
var vars Cache
vars.Key = p.ByName("name")
//vars.Value = p.ByName("names")
//fmt.Println("initial state [A, B, C]")
// for _, u := range vars {
server, err := c.Get(vars.Key)
if err != nil {
log.Fatal(err)
}
//fmt.Printf("%s => %s\n", vars.Key, server)
var Url string
Url = server + "/" + vars.Key
// fmt.Println(Url)
req, err = http.NewRequest("GET", Url, nil)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println("Error in Request")
// log.Fatal(err)
}
// var dat map[string]interface{}
var Result Cache
var Results Caches
// var m map[string]interface{}
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err := json.Unmarshal(body, &Results); err != nil {
panic(err)
}
rw.WriteHeader(http.StatusOK)
Result.Key = strconv.Itoa(Results.Key)
Result.Value = Results.Value
mapB, _ := json.Marshal(Result)
fmt.Fprintf(rw, string(mapB))
}
func main() {
Init()
// hostname,_ =os.Hostname()
// servers=make(map[string]string)
// servers["clive"]=ip
// servers["clive0"]="7.7.7.1"
// servers["clive1"]="7.7.7.7"
// servers["clive2"]="7.7.7.9"
// for k,_ :=range servers{
// c.Add(k)
// }
client.ConsistentHash = consistent.New()
client.ConsistentHash.Add("7.7.7.1")
client.ConsistentHash.Add("7.7.7.7")
conn1, _ := net.Dial("tcp", "7.7.7.1:7777")
conn2, _ := net.Dial("tcp", "7.7.7.7:7777")
client.Conns["7.7.7.1"] = conn1
client.Conns["7.7.7.7"] = conn2
client.Conns["localhost"] = client.Conns[ip]
fmt.Println(flag.Args())
if flag.Args()[1] == "server" {
if flag.Args()[0] == "start" {
StartServer()
} else if flag.Args()[0] == "stop" {
//quit <- 0
}
}
//client.TestClient()
//load Graph, why so slow
if flag.Args()[1] == "loadGraph" {
time.Sleep(100 * time.Millisecond)
fmt.Println("loadGraph start: " + time.Now().Format("2006-01-02 15:04:05"))
client.LoadWebGraph()
fmt.Println("loadGraph end: " + time.Now().Format("2006-01-02 15:04:05"))
}
if flag.Args()[1] == "pageRank" {
fmt.Println("pagerank start: " + time.Now().Format("2006-01-02 15:04:05"))
// client.PageRank()
fmt.Println("pagerank end: " + time.Now().Format("2006-01-02 15:04:05"))
}
if flag.Args()[1] == "test" {
client.TestClientProto()
}
}
func main() {
Init()
client.ConsistentHash = consistent.New()
client.ConsistentHash.Add("7.7.7.1")
client.ConsistentHash.Add("7.7.7.7")
conn1, _ := net.Dial("tcp", "7.7.7.1:7777")
conn2, _ := net.Dial("tcp", "7.7.7.7:7777")
client.Conns["7.7.7.1"] = conn1
client.Conns["7.7.7.7"] = conn2
client.Conns["localhost"] = client.Conns[ip]
fmt.Println(flag.Args())
if flag.Args()[1] == "master" {
StartServer()
}
if flag.Args()[1] == "worker" {
StartWorker()
}
}
// serviceName is like "/services/busybox"
// endPoints is an array of "http://<etcd client ip:port>"
func (service *ConsistentService) Connect(serviceName string, endPoints []string) error {
if service.connected {
log.Printf("Can't connected twice")
return errors.New("math: square root of negative number")
}
service.consis = consistent.New()
cfg := client.Config{
Endpoints: endPoints,
Transport: client.DefaultTransport,
HeaderTimeoutPerRequest: time.Second,
}
var err error
service.etcdClient, err = client.New(cfg)
if err != nil {
return err
}
kapi := client.NewKeysAPI(service.etcdClient)
resp, err := kapi.Get(context.Background(), serviceName, nil)
if err != nil {
return err
} else {
if resp.Node.Dir {
for _, peer := range resp.Node.Nodes {
n := peer.Value
service.consis.Add(n)
}
}
}
watcher := kapi.Watcher(serviceName, &client.WatcherOptions{Recursive: true})
go service.watch(watcher)
service.connected = true
return nil
}
func New(localNode string, allNodes []string) (conrd *consistentRoad, err error) {
conrd = &consistentRoad{allNodes: make(map[string]*node)}
conrd.hashRing = consistent.New()
conrd.hashRing.NumberOfReplicas = NumRingReplicas
conrd.localNode, err = newNode(localNode)
if err != nil {
return nil, err
}
conrd.hashRing.Add(conrd.localNode.hashId)
for _, s := range allNodes {
n, err := newNode(s)
if err != nil {
return nil, err
}
conrd.hashRing.Add(n.hashId)
conrd.allNodes[n.hashId] = n
}
conrd.httpClient = &http.Client{
Timeout: NodeReqTimeout,
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: NodeConnTimeout,
KeepAlive: NodeConnKeepAlive,
}).Dial,
TLSHandshakeTimeout: NodeReqTimeout,
MaxIdleConnsPerHost: 30,
DisableCompression: true,
DisableKeepAlives: false,
},
}
return conrd, nil
}
"9575CD": &color.RGBA{149, 117, 205, 255},
"B0855E": &color.RGBA{176, 133, 94, 255},
"F06292": &color.RGBA{240, 98, 146, 255},
"A3D36C": &color.RGBA{163, 211, 108, 255},
"7986CB": &color.RGBA{121, 134, 203, 255},
"F1B91D": &color.RGBA{241, 185, 29, 255},
}
defaultColorKey = "45BDF3"
// ErrUnsupportChar is returned when the character is not supported
ErrUnsupportChar = errors.New("unsupported character")
// ErrUnsupportedEncoding is returned when the given image encoding is not supported
ErrUnsupportedEncoding = errors.New("avatar: Unsuppored encoding")
c = consistent.New()
)
// InitialsAvatar represents an initials avatar.
type InitialsAvatar struct {
drawer *drawer
cache *lru.Cache
}
// New creates an instance of InitialsAvatar
func New(fontFile string) *InitialsAvatar {
avatar := NewWithConfig(Config{
MaxItems: 1024, // default to 1024 items.
FontFile: fontFile,
})
return avatar
func init() {
Consistent = consistent.New()
Net_task_ch = make(map[string]chan *Net_task_t)
clients = make(map[string][]*rpc.Client)
}
func newConsistentHashNodesRing(numberOfReplicas int, nodes []string) *ConsistentHashNodeRing {
ret := &ConsistentHashNodeRing{ring: consistent.New()}
ret.SetNumberOfReplicas(numberOfReplicas)
ret.SetNodes(nodes)
return ret
}
