// create eureka service registries in each zone
func CreateEureka() {
// setup name service and cross zone replication links
znames := archaius.Conf.ZoneNames
Create("eureka", EurekaPkg, archaius.Conf.Regions, 3)
for n, ch := range eurekachan {
var n1, n2 string
switch names.Zone(n) {
case znames[0]:
n1 = znames[1]
n2 = znames[2]
case znames[1]:
n1 = znames[0]
n2 = znames[2]
case znames[2]:
n1 = znames[0]
n2 = znames[1]
}
for nn, cch := range eurekachan {
if names.Region(nn) == names.Region(n) && (names.Zone(nn) == n1 || names.Zone(nn) == n2) {
//log.Println("Eureka cross connect from: " + n + " to " + nn)
gotocol.Send(ch, gotocol.Message{gotocol.NameDrop, cch, time.Now(), gotocol.NilContext, nn})
}
}
}
}
func Lookup(name string) string {
ip := mapped[name]
if ip != "" {
return ip
}
// find indexes for matching zone and region in the config
r := names.Region(name)
ri := 0
z := names.Zone(name)
zi := 0
for i, rr := range archaius.Conf.RegionNames {
if rr == r {
ri = i
break
}
}
for i, zr := range archaius.Conf.ZoneNames {
if zr == z {
zi = i
break
}
}
// increment first to avoid IP 0.0 and get the node counter in the region/zone
allocated[ri][zi]++
node := allocated[ri][zi]
// format as xxx.xxx.xxx.xxx
addr := fmt.Sprintf("%v%v.%v", archaius.Conf.IPRanges[ri][zi], node/256, node%256)
mapped[name] = addr
return addr
}
// Start store, all configuration and state is sent via messages
func Start(listener chan gotocol.Message) {
dunbar := 30 // starting point for how many nodes to remember
// remember the channel to talk to microservices
microservices := make(map[string]chan gotocol.Message, dunbar)
dependencies := make(map[string]time.Time, dunbar) // dependent services and time last updated
store := make(map[string]string, 4) // key value store
store["why?"] = "because..."
var netflixoss, requestor chan gotocol.Message // remember creator and how to talk back to incoming requests
var name string // remember my name
eureka := make(map[string]chan gotocol.Message, 3) // service registry per zone
var chatrate time.Duration
hist := collect.NewHist("")
ep, _ := time.ParseDuration(archaius.Conf.EurekaPoll)
eurekaTicker := time.NewTicker(ep)
chatTicker := time.NewTicker(time.Hour)
chatTicker.Stop()
for {
select {
case msg := <-listener:
collect.Measure(hist, time.Since(msg.Sent))
if archaius.Conf.Msglog {
log.Printf("%v: %v\n", name, msg)
}
switch msg.Imposition {
case gotocol.Hello:
if name == "" {
// if I don't have a name yet remember what I've been named
netflixoss = msg.ResponseChan // remember how to talk to my namer
name = msg.Intention // message body is my name
hist = collect.NewHist(name)
}
case gotocol.Inform:
eureka[msg.Intention] = gotocol.InformHandler(msg, name, listener)
case gotocol.NameDrop: // cross zone = true
gotocol.NameDropHandler(&dependencies, µservices, msg, name, listener, eureka, true)
case gotocol.Forget:
// forget a buddy
gotocol.ForgetHandler(&dependencies, µservices, msg)
case gotocol.Chat:
// setup the ticker to run at the specified rate
d, e := time.ParseDuration(msg.Intention)
if e == nil && d >= time.Millisecond && d <= time.Hour {
chatrate = d
chatTicker = time.NewTicker(chatrate)
}
case gotocol.GetRequest:
// return any stored value for this key (Cassandra READ.ONE behavior)
gotocol.Message{gotocol.GetResponse, listener, time.Now(), msg.Ctx.NewSpan(), store[msg.Intention]}.GoSend(msg.ResponseChan)
case gotocol.GetResponse:
// return path from a request, send payload back up (not currently used)
if requestor != nil {
gotocol.Message{gotocol.GetResponse, listener, time.Now(), msg.Ctx.NewSpan(), msg.Intention}.GoSend(requestor)
}
case gotocol.Put:
requestor = msg.ResponseChan
// set a key value pair and replicate globally
var key, value string
fmt.Sscanf(msg.Intention, "%s%s", &key, &value)
if key != "" && value != "" {
store[key] = value
// duplicate the request on to all connected store nodes
if len(microservices) > 0 {
// replicate request
for _, c := range microservices {
gotocol.Message{gotocol.Replicate, listener, time.Now(), msg.Ctx.NewSpan(), msg.Intention}.GoSend(c)
}
}
}
case gotocol.Replicate:
// Replicate is only used between store nodes
// end point for a request
var key, value string
fmt.Sscanf(msg.Intention, "%s%s", &key, &value)
// log.Printf("store: %v:%v", key, value)
if key != "" && value != "" {
store[key] = value
}
// name looks like: netflixoss.us-east-1.zoneC.cassTurtle.store.cassTurtle11
myregion := names.Region(name)
//log.Printf("%v: %v\n", name, myregion)
// find if this was a cross region Replicate
for n, c := range microservices {
// find the name matching incoming request channel
if c == msg.ResponseChan {
if myregion != names.Region(n) {
// Replicate from out of region needs to be Replicated only to other zones in this Region
for nz, cz := range microservices {
if myregion == names.Region(nz) {
//log.Printf("%v rep to: %v\n", name, nz)
gotocol.Message{gotocol.Replicate, listener, time.Now(), msg.Ctx.NewSpan(), msg.Intention}.GoSend(cz)
}
}
}
}
}
case gotocol.Goodbye:
if archaius.Conf.Msglog {
log.Printf("%v: Going away, zone: %v\n", name, store["zone"])
}
gotocol.Message{gotocol.Goodbye, nil, time.Now(), gotocol.NilContext, name}.GoSend(netflixoss)
return
}
case <-eurekaTicker.C: // check to see if any new dependencies have appeared
for dep, _ := range dependencies {
for _, ch := range eureka {
ch <- gotocol.Message{gotocol.GetRequest, listener, time.Now(), gotocol.NilContext, dep}
}
}
case <-chatTicker.C:
// nothing to do here at the moment
}
}
}
// Start priamCassandra, all configuration and state is sent via messages
func Start(listener chan gotocol.Message) {
dunbar := archaius.Conf.Population // starting point for how many nodes to remember
// remember the channel to talk to microservices
microservices := make(map[string]chan gotocol.Message, dunbar)
// track the hash values owned by each node in the ring
var ring ByToken
dependencies := make(map[string]time.Time, dunbar) // dependent services and time last updated
store := make(map[string]string, 4) // key value store
store["why?"] = "because..."
var parent chan gotocol.Message // remember how to talk back to creator
var name string // remember my name
eureka := make(map[string]chan gotocol.Message, 3*archaius.Conf.Regions) // service registry per zone and region
hist := collect.NewHist("")
ep, _ := time.ParseDuration(archaius.Conf.EurekaPoll)
eurekaTicker := time.NewTicker(ep)
for {
select {
case msg := <-listener:
collect.Measure(hist, time.Since(msg.Sent))
if archaius.Conf.Msglog {
log.Printf("%v: %v\n", name, msg)
}
switch msg.Imposition {
case gotocol.Hello:
if name == "" {
// if I don't have a name yet remember what I've been named
parent = msg.ResponseChan // remember how to talk to my namer
name = msg.Intention // message body is my name
hist = collect.NewHist(name)
}
case gotocol.Inform:
eureka[msg.Intention] = gotocol.InformHandler(msg, name, listener)
case gotocol.NameDrop: // cross zone = true
gotocol.NameDropHandler(&dependencies, µservices, msg, name, listener, eureka, true)
case gotocol.Forget:
// forget a buddy
gotocol.ForgetHandler(&dependencies, µservices, msg)
case gotocol.Chat:
// Gossip setup notification of hash values for nodes, cass1:123,cass2:456
ring = RingConfig(msg.Intention)
case gotocol.GetRequest:
// see if the data is stored on this node
i := ring.Find(ringHash(msg.Intention))
//log.Printf("%v: %v %v\n", name, i, ringHash(msg.Intention))
span := msg.Ctx.NewSpan()
flow.Update(span, name)
if len(ring) == 0 || ring[i].name == name { // ring is setup so only respond if this is the right place
// return any stored value for this key (Cassandra READ.ONE behavior)
gotocol.Message{gotocol.GetResponse, listener, time.Now(), span, store[msg.Intention]}.GoSend(msg.ResponseChan)
} else {
// send the message to the right place, but don't change the ResponseChan
gotocol.Message{gotocol.GetRequest, msg.ResponseChan, time.Now(), span, msg.Intention}.GoSend(microservices[ring[i].name])
}
case gotocol.GetResponse:
// return path from a request, send payload back up, not used by priamCassandra currently
case gotocol.Put:
// set a key value pair and replicate globally
var key, value string
span := msg.Ctx.NewSpan()
flow.Update(span, name)
fmt.Sscanf(msg.Intention, "%s%s", &key, &value)
if key != "" && value != "" {
i := ring.Find(ringHash(key))
if len(ring) == 0 || ring[i].name == name { // ring is setup so only store if this is the right place
store[key] = value
} else {
// send the message to the right place, but don't change the ResponseChan
gotocol.Message{gotocol.Put, msg.ResponseChan, time.Now(), span, msg.Intention}.GoSend(microservices[ring[i].name])
}
// duplicate the request on to priamCassandra nodes in each zone and one in each region
for _, z := range names.OtherZones(name, archaius.Conf.ZoneNames) {
// replicate request
for n, c := range microservices {
if names.Region(n) == names.Region(name) && names.Zone(n) == z {
gotocol.Message{gotocol.Replicate, listener, time.Now(), span, msg.Intention}.GoSend(c)
break // only need to send it to one node in each zone, no tokens yet
}
}
}
for _, r := range names.OtherRegions(name, archaius.Conf.RegionNames[0:archaius.Conf.Regions]) {
for n, c := range microservices {
if names.Region(n) == r {
gotocol.Message{gotocol.Replicate, listener, time.Now(), span, msg.Intention}.GoSend(c)
break // only need to send it to one node in each region, no tokens yet
}
}
}
}
case gotocol.Replicate:
// Replicate is only used between priamCassandra nodes
// end point for a request
var key, value string
span := msg.Ctx.NewSpan()
flow.Update(span, name)
fmt.Sscanf(msg.Intention, "%s%s", &key, &value)
// log.Printf("priamCassandra: %v:%v", key, value)
if key != "" && value != "" {
i := ring.Find(ringHash(key))
if len(ring) == 0 || ring[i].name == name { // ring is setup so only store if this is the right place
store[key] = value
} else {
// send the message to the right place, but don't change the ResponseChan
gotocol.Message{gotocol.Replicate, msg.ResponseChan, time.Now(), span, msg.Intention}.GoSend(microservices[ring[i].name])
}
}
// name looks like: netflixoss.us-east-1.zoneC.cassTurtle.priamCassandra.cassTurtle11
myregion := names.Region(name)
//log.Printf("%v: %v\n", name, myregion)
// find if this was a cross region Replicate
for in, c := range microservices {
// find the name matching incoming request channel to see where its coming from
if c == msg.ResponseChan && myregion != names.Region(in) {
// Replicate from out of region needs to be Replicated once only to other zones in this Region
for _, z := range names.OtherZones(name, archaius.Conf.ZoneNames) {
// replicate request
for n, c := range microservices {
if names.Region(n) == myregion && names.Zone(n) == z {
gotocol.Message{gotocol.Replicate, listener, time.Now(), span, msg.Intention}.GoSend(c)
break // only need to send it to one node in each zone, no tokens yet
}
}
}
break
}
}
case gotocol.Goodbye:
if archaius.Conf.Msglog {
log.Printf("%v: Going away, zone: %v\n", name, store["zone"])
}
gotocol.Message{gotocol.Goodbye, nil, time.Now(), gotocol.NilContext, name}.GoSend(parent)
return
}
case <-eurekaTicker.C: // check to see if any new dependencies have appeared
for dep, _ := range dependencies {
for _, ch := range eureka {
ch <- gotocol.Message{gotocol.GetRequest, listener, time.Now(), gotocol.NilContext, dep}
}
}
}
}
}
// Start a node using the named package, and connect it to any dependencies
func StartNode(name string, dependencies ...string) {
if names.Package(name) == "eureka" {
eurekachan[name] = make(chan gotocol.Message, archaius.Conf.Population)
go eureka.Start(eurekachan[name], name)
return
} else {
noodles[name] = make(chan gotocol.Message)
}
// start the service and tell it it's name
switch names.Package(name) {
case "pirate":
go pirate.Start(noodles[name])
case "elb":
go elb.Start(noodles[name])
case "denominator":
go denominator.Start(noodles[name])
case "zuul":
go zuul.Start(noodles[name])
case "karyon":
go karyon.Start(noodles[name])
case "monolith":
go monolith.Start(noodles[name])
case "staash":
go staash.Start(noodles[name])
case "priamCassandra":
go priamCassandra.Start(noodles[name])
case "store":
go store.Start(noodles[name])
default:
log.Fatal("asgard: unknown package: " + names.Package(name))
}
noodles[name] <- gotocol.Message{gotocol.Hello, listener, time.Now(), gotocol.NilContext, name}
// there is a eureka service registry in each zone, so in-zone services just get to talk to their local registry
// elb are cross zone, so need to see all registries in a region
// denominator are cross region so need to see all registries globally
// priamCassandra depends explicitly on eureka for cross region clusters
crossregion := false
for _, d := range dependencies {
if d == "eureka" {
crossregion = true
}
}
for n, ch := range eurekachan {
if names.Region(name) == "*" || crossregion {
// need to know every eureka in all zones and regions
gotocol.Send(noodles[name], gotocol.Message{gotocol.Inform, ch, time.Now(), gotocol.NilContext, n})
} else {
if names.Zone(name) == "*" && names.Region(name) == names.Region(n) {
// need every eureka in my region
gotocol.Send(noodles[name], gotocol.Message{gotocol.Inform, ch, time.Now(), gotocol.NilContext, n})
} else {
if names.RegionZone(name) == names.RegionZone(n) {
// just the eureka in this specific zone
gotocol.Send(noodles[name], gotocol.Message{gotocol.Inform, ch, time.Now(), gotocol.NilContext, n})
}
}
}
}
//log.Println(dependencies)
// pass on symbolic dependencies without channels that will be looked up in Eureka later
for _, dep := range dependencies {
if dep != "" && dep != "eureka" { // ignore special case of eureka in dependency list
//log.Println(name + " depends on " + dep)
gotocol.Send(noodles[name], gotocol.Message{gotocol.NameDrop, nil, time.Now(), gotocol.NilContext, dep})
}
}
}
// Start store, all configuration and state is sent via messages
func Start(listener chan gotocol.Message) {
// remember the channel to talk to microservices
microservices := make(map[string]chan gotocol.Message)
dependencies := make(map[string]time.Time) // dependent services and time last updated
store := make(map[string]string, 4) // key value store
store["why?"] = "because..."
var netflixoss chan gotocol.Message // remember creator and how to talk back to incoming requests
var name string // remember my name
eureka := make(map[string]chan gotocol.Message, 3) // service registry per zone
hist := collect.NewHist("")
ep, _ := time.ParseDuration(archaius.Conf.EurekaPoll)
eurekaTicker := time.NewTicker(ep)
for {
select {
case msg := <-listener:
flow.Instrument(msg, name, hist)
switch msg.Imposition {
case gotocol.Hello:
if name == "" {
// if I don't have a name yet remember what I've been named
netflixoss = msg.ResponseChan // remember how to talk to my namer
name = msg.Intention // message body is my name
hist = collect.NewHist(name)
}
case gotocol.Inform:
eureka[msg.Intention] = handlers.Inform(msg, name, listener)
case gotocol.NameDrop: // cross zone = true
handlers.NameDrop(&dependencies, µservices, msg, name, listener, eureka, true)
case gotocol.Forget:
// forget a buddy
handlers.Forget(&dependencies, µservices, msg)
case gotocol.GetRequest:
// return any stored value for this key
outmsg := gotocol.Message{gotocol.GetResponse, listener, time.Now(), msg.Ctx, store[msg.Intention]}
flow.AnnotateSend(outmsg, name)
outmsg.GoSend(msg.ResponseChan)
case gotocol.GetResponse:
// return path from a request, send payload back up (not currently used)
case gotocol.Put:
// set a key value pair and replicate globally
var key, value string
fmt.Sscanf(msg.Intention, "%s%s", &key, &value)
if key != "" && value != "" {
store[key] = value
// duplicate the request on to all connected store nodes
if len(microservices) > 0 {
// replicate request
for _, c := range microservices {
outmsg := gotocol.Message{gotocol.Replicate, listener, time.Now(), msg.Ctx.NewParent(), msg.Intention}
flow.AnnotateSend(outmsg, name)
outmsg.GoSend(c)
}
}
}
case gotocol.Replicate:
// Replicate is only used between store nodes
// end point for a request
var key, value string
fmt.Sscanf(msg.Intention, "%s%s", &key, &value)
// log.Printf("store: %v:%v", key, value)
if key != "" && value != "" {
store[key] = value
}
// name looks like: netflixoss.us-east-1.zoneC.mysql.store.mysql1
myregion := names.Region(name)
//log.Printf("%v: %v\n", name, myregion)
// find if this was a cross region Replicate
for n, c := range microservices {
// find the name matching incoming request channel
if c == msg.ResponseChan {
if myregion != names.Region(n) {
// Replicate from out of region needs to be Replicated only to other zones in this Region
for nz, cz := range microservices {
if myregion == names.Region(nz) {
//log.Printf("%v rep to: %v\n", name, nz)
outmsg := gotocol.Message{gotocol.Replicate, listener, time.Now(), msg.Ctx.NewParent(), msg.Intention}
flow.AnnotateSend(outmsg, name)
outmsg.GoSend(cz)
}
}
}
}
}
case gotocol.Goodbye:
gotocol.Message{gotocol.Goodbye, nil, time.Now(), gotocol.NilContext, name}.GoSend(netflixoss)
return
}
case <-eurekaTicker.C: // check to see if any new dependencies have appeared
for dep, _ := range dependencies {
for _, ch := range eureka {
ch <- gotocol.Message{gotocol.GetRequest, listener, time.Now(), gotocol.NilContext, dep}
}
}
}
}
}