// List all services
func getServices(rw http.ResponseWriter, req *http.Request) {
// /services
svcs, err := common.GetServices()
if err != nil {
writeError(rw, req, err, http.StatusInternalServerError)
return
}
writeBody(rw, req, svcs, http.StatusOK)
}
func (n None) Init() error {
// load services
services, err := common.GetServices()
if err != nil {
return err
}
// apply services
err = common.SetServices(services)
if err != nil {
return err
}
// load routes
routes, err := common.GetRoutes()
if err != nil {
return err
}
// apply routes
err = common.SetRoutes(routes)
if err != nil {
return err
}
// load certs
certs, err := common.GetCerts()
if err != nil {
return err
}
// apply certs
err = common.SetCerts(certs)
if err != nil {
return err
}
// load vips
vips, err := common.GetVips()
if err != nil {
return err
}
// apply vips
err = common.SetVips(vips)
if err != nil {
return err
}
return nil
}
// SetServices tells all members to replace the services in their database with a new set.
// rolls back on failure
func (r *Redis) SetServices(services []core.Service) error {
conn := pool.Get()
defer conn.Close()
oldServices, err := common.GetServices()
if err != nil {
return err
}
// publishJson to others
err = r.publishJson(conn, "set-services", services)
if err != nil {
// if i failed to publishJson, request should fail
return err
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-services %s", services))))
// ensure all members applied action
err = r.waitForMembers(conn, actionHash)
if err != nil {
uActionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-services %s", oldServices))))
// cleanup rollback cruft. clear actionHash ensures no mistakes on re-submit
defer conn.Do("DEL", uActionHash, actionHash)
// attempt rollback - no need to waitForMembers here
uerr := r.publishJson(conn, "set-services", oldServices)
if uerr != nil {
err = fmt.Errorf("%v - %v", err, uerr)
}
return err
}
if database.CentralStore {
return database.SetServices(services)
}
return nil
}
// Replace a service (by replacing all services)
func putService(rw http.ResponseWriter, req *http.Request) {
// /services/{svcId}
svcId := req.URL.Query().Get(":svcId")
// rough sanitization
if len(strings.Split(svcId, "-")) != 3 {
writeError(rw, req, NoServiceError, http.StatusBadRequest)
return
}
service, err := parseReqService(req)
if err != nil {
writeError(rw, req, err, http.StatusBadRequest)
return
}
services, err := common.GetServices()
if err != nil {
writeError(rw, req, err, http.StatusInternalServerError)
return
}
// update service by id
for i := range services {
if services[i].Id == svcId {
services[i] = *service
break
}
}
// save to cluster
if err := cluster.SetServices(services); err != nil {
writeError(rw, req, err, http.StatusInternalServerError)
return
}
writeBody(rw, req, service, http.StatusOK)
}
func (n None) GetServices() ([]core.Service, error) {
return common.GetServices()
}
// GetServices gets a list of services from the database, or another cluster member.
func (r *Redis) GetServices() ([]core.Service, error) {
if database.CentralStore {
return database.GetServices()
}
conn := pool.Get()
defer conn.Close()
// get known members(other than me) to 'poll' for services
members, _ := redis.Strings(conn.Do("SMEMBERS", "members"))
if len(members) == 0 {
// should only happen on new cluster
// assume i'm ok to be master so don't reset imported services
config.Log.Trace("[cluster] - Assuming OK to be master, using services from my database...")
return common.GetServices()
}
for i := range members {
if members[i] == self {
// if i'm in the list of members, new requests should have failed while `waitForMembers`ing
config.Log.Trace("[cluster] - Assuming I was in sync, using services from my database...")
return common.GetServices()
}
}
c, err := redis.DialURL(config.ClusterConnection, redis.DialConnectTimeout(15*time.Second), redis.DialPassword(config.ClusterToken))
if err != nil {
return nil, fmt.Errorf("Failed to reach redis for services subscriber - %v", err)
}
defer c.Close()
message := make(chan interface{})
subconn := redis.PubSubConn{c}
// subscribe to channel that services will be published on
if err := subconn.Subscribe("services"); err != nil {
return nil, fmt.Errorf("Failed to reach redis for services subscriber - %v", err)
}
defer subconn.Close()
// listen always
go func() {
for {
message <- subconn.Receive()
}
}()
// todo: maybe use ttl?
// timeout is how long to wait for the listed members to come back online
timeout := time.After(time.Duration(20) * time.Second)
// loop attempts for timeout, allows last dead members to start back up
for {
select {
case <-timeout:
return nil, fmt.Errorf("Timed out waiting for services from %v", strings.Join(members, ", "))
default:
// request services from each member until successful
for _, member := range members {
// memberTimeout is how long to wait for a member to respond with list of services
memberTimeout := time.After(3 * time.Second)
// ask a member for its services
config.Log.Trace("[cluster] - Attempting to request services from %v...", member)
_, err := conn.Do("PUBLISH", "portal", fmt.Sprintf("get-services %s", member))
if err != nil {
return nil, err
}
// wait for member to respond
for {
select {
case <-memberTimeout:
config.Log.Debug("[cluster] - Timed out waiting for services from %v", member)
goto nextMember
case msg := <-message:
switch v := msg.(type) {
case redis.Message:
config.Log.Trace("[cluster] - Received message on 'services' channel")
services, err := marshalSvcs(v.Data)
if err != nil {
return nil, fmt.Errorf("Failed to marshal services - %v", err.Error())
}
config.Log.Trace("[cluster] - Services from cluster: %#v\n", *services)
return *services, nil
case error:
return nil, fmt.Errorf("Subscriber failed to receive services - %v", v.Error())
}
}
}
nextMember:
}
}
}
}
// subscribe listens on the portal channel and acts based on messages received
func (r Redis) subscribe() {
config.Log.Info("[cluster] - Redis subscribing on %s...", config.ClusterConnection)
// listen for published messages
for {
switch v := r.subconn.Receive().(type) {
case redis.Message:
switch pdata := strings.FieldsFunc(string(v.Data), keepSubstrings); pdata[0] {
// SERVICES ///////////////////////////////////////////////////////////////////////////////////////////////
case "get-services":
if len(pdata) != 2 {
config.Log.Error("[cluster] - member not passed in message")
break
}
member := pdata[1]
if member == self {
svcs, err := common.GetServices()
if err != nil {
config.Log.Error("[cluster] - Failed to get services - %v", err.Error())
break
}
services, err := json.Marshal(svcs)
if err != nil {
config.Log.Error("[cluster] - Failed to marshal services - %v", err.Error())
break
}
config.Log.Debug("[cluster] - get-services requested, publishing my services")
conn := pool.Get()
conn.Do("PUBLISH", "services", fmt.Sprintf("%s", services))
conn.Close()
}
case "set-services":
if len(pdata) != 2 {
config.Log.Error("[cluster] - services not passed in message")
break
}
services, err := marshalSvcs([]byte(pdata[1]))
if err != nil {
config.Log.Error("[cluster] - Failed to marshal services - %v", err.Error())
break
}
err = common.SetServices(*services)
if err != nil {
config.Log.Error("[cluster] - Failed to set services - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-services %s", *services))))
config.Log.Trace("[cluster] - set-services hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - set-services successful")
case "set-service":
if len(pdata) != 2 {
// shouldn't happen unless redis is not secure and someone manually `publishJson`es
config.Log.Error("[cluster] - service not passed in message")
break
}
svc, err := marshalSvc([]byte(pdata[1]))
if err != nil {
config.Log.Error("[cluster] - Failed to marshal service - %v", err.Error())
break
}
err = common.SetService(svc)
if err != nil {
config.Log.Error("[cluster] - Failed to set service - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-service %s", *svc))))
config.Log.Trace("[cluster] - set-service hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - set-service successful")
case "delete-service":
if len(pdata) != 2 {
config.Log.Error("[cluster] - service id not passed in message")
break
}
svcId := pdata[1]
err := common.DeleteService(svcId)
if err != nil {
config.Log.Error("[cluster] - Failed to delete service - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("delete-service %s", svcId))))
config.Log.Trace("[cluster] - delete-service hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - delete-service successful")
// SERVERS ///////////////////////////////////////////////////////////////////////////////////////////////
case "set-servers":
if len(pdata) != 3 {
config.Log.Error("[cluster] - service id not passed in message")
break
}
svcId := pdata[2]
servers, err := marshalSrvs([]byte(pdata[1]))
if err != nil {
config.Log.Error("[cluster] - Failed to marshal server - %v", err.Error())
break
}
err = common.SetServers(svcId, *servers)
if err != nil {
config.Log.Error("[cluster] - Failed to set servers - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-servers %s %s", *servers, svcId))))
config.Log.Trace("[cluster] - set-servers hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - set-servers successful")
case "set-server":
if len(pdata) != 3 {
// shouldn't happen unless redis is not secure and someone manually publishJson
config.Log.Error("[cluster] - service id not passed in message")
break
}
svcId := pdata[2]
server, err := marshalSrv([]byte(pdata[1]))
if err != nil {
config.Log.Error("[cluster] - Failed to marshal server - %v", err.Error())
break
}
err = common.SetServer(svcId, server)
if err != nil {
config.Log.Error("[cluster] - Failed to set server - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-server %s %s", *server, svcId))))
config.Log.Trace("[cluster] - set-server hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - set-server successful")
case "delete-server":
if len(pdata) != 3 {
config.Log.Error("[cluster] - service id not passed in message")
break
}
srvId := pdata[1]
svcId := pdata[2]
err := common.DeleteServer(svcId, srvId)
if err != nil {
config.Log.Error("[cluster] - Failed to delete server - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("delete-server %s %s", srvId, svcId))))
config.Log.Trace("[cluster] - delete-server hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - delete-server successful")
// ROUTES ///////////////////////////////////////////////////////////////////////////////////////////////
case "get-routes":
if len(pdata) != 2 {
config.Log.Error("[cluster] - member not passed in message")
break
}
member := pdata[1]
if member == self {
rts, err := common.GetRoutes()
if err != nil {
config.Log.Error("[cluster] - Failed to get routes - %v", err.Error())
break
}
routes, err := json.Marshal(rts)
if err != nil {
config.Log.Error("[cluster] - Failed to marshal routes - %v", err.Error())
break
}
config.Log.Debug("[cluster] - get-routes requested, publishing my routes")
conn := pool.Get()
conn.Do("PUBLISH", "routes", fmt.Sprintf("%s", routes))
conn.Close()
}
case "set-routes":
if len(pdata) != 2 {
config.Log.Error("[cluster] - routes not passed in message")
break
}
var routes []core.Route
err := parseBody([]byte(pdata[1]), &routes)
if err != nil {
config.Log.Error("[cluster] - Failed to marshal routes - %v", err.Error())
break
}
err = common.SetRoutes(routes)
if err != nil {
config.Log.Error("[cluster] - Failed to set routes - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-routes %s", routes))))
config.Log.Trace("[cluster] - set-routes hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - set-routes successful")
case "set-route":
if len(pdata) != 2 {
// shouldn't happen unless redis is not secure and someone manually `publishJson`es
config.Log.Error("[cluster] - route not passed in message")
break
}
var rte core.Route
err := parseBody([]byte(pdata[1]), &rte)
if err != nil {
config.Log.Error("[cluster] - Failed to marshal route - %v", err.Error())
break
}
err = common.SetRoute(rte)
if err != nil {
config.Log.Error("[cluster] - Failed to set route - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-route %s", rte))))
config.Log.Trace("[cluster] - set-route hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - set-route successful")
case "delete-route":
if len(pdata) != 2 {
config.Log.Error("[cluster] - route not passed in message")
break
}
var rte core.Route
err := parseBody([]byte(pdata[1]), &rte)
if err != nil {
config.Log.Error("[cluster] - Failed to marshal route - %v", err.Error())
break
}
err = common.DeleteRoute(rte)
if err != nil {
config.Log.Error("[cluster] - Failed to delete route - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("delete-route %s", rte))))
config.Log.Trace("[cluster] - delete-route hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - delete-route successful")
// CERTS ///////////////////////////////////////////////////////////////////////////////////////////////
case "get-certs":
if len(pdata) != 2 {
config.Log.Error("[cluster] - member not passed in message")
break
}
member := pdata[1]
if member == self {
rts, err := common.GetCerts()
if err != nil {
config.Log.Error("[cluster] - Failed to get certs - %v", err.Error())
break
}
certs, err := json.Marshal(rts)
if err != nil {
config.Log.Error("[cluster] - Failed to marshal certs - %v", err.Error())
break
}
config.Log.Debug("[cluster] - get-certs requested, publishing my certs")
conn := pool.Get()
conn.Do("PUBLISH", "certs", fmt.Sprintf("%s", certs))
conn.Close()
}
case "set-certs":
if len(pdata) != 2 {
config.Log.Error("[cluster] - certs not passed in message")
break
}
var certs []core.CertBundle
err := parseBody([]byte(pdata[1]), &certs)
if err != nil {
config.Log.Error("[cluster] - Failed to marshal certs - %v", err.Error())
break
}
err = common.SetCerts(certs)
if err != nil {
config.Log.Error("[cluster] - Failed to set certs - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-certs %s", certs))))
config.Log.Trace("[cluster] - set-certs hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - set-certs successful")
case "set-cert":
if len(pdata) != 2 {
// shouldn't happen unless redis is not secure and someone manually `publishJson`es
config.Log.Error("[cluster] - cert not passed in message")
break
}
var crt core.CertBundle
err := parseBody([]byte(pdata[1]), &crt)
if err != nil {
config.Log.Error("[cluster] - Failed to marshal cert - %v", err.Error())
break
}
err = common.SetCert(crt)
if err != nil {
config.Log.Error("[cluster] - Failed to set cert - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-cert %s", crt))))
config.Log.Trace("[cluster] - set-cert hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - set-cert successful")
case "delete-cert":
if len(pdata) != 2 {
config.Log.Error("[cluster] - cert id not passed in message")
break
}
var crt core.CertBundle
err := parseBody([]byte(pdata[1]), &crt)
if err != nil {
config.Log.Error("[cluster] - Failed to marshal cert - %v", err.Error())
break
}
err = common.DeleteCert(crt)
if err != nil {
config.Log.Error("[cluster] - Failed to delete cert - %v", err.Error())
break
}
actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("delete-cert %s", crt))))
config.Log.Trace("[cluster] - delete-cert hash - %v", actionHash)
conn := pool.Get()
conn.Do("SADD", actionHash, self)
conn.Close()
config.Log.Debug("[cluster] - delete-cert successful")
default:
config.Log.Error("[cluster] - Recieved unknown data on %v: %v", v.Channel, string(v.Data))
}
case error:
config.Log.Error("[cluster] - Subscriber failed to receive - %v", v.Error())
if strings.Contains(v.Error(), "closed network connection") {
// clear balancer rules ("stop balancing if we are 'dead'")
balance.SetServices(make([]core.Service, 0, 0))
// exit so we don't get spammed with logs
os.Exit(1)
}
continue
}
}
}