// List the certs registered in my system func getCerts(rw http.ResponseWriter, req *http.Request) { certs, err := common.GetCerts() if err != nil { writeError(rw, req, err, http.StatusInternalServerError) return } writeBody(rw, req, certs, http.StatusOK) }
func (n None) Init() error { // load services services, err := common.GetServices() if err != nil { return err } // apply services err = common.SetServices(services) if err != nil { return err } // load routes routes, err := common.GetRoutes() if err != nil { return err } // apply routes err = common.SetRoutes(routes) if err != nil { return err } // load certs certs, err := common.GetCerts() if err != nil { return err } // apply certs err = common.SetCerts(certs) if err != nil { return err } // load vips vips, err := common.GetVips() if err != nil { return err } // apply vips err = common.SetVips(vips) if err != nil { return err } return nil }
// DeleteCert tells all members to remove the cert from their database. // rolls back on failure func (r Redis) DeleteCert(cert core.CertBundle) error { conn := pool.Get() defer conn.Close() oldCerts, err := common.GetCerts() // this should not return nil to ensure the cert is gone from entire cluster if err != nil && !strings.Contains(err.Error(), "No Cert Found") { return err } // publishJson to others err = r.publishJson(conn, "delete-cert", cert) if err != nil { // if i failed to publishJson, request should fail return err } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("delete-cert %s", cert)))) // ensure all members applied action err = r.waitForMembers(conn, actionHash) if err != nil { uActionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-certs", oldCerts)))) // cleanup rollback cruft. clear actionHash ensures no mistakes on re-submit defer conn.Do("DEL", uActionHash, actionHash) // attempt rollback - no need to waitForMembers here uerr := r.publishJson(conn, "set-certs", oldCerts) if uerr != nil { err = fmt.Errorf("%v - %v", err, uerr) } return err } if database.CentralStore { return database.DeleteCert(cert) } return nil }
// SetCerts tells all members to replace the certs in their database with a new set. // rolls back on failure func (r Redis) SetCerts(certs []core.CertBundle) error { conn := pool.Get() defer conn.Close() oldCerts, err := common.GetCerts() if err != nil { return err } // publishJson to others err = r.publishJson(conn, "set-certs", certs) if err != nil { // if i failed to publishJson, request should fail return err } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-certs %s", certs)))) // ensure all members applied action err = r.waitForMembers(conn, actionHash) if err != nil { uActionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-certs %s", oldCerts)))) // cleanup rollback cruft. clear actionHash ensures no mistakes on re-submit defer conn.Do("DEL", uActionHash, actionHash) // attempt rollback - no need to waitForMembers here uerr := r.publishJson(conn, "set-certs", oldCerts) if uerr != nil { err = fmt.Errorf("%v - %v", err, uerr) } return err } if database.CentralStore { return database.SetCerts(certs) } return nil }
func (n None) GetCerts() ([]core.CertBundle, error) { return common.GetCerts() }
// GetCerts gets a list of certs from the database, or another cluster member. func (r *Redis) GetCerts() ([]core.CertBundle, error) { if database.CentralStore { return database.GetCerts() } conn := pool.Get() defer conn.Close() // get known members(other than me) to 'poll' for certs members, _ := redis.Strings(conn.Do("SMEMBERS", "members")) if len(members) == 0 { // should only happen on new cluster // assume i'm ok to be master so don't reset imported certs config.Log.Trace("[cluster] - Assuming OK to be master, using certs from my database...") return common.GetCerts() } for i := range members { if members[i] == self { // if i'm in the list of members, new requests should have failed while `waitForMembers`ing config.Log.Trace("[cluster] - Assuming I was in sync, using certs from my database...") return common.GetCerts() } } c, err := redis.DialURL(config.ClusterConnection, redis.DialConnectTimeout(15*time.Second), redis.DialPassword(config.ClusterToken)) if err != nil { return nil, fmt.Errorf("Failed to reach redis for certs subscriber - %v", err) } defer c.Close() message := make(chan interface{}) subconn := redis.PubSubConn{c} // subscribe to channel that certs will be published on if err := subconn.Subscribe("certs"); err != nil { return nil, fmt.Errorf("Failed to reach redis for certs subscriber - %v", err) } defer subconn.Close() // listen always go func() { for { message <- subconn.Receive() } }() // todo: maybe use ttl? // timeout is how long to wait for the listed members to come back online timeout := time.After(time.Duration(20) * time.Second) // loop attempts for timeout, allows last dead members to start back up for { select { case <-timeout: return nil, fmt.Errorf("Timed out waiting for certs from %v", strings.Join(members, ", ")) default: // request certs from each member until successful for _, member := range members { // memberTimeout is how long to wait for a member to respond with list of certs memberTimeout := time.After(3 * time.Second) // ask a member for its certs config.Log.Trace("[cluster] - Attempting to request certs from %v...", member) _, err := conn.Do("PUBLISH", "portal", fmt.Sprintf("get-certs %s", member)) if err != nil { return nil, err } // wait for member to respond for { select { case <-memberTimeout: config.Log.Debug("[cluster] - Timed out waiting for certs from %v", member) goto nextCertMember case msg := <-message: switch v := msg.(type) { case redis.Message: config.Log.Trace("[cluster] - Received message on 'certs' channel") var certs []core.CertBundle err = parseBody(v.Data, &certs) if err != nil { return nil, fmt.Errorf("Failed to marshal certs - %v", err.Error()) } config.Log.Trace("[cluster] - Certs from cluster: %#v\n", certs) return certs, nil case error: return nil, fmt.Errorf("Subscriber failed to receive certs - %v", v.Error()) } } } nextCertMember: } } } }
// subscribe listens on the portal channel and acts based on messages received func (r Redis) subscribe() { config.Log.Info("[cluster] - Redis subscribing on %s...", config.ClusterConnection) // listen for published messages for { switch v := r.subconn.Receive().(type) { case redis.Message: switch pdata := strings.FieldsFunc(string(v.Data), keepSubstrings); pdata[0] { // SERVICES /////////////////////////////////////////////////////////////////////////////////////////////// case "get-services": if len(pdata) != 2 { config.Log.Error("[cluster] - member not passed in message") break } member := pdata[1] if member == self { svcs, err := common.GetServices() if err != nil { config.Log.Error("[cluster] - Failed to get services - %v", err.Error()) break } services, err := json.Marshal(svcs) if err != nil { config.Log.Error("[cluster] - Failed to marshal services - %v", err.Error()) break } config.Log.Debug("[cluster] - get-services requested, publishing my services") conn := pool.Get() conn.Do("PUBLISH", "services", fmt.Sprintf("%s", services)) conn.Close() } case "set-services": if len(pdata) != 2 { config.Log.Error("[cluster] - services not passed in message") break } services, err := marshalSvcs([]byte(pdata[1])) if err != nil { config.Log.Error("[cluster] - Failed to marshal services - %v", err.Error()) break } err = common.SetServices(*services) if err != nil { config.Log.Error("[cluster] - Failed to set services - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-services %s", *services)))) config.Log.Trace("[cluster] - set-services hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - set-services successful") case "set-service": if len(pdata) != 2 { // shouldn't happen unless redis is not secure and someone manually `publishJson`es config.Log.Error("[cluster] - service not passed in message") break } svc, err := marshalSvc([]byte(pdata[1])) if err != nil { config.Log.Error("[cluster] - Failed to marshal service - %v", err.Error()) break } err = common.SetService(svc) if err != nil { config.Log.Error("[cluster] - Failed to set service - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-service %s", *svc)))) config.Log.Trace("[cluster] - set-service hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - set-service successful") case "delete-service": if len(pdata) != 2 { config.Log.Error("[cluster] - service id not passed in message") break } svcId := pdata[1] err := common.DeleteService(svcId) if err != nil { config.Log.Error("[cluster] - Failed to delete service - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("delete-service %s", svcId)))) config.Log.Trace("[cluster] - delete-service hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - delete-service successful") // SERVERS /////////////////////////////////////////////////////////////////////////////////////////////// case "set-servers": if len(pdata) != 3 { config.Log.Error("[cluster] - service id not passed in message") break } svcId := pdata[2] servers, err := marshalSrvs([]byte(pdata[1])) if err != nil { config.Log.Error("[cluster] - Failed to marshal server - %v", err.Error()) break } err = common.SetServers(svcId, *servers) if err != nil { config.Log.Error("[cluster] - Failed to set servers - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-servers %s %s", *servers, svcId)))) config.Log.Trace("[cluster] - set-servers hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - set-servers successful") case "set-server": if len(pdata) != 3 { // shouldn't happen unless redis is not secure and someone manually publishJson config.Log.Error("[cluster] - service id not passed in message") break } svcId := pdata[2] server, err := marshalSrv([]byte(pdata[1])) if err != nil { config.Log.Error("[cluster] - Failed to marshal server - %v", err.Error()) break } err = common.SetServer(svcId, server) if err != nil { config.Log.Error("[cluster] - Failed to set server - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-server %s %s", *server, svcId)))) config.Log.Trace("[cluster] - set-server hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - set-server successful") case "delete-server": if len(pdata) != 3 { config.Log.Error("[cluster] - service id not passed in message") break } srvId := pdata[1] svcId := pdata[2] err := common.DeleteServer(svcId, srvId) if err != nil { config.Log.Error("[cluster] - Failed to delete server - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("delete-server %s %s", srvId, svcId)))) config.Log.Trace("[cluster] - delete-server hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - delete-server successful") // ROUTES /////////////////////////////////////////////////////////////////////////////////////////////// case "get-routes": if len(pdata) != 2 { config.Log.Error("[cluster] - member not passed in message") break } member := pdata[1] if member == self { rts, err := common.GetRoutes() if err != nil { config.Log.Error("[cluster] - Failed to get routes - %v", err.Error()) break } routes, err := json.Marshal(rts) if err != nil { config.Log.Error("[cluster] - Failed to marshal routes - %v", err.Error()) break } config.Log.Debug("[cluster] - get-routes requested, publishing my routes") conn := pool.Get() conn.Do("PUBLISH", "routes", fmt.Sprintf("%s", routes)) conn.Close() } case "set-routes": if len(pdata) != 2 { config.Log.Error("[cluster] - routes not passed in message") break } var routes []core.Route err := parseBody([]byte(pdata[1]), &routes) if err != nil { config.Log.Error("[cluster] - Failed to marshal routes - %v", err.Error()) break } err = common.SetRoutes(routes) if err != nil { config.Log.Error("[cluster] - Failed to set routes - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-routes %s", routes)))) config.Log.Trace("[cluster] - set-routes hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - set-routes successful") case "set-route": if len(pdata) != 2 { // shouldn't happen unless redis is not secure and someone manually `publishJson`es config.Log.Error("[cluster] - route not passed in message") break } var rte core.Route err := parseBody([]byte(pdata[1]), &rte) if err != nil { config.Log.Error("[cluster] - Failed to marshal route - %v", err.Error()) break } err = common.SetRoute(rte) if err != nil { config.Log.Error("[cluster] - Failed to set route - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-route %s", rte)))) config.Log.Trace("[cluster] - set-route hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - set-route successful") case "delete-route": if len(pdata) != 2 { config.Log.Error("[cluster] - route not passed in message") break } var rte core.Route err := parseBody([]byte(pdata[1]), &rte) if err != nil { config.Log.Error("[cluster] - Failed to marshal route - %v", err.Error()) break } err = common.DeleteRoute(rte) if err != nil { config.Log.Error("[cluster] - Failed to delete route - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("delete-route %s", rte)))) config.Log.Trace("[cluster] - delete-route hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - delete-route successful") // CERTS /////////////////////////////////////////////////////////////////////////////////////////////// case "get-certs": if len(pdata) != 2 { config.Log.Error("[cluster] - member not passed in message") break } member := pdata[1] if member == self { rts, err := common.GetCerts() if err != nil { config.Log.Error("[cluster] - Failed to get certs - %v", err.Error()) break } certs, err := json.Marshal(rts) if err != nil { config.Log.Error("[cluster] - Failed to marshal certs - %v", err.Error()) break } config.Log.Debug("[cluster] - get-certs requested, publishing my certs") conn := pool.Get() conn.Do("PUBLISH", "certs", fmt.Sprintf("%s", certs)) conn.Close() } case "set-certs": if len(pdata) != 2 { config.Log.Error("[cluster] - certs not passed in message") break } var certs []core.CertBundle err := parseBody([]byte(pdata[1]), &certs) if err != nil { config.Log.Error("[cluster] - Failed to marshal certs - %v", err.Error()) break } err = common.SetCerts(certs) if err != nil { config.Log.Error("[cluster] - Failed to set certs - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-certs %s", certs)))) config.Log.Trace("[cluster] - set-certs hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - set-certs successful") case "set-cert": if len(pdata) != 2 { // shouldn't happen unless redis is not secure and someone manually `publishJson`es config.Log.Error("[cluster] - cert not passed in message") break } var crt core.CertBundle err := parseBody([]byte(pdata[1]), &crt) if err != nil { config.Log.Error("[cluster] - Failed to marshal cert - %v", err.Error()) break } err = common.SetCert(crt) if err != nil { config.Log.Error("[cluster] - Failed to set cert - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("set-cert %s", crt)))) config.Log.Trace("[cluster] - set-cert hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - set-cert successful") case "delete-cert": if len(pdata) != 2 { config.Log.Error("[cluster] - cert id not passed in message") break } var crt core.CertBundle err := parseBody([]byte(pdata[1]), &crt) if err != nil { config.Log.Error("[cluster] - Failed to marshal cert - %v", err.Error()) break } err = common.DeleteCert(crt) if err != nil { config.Log.Error("[cluster] - Failed to delete cert - %v", err.Error()) break } actionHash := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("delete-cert %s", crt)))) config.Log.Trace("[cluster] - delete-cert hash - %v", actionHash) conn := pool.Get() conn.Do("SADD", actionHash, self) conn.Close() config.Log.Debug("[cluster] - delete-cert successful") default: config.Log.Error("[cluster] - Recieved unknown data on %v: %v", v.Channel, string(v.Data)) } case error: config.Log.Error("[cluster] - Subscriber failed to receive - %v", v.Error()) if strings.Contains(v.Error(), "closed network connection") { // clear balancer rules ("stop balancing if we are 'dead'") balance.SetServices(make([]core.Service, 0, 0)) // exit so we don't get spammed with logs os.Exit(1) } continue } } }