func (p *dockerProvisioner) Destroy(app provision.App) error {
containers, _ := listContainersByApp(app.GetName())
go func(c []container) {
var containersGroup sync.WaitGroup
containersGroup.Add(len(containers))
for _, c := range containers {
go func(c container) {
defer containersGroup.Done()
err := removeContainer(&c)
if err != nil {
log.Error(err.Error())
}
}(c)
}
containersGroup.Wait()
err := removeImage(assembleImageName(app.GetName()))
if err != nil {
log.Error(err.Error())
}
}(containers)
r, err := getRouter()
if err != nil {
log.Errorf("Failed to get router: %s", err)
return err
}
return r.RemoveBackend(app.GetName())
}
// handle is the function called by the queue handler on each message.
func handle(msg *queue.Message) {
switch msg.Action {
case regenerateApprc:
if len(msg.Args) < 1 {
log.Errorf("Error handling %q: this action requires at least 1 argument.", msg.Action)
return
}
app, err := ensureAppIsStarted(msg)
if err != nil {
log.Error(err.Error())
return
}
err = app.SerializeEnvVars()
if err != nil {
log.Error(err.Error())
}
case BindService:
err := bindUnit(msg)
if err != nil {
log.Error(err.Error())
return
}
default:
log.Errorf("Error handling %q: invalid action.", msg.Action)
}
}
func (c *Client) BindUnit(instance *ServiceInstance, app bind.App, unit bind.Unit) error {
log.Debugf("Calling bind of instance %q and %q unit at %q API",
instance.Name, unit.GetIp(), instance.ServiceName)
var resp *http.Response
params := map[string][]string{
"app-host": {app.GetIp()},
"unit-host": {unit.GetIp()},
}
resp, err := c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind", "POST", params)
if err != nil {
if m, _ := regexp.MatchString("", err.Error()); m {
return fmt.Errorf("%s api is down.", instance.Name)
}
return err
}
switch resp.StatusCode {
case http.StatusPreconditionFailed:
return ErrInstanceNotReady
case http.StatusNotFound:
return ErrInstanceNotFoundInAPI
}
if resp.StatusCode > 299 {
msg := fmt.Sprintf(`Failed to bind the instance "%s/%s" to the unit %q: %s`, instance.ServiceName, instance.Name, unit.GetIp(), c.buildErrorMessage(err, resp))
log.Error(msg)
return errors.New(msg)
}
return nil
}
func (c *Client) BindApp(instance *ServiceInstance, app bind.App) (map[string]string, error) {
log.Debugf("Calling bind of instance %q and %q app at %q API",
instance.Name, app.GetName(), instance.ServiceName)
var resp *http.Response
params := map[string][]string{
"app-host": {app.GetIp()},
}
resp, err := c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind-app", "POST", params)
if resp != nil && resp.StatusCode == http.StatusNotFound {
resp, err = c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind", "POST", params)
}
if err != nil {
log.Errorf(`Failed to bind app %q to service instance "%s/%s": %s`, app.GetName(), instance.ServiceName, instance.Name, err)
return nil, fmt.Errorf("%s api is down.", instance.Name)
}
if err == nil && resp.StatusCode < 300 {
var result map[string]string
err = c.jsonFromResponse(resp, &result)
if err != nil {
return nil, err
}
return result, nil
}
switch resp.StatusCode {
case http.StatusPreconditionFailed:
return nil, ErrInstanceNotReady
case http.StatusNotFound:
return nil, ErrInstanceNotFoundInAPI
}
msg := fmt.Sprintf(`Failed to bind the instance "%s/%s" to the app %q: %s`, instance.ServiceName, instance.Name, app.GetName(), c.buildErrorMessage(err, resp))
log.Error(msg)
return nil, errors.New(msg)
}
func (c *Client) Status(instance *ServiceInstance) (string, error) {
log.Debugf("Attempting to call status of service instance %q at %q api", instance.Name, instance.ServiceName)
var (
resp *http.Response
err error
)
url := "/resources/" + instance.GetIdentifier() + "/status"
if resp, err = c.issueRequest(url, "GET", nil); err == nil {
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
var data []byte
data, err = ioutil.ReadAll(resp.Body)
return string(data), err
case http.StatusAccepted:
return "pending", nil
case http.StatusNoContent:
return "up", nil
case http.StatusNotFound:
return "not implemented for this service", nil
case http.StatusInternalServerError:
return "down", nil
}
}
msg := "Failed to get status of instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
return "", errors.New(msg)
}
// RecreateContainers relaunch all bs containers in the cluster for the given
// DockerProvisioner, logging progress to the given writer.
//
// It assumes that the given writer is thread safe.
func RecreateContainers(p DockerProvisioner, w io.Writer) error {
cluster := p.Cluster()
nodes, err := cluster.UnfilteredNodes()
if err != nil {
return err
}
errChan := make(chan error, len(nodes))
wg := sync.WaitGroup{}
log.Debugf("[bs containers] recreating %d containers", len(nodes))
for i := range nodes {
wg.Add(1)
go func(i int) {
defer wg.Done()
node := &nodes[i]
pool := node.Metadata["pool"]
log.Debugf("[bs containers] recreating container in %s [%s]", node.Address, pool)
fmt.Fprintf(w, "relaunching bs container in the node %s [%s]\n", node.Address, pool)
err := createContainer(node.Address, pool, p, true)
if err != nil {
msg := fmt.Sprintf("[bs containers] failed to create container in %s [%s]: %s", node.Address, pool, err)
log.Error(msg)
err = errors.New(msg)
errChan <- err
}
}(i)
}
wg.Wait()
close(errChan)
return <-errChan
}
func (c *Client) Create(instance *ServiceInstance, user string) error {
var err error
var resp *http.Response
params := map[string][]string{
"name": {instance.Name},
"user": {user},
"team": {instance.TeamOwner},
}
if instance.PlanName != "" {
params["plan"] = []string{instance.PlanName}
}
if instance.Description != "" {
params["description"] = []string{instance.Description}
}
log.Debugf("Attempting to call creation of service instance for %q, params: %#v", instance.ServiceName, params)
if resp, err = c.issueRequest("/resources", "POST", params); err == nil && resp.StatusCode < 300 {
return nil
}
if resp.StatusCode == http.StatusConflict {
return ErrInstanceAlreadyExistsInAPI
}
msg := "Failed to create the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
return errors.New(msg)
}
func addLogs(ws *websocket.Conn) {
var err error
defer func() {
data := map[string]interface{}{}
if err != nil {
data["error"] = err.Error()
log.Error(err.Error())
} else {
data["error"] = nil
}
msg, _ := json.Marshal(data)
ws.Write(msg)
ws.Close()
}()
req := ws.Request()
t := context.GetAuthToken(req)
if t == nil {
err = fmt.Errorf("wslogs: no token")
return
}
if t.GetAppName() != app.InternalAppName {
err = fmt.Errorf("wslogs: invalid token app name: %q", t.GetAppName())
return
}
err = scanLogs(ws)
if err != nil {
return
}
}
func (c *Client) BindApp(instance *ServiceInstance, app bind.App) (map[string]string, error) {
log.Debugf("Calling bind of instance %q and %q app at %q API",
instance.Name, app.GetName(), instance.ServiceName)
var resp *http.Response
params := map[string][]string{
"app-host": {app.GetIp()},
}
resp, err := c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind-app", "POST", params)
if resp != nil && resp.StatusCode == http.StatusNotFound {
resp, err = c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind", "POST", params)
}
if err != nil {
if m, _ := regexp.MatchString("", err.Error()); m {
return nil, fmt.Errorf("%s api is down.", instance.Name)
}
return nil, err
}
if err == nil && resp.StatusCode < 300 {
var result map[string]string
err = c.jsonFromResponse(resp, &result)
if err != nil {
return nil, err
}
return result, nil
}
if resp.StatusCode == http.StatusPreconditionFailed {
return nil, &errors.HTTP{Code: resp.StatusCode, Message: "You cannot bind any app to this service instance because it is not ready yet."}
}
msg := fmt.Sprintf("Failed to bind the instance %q to the app %q: %s", instance.Name, app.GetName(), c.buildErrorMessage(err, resp))
log.Error(msg)
return nil, &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
func (c *Client) BindUnit(instance *ServiceInstance, app bind.App, unit bind.Unit) error {
log.Debugf("Calling bind of instance %q and %q unit at %q API",
instance.Name, unit.GetIp(), instance.ServiceName)
var resp *http.Response
params := map[string][]string{
"app-host": {app.GetIp()},
"unit-host": {unit.GetIp()},
}
resp, err := c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind", "POST", params)
if err != nil {
if m, _ := regexp.MatchString("", err.Error()); m {
return fmt.Errorf("%s api is down.", instance.Name)
}
return err
}
if resp.StatusCode == http.StatusPreconditionFailed {
return &errors.HTTP{Code: resp.StatusCode, Message: "You cannot bind any app to this service instance because it is not ready yet."}
}
if resp.StatusCode > 299 {
msg := fmt.Sprintf("Failed to bind the instance %q to the unit %q: %s", instance.Name, unit.GetIp(), c.buildErrorMessage(err, resp))
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
return nil
}
func (fn AdminRequiredHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
setVersionHeaders(w)
defer func() {
if r.Body != nil {
r.Body.Close()
}
}()
fw := io.FlushingWriter{ResponseWriter: w}
header := r.Header.Get("Authorization")
if header == "" {
http.Error(&fw, "You must provide the Authorization header", http.StatusUnauthorized)
} else if t, err := app.AuthScheme.Auth(header); err != nil {
http.Error(&fw, "Invalid token", http.StatusUnauthorized)
} else if user, err := t.User(); err != nil || !user.IsAdmin() {
http.Error(&fw, "Forbidden", http.StatusForbidden)
} else if err = fn(&fw, r, t); err != nil {
code := http.StatusInternalServerError
if e, ok := err.(*errors.HTTP); ok {
code = e.Code
}
if fw.Wrote() {
fmt.Fprintln(&fw, err)
} else {
http.Error(&fw, err.Error(), code)
}
log.Error(err.Error())
}
}
// handle is the function called by the queue handler on each message.
func handle(msg *queue.Message) {
switch msg.Action {
case RegenerateApprcAndStart:
fallthrough
case regenerateApprc:
if len(msg.Args) < 1 {
log.Errorf("Error handling %q: this action requires at least 1 argument.", msg.Action)
return
}
app, err := ensureAppIsStarted(msg)
if err != nil {
log.Error(err.Error())
return
}
err = app.SerializeEnvVars()
if err != nil {
log.Error(err.Error())
}
fallthrough
case startApp:
if msg.Action == regenerateApprc {
break
}
if len(msg.Args) < 1 {
log.Errorf("Error handling %q: this action requires at least 1 argument.", msg.Action)
}
app, err := ensureAppIsStarted(msg)
if err != nil {
log.Error(err.Error())
return
}
err = app.Restart(ioutil.Discard)
if err != nil {
log.Errorf("Error handling %q. App failed to start:\n%s.", msg.Action, err)
return
}
case BindService:
err := bindUnit(msg)
if err != nil {
log.Error(err.Error())
return
}
default:
log.Errorf("Error handling %q: invalid action.", msg.Action)
}
}
func getELBEndpoint() *elb.ELB {
access, err := config.GetString("aws:access-key-id")
if err != nil {
log.Error(err.Error())
}
secret, err := config.GetString("aws:secret-access-key")
if err != nil {
log.Error(err.Error())
}
endpoint, err := config.GetString("juju:elb-endpoint")
if err != nil {
log.Error(err.Error())
}
auth := aws.Auth{AccessKey: access, SecretKey: secret}
region := aws.Region{ELBEndpoint: endpoint}
return elb.New(auth, region)
}
// ServerURL returns the URL to Gandalf API.
func ServerURL() string {
server, err := config.GetString("git:api-server")
if err != nil {
log.Error("git:api-server config not found")
panic(err)
}
return server
}
func addLogs(ws *websocket.Conn) {
var err error
defer func() {
data := map[string]interface{}{}
if err != nil {
data["error"] = err.Error()
log.Error(err.Error())
} else {
data["error"] = nil
}
msg, _ := json.Marshal(data)
ws.Write(msg)
ws.Close()
}()
req := ws.Request()
t := context.GetAuthToken(req)
if t == nil {
err = fmt.Errorf("wslogs: no token")
return
}
if t.GetAppName() != app.InternalAppName {
err = fmt.Errorf("wslogs: invalid token app name: %q", t.GetAppName())
return
}
dispatcher := app.NewlogDispatcher()
scanner := bufio.NewScanner(ws)
for scanner.Scan() {
var entry app.Applog
data := bytes.TrimSpace(scanner.Bytes())
if len(data) == 0 {
continue
}
err = json.Unmarshal(data, &entry)
if err != nil {
dispatcher.Stop()
err = fmt.Errorf("wslogs: parsing log line %q: %s", string(data), err)
return
}
err = dispatcher.Send(&entry)
if err != nil {
// Do not disconnect by returning here, dispatcher will already
// retry db connection and we gain nothing by ending the WS
// connection.
log.Errorf("wslogs: error storing log: %s", err)
}
}
err = dispatcher.Stop()
if err != nil {
err = fmt.Errorf("wslogs: error storing log: %s", err)
return
}
err = scanner.Err()
if err != nil {
err = fmt.Errorf("wslogs: waiting for log data: %s", err)
return
}
}
func addLogs(ws *websocket.Conn) {
var err error
defer func() {
data := map[string]interface{}{}
if err != nil {
data["error"] = err.Error()
log.Error(err.Error())
} else {
data["error"] = nil
}
msg, _ := json.Marshal(data)
ws.Write(msg)
ws.Close()
}()
req := ws.Request()
t := context.GetAuthToken(req)
if t == nil {
err = fmt.Errorf("wslogs: no token")
return
}
if t.GetAppName() != app.InternalAppName {
err = fmt.Errorf("wslogs: invalid token app name: %q", t.GetAppName())
return
}
logCh, errCh := app.LogReceiver()
scanner := bufio.NewScanner(ws)
for scanner.Scan() {
var entry app.Applog
data := bytes.TrimSpace(scanner.Bytes())
if len(data) == 0 {
continue
}
err = json.Unmarshal(data, &entry)
if err != nil {
close(logCh)
err = fmt.Errorf("wslogs: parsing log line %q: %s", string(data), err)
return
}
select {
case logCh <- &entry:
case err := <-errCh:
close(logCh)
err = fmt.Errorf("wslogs: storing log: %s", err)
return
}
}
close(logCh)
err = scanner.Err()
if err != nil {
err = fmt.Errorf("wslogs: waiting for log data: %s", err)
return
}
err = <-errCh
if err != nil {
err = fmt.Errorf("wslogs: storing log: %s", err)
}
}
// RemoveUnit removes a unit by its Name.
func (app *App) RemoveUnit(name string) error {
unit, err := app.findUnitByName(name)
if err != nil {
return err
}
if err = Provisioner.RemoveUnit(app, unit.Name); err != nil {
log.Error(err.Error())
}
return app.unbindUnit(unit)
}
// RemoveUnit removes a unit by its InstanceId or Name.
//
// Will search first by InstanceId, if no instance is found, then tries to
// search using the unit name, then calls the removal function from provisioner
//
// Returns an error in case of failure.
func (app *App) RemoveUnit(id string) error {
unit, i, err := app.findUnitByID(id)
if err != nil {
return err
}
if err = Provisioner.RemoveUnit(app, unit.GetName()); err != nil {
log.Error(err.Error())
}
return app.removeUnitByIdx(i, unit)
}
func (c *Client) Destroy(instance *ServiceInstance) error {
log.Debugf("Attempting to call destroy of service instance %q at %q api", instance.Name, instance.ServiceName)
resp, err := c.issueRequest("/resources/"+instance.GetIdentifier(), "DELETE", nil)
if err == nil && resp.StatusCode > 299 {
msg := "Failed to destroy the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
return err
}
func ensureContainersStarted(p DockerProvisioner, w io.Writer, relaunch bool, nodes ...cluster.Node) error {
if w == nil {
w = ioutil.Discard
}
confNames, err := scopedconfig.FindAllScopedConfigNames(nodeContainerCollection)
if err != nil {
return err
}
if len(nodes) == 0 {
nodes, err = p.Cluster().UnfilteredNodes()
if err != nil {
return err
}
}
errChan := make(chan error, len(nodes))
wg := sync.WaitGroup{}
log.Debugf("[node containers] recreating %d containers", len(nodes))
recreateContainer := func(node *cluster.Node, confName string) {
defer wg.Done()
pool := node.Metadata["pool"]
containerConfig, confErr := LoadNodeContainer(pool, confName)
if confErr != nil {
errChan <- confErr
return
}
log.Debugf("[node containers] recreating container %q in %s [%s]", confName, node.Address, pool)
fmt.Fprintf(w, "relaunching node container %q in the node %s [%s]\n", confName, node.Address, pool)
confErr = containerConfig.create(node.Address, pool, p, relaunch)
if confErr != nil {
msg := fmt.Sprintf("[node containers] failed to create container in %s [%s]: %s", node.Address, pool, confErr)
log.Error(msg)
errChan <- errors.New(msg)
}
}
for i := range nodes {
wg.Add(1)
go func(node *cluster.Node) {
defer wg.Done()
for j := range confNames {
wg.Add(1)
go recreateContainer(node, confNames[j])
}
}(&nodes[i])
}
wg.Wait()
close(errChan)
var allErrors []string
for err = range errChan {
allErrors = append(allErrors, err.Error())
}
if len(allErrors) == 0 {
return nil
}
return fmt.Errorf("multiple errors: %s", strings.Join(allErrors, ", "))
}
// getPrivateDns returns the private dns for an instance.
func (h *instanceAgentsConfigHealer) getPrivateDns(instanceId string) (string, error) {
log.Debugf("getting dns for %s", instanceId)
resp, err := h.ec2().Instances([]string{instanceId}, nil)
if err != nil {
log.Errorf("error in gettings dns for %s", instanceId)
log.Error(err.Error())
return "", err
}
dns := resp.Reservations[0].Instances[0].PrivateDNSName
return dns, nil
}
func (c *Client) Unbind(instance *ServiceInstance, unit bind.Unit) error {
log.Debug("Attempting to call unbind of service instance " + instance.Name + " and unit " + unit.GetIp() + " at " + instance.ServiceName + " api")
var resp *http.Response
url := "/resources/" + instance.Name + "/hostname/" + unit.GetIp()
resp, err := c.issueRequest(url, "DELETE", nil)
if err == nil && resp.StatusCode > 299 {
msg := fmt.Sprintf("Failed to unbind (%q): %s", url, c.buildErrorMessage(err, resp))
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
return err
}
func (p *dockerProvisioner) HandleMoveErrors(moveErrors chan error, writer io.Writer) error {
hasError := false
for err := range moveErrors {
errMsg := fmt.Sprintf("Error moving container: %s", err.Error())
log.Error(errMsg)
fmt.Fprintf(writer, "%s\n", errMsg)
hasError = true
}
if hasError {
return containerMovementErr
}
return nil
}
func (c *Client) Destroy(instance *ServiceInstance) error {
log.Debugf("Attempting to call destroy of service instance %q at %q api", instance.Name, instance.ServiceName)
resp, err := c.issueRequest("/resources/"+instance.GetIdentifier(), "DELETE", nil)
if err == nil && resp.StatusCode > 299 {
if resp.StatusCode == http.StatusNotFound {
return ErrInstanceNotFoundInAPI
}
msg := "Failed to destroy the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
return errors.New(msg)
}
return err
}
// getImage returns the image name or id from an app.
// when the container image is empty is returned the platform image.
// when a deploy is multiple of 10 is returned the platform image.
func getImage(app provision.App) string {
c, err := getOneContainerByAppName(app.GetName())
if err != nil || c.Image == "" {
return assembleImageName(app.GetPlatform())
}
if usePlatformImage(app) {
err := removeImage(c.Image)
if err != nil {
log.Error(err.Error())
}
return assembleImageName(app.GetPlatform())
}
return c.Image
}
func (c *Client) UnbindApp(instance *ServiceInstance, app bind.App) error {
log.Debugf("Calling unbind of service instance %q and app %q at %q", instance.Name, app.GetName(), instance.ServiceName)
var resp *http.Response
url := "/resources/" + instance.GetIdentifier() + "/bind-app"
params := map[string][]string{
"app-host": {app.GetIp()},
}
resp, err := c.issueRequest(url, "DELETE", params)
if err == nil && resp.StatusCode > 299 {
msg := fmt.Sprintf("Failed to unbind (%q): %s", url, c.buildErrorMessage(err, resp))
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
return err
}
func errorHandlingMiddleware(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
next(w, r)
err := context.GetRequestError(r)
if err != nil {
code := http.StatusInternalServerError
if e, ok := err.(*errors.HTTP); ok {
code = e.Code
}
flushing, ok := w.(*io.FlushingWriter)
if ok && flushing.Wrote() {
fmt.Fprintln(w, err)
} else {
http.Error(w, err.Error(), code)
}
log.Error(err.Error())
}
}
func (c *Client) Create(instance *ServiceInstance) error {
var err error
log.Debug("Attempting to call creation of service instance " + instance.Name + " at " + instance.ServiceName + " api")
var resp *http.Response
params := map[string][]string{
"name": {instance.Name},
}
if instance.PlanName != "" {
params["plan"] = []string{instance.PlanName}
}
if resp, err = c.issueRequest("/resources", "POST", params); err == nil && resp.StatusCode < 300 {
return nil
}
msg := "Failed to create the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
func (fn Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
setVersionHeaders(w)
defer func() {
if r.Body != nil {
r.Body.Close()
}
}()
fw := io.FlushingWriter{ResponseWriter: w}
if err := fn(&fw, r); err != nil {
if fw.Wrote() {
fmt.Fprintln(&fw, err)
} else {
http.Error(&fw, err.Error(), http.StatusInternalServerError)
}
log.Error(err.Error())
}
}
// getImage returns the image name or id from an app.
// when the container image is empty is returned the platform image.
// when a deploy is multiple of 10 is returned the platform image.
func getImage(app provision.App) string {
var c container
coll := collection()
defer coll.Close()
coll.Find(bson.M{"appname": app.GetName()}).One(&c)
if c.Image == "" {
return assembleImageName(app.GetPlatform())
}
if usePlatformImage(app) {
err := removeImage(c.Image)
if err != nil {
log.Error(err.Error())
}
return assembleImageName(app.GetPlatform())
}
return c.Image
}