// Get returns the frontend record for the given id.
// If the ID is not found, an IDNotFoundError is returned.
func (eb *etcdBackend) Get(id string) (api.FrontendRecord, error) {
if err := validateID(id); err != nil {
return api.FrontendRecord{}, maskAny(err)
}
etcdPath := path.Join(eb.prefix, frontEndPrefix, id)
kAPI := client.NewKeysAPI(eb.client)
options := &client.GetOptions{
Recursive: false,
Sort: false,
}
resp, err := kAPI.Get(context.Background(), etcdPath, options)
if isEtcdError(err, client.ErrorCodeKeyNotFound) {
return api.FrontendRecord{}, maskAny(errgo.WithCausef(nil, api.IDNotFoundError, "ID '%s' not found", id))
}
if err != nil {
eb.Logger.Warningf("ETCD error in Get: %#v", err)
return api.FrontendRecord{}, maskAny(err)
}
if resp.Node == nil {
return api.FrontendRecord{}, maskAny(errgo.WithCausef(nil, api.IDNotFoundError, "ID '%s' not found", id))
}
rawJSON := resp.Node.Value
record := api.FrontendRecord{}
if err := json.Unmarshal([]byte(rawJSON), &record); err != nil {
return api.FrontendRecord{}, maskAny(fmt.Errorf("Cannot unmarshal registration of %s", id))
}
return record, nil
}
// Restart behaves as `systemctl restart <unit>`
func (sdc *SystemdClient) Restart(unit string) error {
sdc.Logger.Debugf("restarting %s", unit)
conn, err := dbus.New()
if err != nil {
return maskAny(err)
}
responseChan := make(chan string, 1)
if _, err := conn.RestartUnit(unit, "replace", responseChan); err != nil {
sdc.Logger.Errorf("restarting %s failed: %#v", unit, err)
return maskAny(err)
}
select {
case res := <-responseChan:
switch res {
case "done":
return nil
case "failed":
// We need a start considered to be failed, when the unit is already running.
return nil
case "canceled", "timeout", "dependency", "skipped":
return maskAny(errgo.WithCausef(nil, SystemdError, res))
default:
// that should never happen
sdc.Logger.Errorf("unexpected systemd response: '%s'", res)
return maskAny(errgo.WithCausef(nil, SystemdError, res))
}
case <-time.After(jobTimeout):
return maskAny(errgo.WithCausef(nil, SystemdError, "job timeout"))
}
return nil
}
// Validate checks the given object for invalid values.
func (r FrontendSelectorRecord) Validate() error {
if r.Weight < 0 || r.Weight > 100 {
return maskAny(errgo.WithCausef(nil, ValidationError, "weight must be between 0-100"))
}
if r.ServicePort < 0 || r.ServicePort > maxPort {
return maskAny(errgo.WithCausef(nil, ValidationError, "port must be between 0-%d", maxPort))
}
if r.FrontendPort < 0 || r.FrontendPort > maxPort {
return maskAny(errgo.WithCausef(nil, ValidationError, "frontend-port must be between 0-%d", maxPort))
}
if r.Domain == "" && r.PathPrefix == "" && r.FrontendPort == 0 {
return maskAny(errgo.WithCausef(nil, ValidationError, "domain, path-prefix or frontend-port must be set"))
}
for _, ur := range r.Users {
if err := ur.Validate(); err != nil {
return maskAny(err)
}
}
for _, rr := range r.RewriteRules {
if err := rr.Validate(); err != nil {
return maskAny(err)
}
}
return nil
}
// Check for errors
func (j *Job) Validate() error {
if err := j.Name.Validate(); err != nil {
return maskAny(err)
}
if len(j.Groups) == 0 {
return maskAny(errgo.WithCausef(nil, ValidationError, "job has no groups"))
}
for i, tg := range j.Groups {
err := tg.Validate()
if err != nil {
return maskAny(err)
}
for k := i + 1; k < len(j.Groups); k++ {
if j.Groups[k].Name == tg.Name {
return maskAny(errgo.WithCausef(nil, ValidationError, "job has duplicate taskgroup %s", tg.Name))
}
}
}
if err := j.Constraints.Validate(); err != nil {
return maskAny(err)
}
if err := j.Dependencies.Validate(); err != nil {
return maskAny(err)
}
return nil
}
// Check for configuration errors
func (tg *TaskGroup) Validate() error {
if err := tg.Name.Validate(); err != nil {
return maskAny(err)
}
if tg.Count <= 0 {
return maskAny(errgo.WithCausef(nil, ValidationError, "group %s count <= 0", tg.Name))
}
if len(tg.Tasks) == 0 {
return maskAny(errgo.WithCausef(nil, ValidationError, "group %s has no tasks", tg.Name))
}
for i, t := range tg.Tasks {
err := t.Validate()
if err != nil {
return maskAny(err)
}
for j := i + 1; j < len(tg.Tasks); j++ {
if tg.Tasks[j].Name == t.Name {
return maskAny(errgo.WithCausef(nil, ValidationError, "group %s has duplicate task %s", tg.Name, t.Name))
}
}
}
if err := tg.Constraints.Validate(); err != nil {
return maskAny(err)
}
if err := tg.RestartPolicy.Validate(); err != nil {
return maskAny(err)
}
return nil
}
// Validate checks the given object for invalid values.
func (r UserRecord) Validate() error {
if r.Name == "" {
return maskAny(errgo.WithCausef(nil, ValidationError, "name must be set"))
}
if r.PasswordHash == "" {
return maskAny(errgo.WithCausef(nil, ValidationError, "pwhash must be set"))
}
return nil
}
// Validate checks the values of the given secret.
// If ok, return nil, otherwise returns an error.
func (s *Secret) Validate() error {
if s.Path == "" {
return maskAny(errgo.WithCausef(nil, ValidationError, "path is empty"))
}
if s.Environment == "" && s.File == "" {
return maskAny(errgo.WithCausef(nil, ValidationError, "environment and file is empty"))
}
return nil
}
// Validate checks the given object for invalid values.
func (r RewriteRule) Validate() error {
if r.PathPrefix == "" && r.RemovePathPrefix == "" && r.Domain == "" {
return maskAny(errgo.WithCausef(nil, ValidationError, "at least 1 property must be set"))
}
if r.PathPrefix != "" && r.RemovePathPrefix != "" {
return maskAny(errgo.WithCausef(nil, ValidationError, "path-prefix and remove-path-prefix cannot be set both"))
}
return nil
}
// SendMessage reads the configuration file, and posts a message about Kocho's invocation to Slack.
func SendMessage(version, build string) error {
expanded, err := homedir.Expand(configPath)
if err != nil {
return err
}
if _, err := os.Stat(expanded); os.IsNotExist(err) {
return errgo.Mask(ErrNotConfigured, errgo.Any)
}
slackConfiguration := SlackConfiguration{
NotificationUsername: "******",
EmojiIcon: ":robot_face:",
}
configFile, err := os.Open(expanded)
if err != nil {
return errgo.WithCausef(err, ErrInvalidConfiguration, "couldn't open Slack configuration file")
}
defer configFile.Close()
if err := json.NewDecoder(configFile).Decode(&slackConfiguration); err != nil {
return errgo.WithCausef(err, ErrInvalidConfiguration, "couldn't decode Slack configuration")
}
client := slack.New(slackConfiguration.Token)
params := slack.PostMessageParameters{}
params.Attachments = []slack.Attachment{
slack.Attachment{
Color: "#2484BE",
Text: fmt.Sprintf("*Kocho*: %s ran `%s`", slackConfiguration.Username, strings.Join(os.Args, " ")),
Fields: []slack.AttachmentField{
slack.AttachmentField{
Title: "Kocho Version",
Value: version,
Short: true,
},
slack.AttachmentField{
Title: "Kocho Build",
Value: build,
Short: true,
},
},
MarkdownIn: []string{"text"},
},
}
params.Username = slackConfiguration.NotificationUsername
params.IconEmoji = slackConfiguration.EmojiIcon
if _, _, err := client.PostMessage(slackConfiguration.NotificationChannel, "", params); err != nil {
return err
}
return nil
}
func runKillInstance(args []string) (exit int) {
if len(args) != 2 {
return exitError("wrong number of arguments. Usage: kocho kill-instance <swarm> <instance>")
}
swarmName := args[0]
instanceID := args[1]
s, err := swarmService.Get(swarmName, swarm.AWS)
if err != nil {
return exitError(fmt.Sprintf("couldn't get instances of swarm: %s", swarmName), err)
}
instances, err := s.GetInstances()
if err != nil {
return exitError(err)
}
killableInstance, err := swarmtypes.FindInstanceById(instances, instanceID)
if err != nil {
return exitError(errgo.WithCausef(err, nil, "failed to find provided instance: %s", instanceID))
}
runningInstances := swarmtypes.FilterInstanceById(instances, instanceID)
if len(runningInstances) == 0 {
return exitError(errgo.Newf("no more instances left in swarm %s. Cannot update Fleet DNS entry", swarmName))
}
if !ignoreQuorumCheck {
etcdQuorumID, err := ssh.GetEtcd2MemberName(killableInstance.PublicIPAddress)
if err != nil {
return exitError(errgo.WithCausef(err, nil, "ssh: failed to check quorum member list: %v", err))
}
if etcdQuorumID != "" {
return exitError(errgo.Newf("Instance %s seems to be part of the etcd quorum. Please remove it beforehand. See %s", killableInstance.Id, etcdDocsLink))
}
}
if err = s.KillInstance(killableInstance); err != nil {
return exitError(errgo.WithCausef(err, nil, "failed to kill instance: %s", instanceID))
}
if changed, err := dns.Update(dnsService, viperConfig.getDNSNamingPattern(), s, runningInstances); err != nil {
return exitError(errgo.WithCausef(err, nil, "failed to update dns records"))
} else if !changed {
return exitError(errgo.Newf("DNS not changed. Couldn't find valid publid DNS name"))
}
fmt.Printf(killInstanceSuccessMessage, killableInstance.Id, etcdDocsLink)
fireNotification()
return 0
}
// Validate checks the values of the given constraint.
// If ok, return nil, otherwise returns an error.
func (c Constraint) Validate() error {
if c.Attribute == "" {
return errgo.WithCausef(nil, ValidationError, "attribute cannot be empty")
}
switch c.Operator {
case "", OperatorEqual, OperatorNotEqual:
// Ok
default:
return errgo.WithCausef(nil, ValidationError, "unknown operator '%s'", c.Operator)
}
return nil
}
// Validate checks the values of the given frontend.
// If ok, return nil, otherwise returns an error.
func (f PrivateFrontEnd) Validate() error {
if f.Weight < 0 || f.Weight > 100 {
return errgo.WithCausef(nil, ValidationError, "weight must be between 0 and 100")
}
switch f.Mode {
case "", "http", "tcp":
// OK
default:
return errgo.WithCausef(nil, ValidationError, "mode must be http or tcp")
}
return nil
}
func (l Link) Validate() error {
if err := l.Target.Validate(); err != nil {
return maskAny(err)
}
if err := l.Type.Validate(); err != nil {
return maskAny(err)
}
if len(l.Ports) == 0 && l.Type.IsTCP() {
return maskAny(errgo.WithCausef(nil, ValidationError, "specify at least one port in a tcp link"))
}
if len(l.Ports) != 0 && !l.Type.IsTCP() {
return maskAny(errgo.WithCausef(nil, ValidationError, "ports are not allowed in non-tcp links"))
}
return nil
}
// parse a private frontend
func (f *PrivateFrontEnd) parse(obj *ast.ObjectType) error {
// Build the frontend
excludedKeys := []string{
"user",
}
defaultValues := map[string]interface{}{
"port": 80,
}
if err := hclutil.Decode(obj, excludedKeys, defaultValues, f); err != nil {
return maskAny(err)
}
if o := obj.List.Filter("user"); len(o.Items) > 0 {
for _, o := range o.Children().Items {
if obj, ok := o.Val.(*ast.ObjectType); ok {
n := o.Keys[0].Token.Value().(string)
u := User{Name: n}
if err := u.parse(obj); err != nil {
return maskAny(err)
}
f.Users = append(f.Users, u)
} else {
return maskAny(errgo.WithCausef(nil, ValidationError, "user of frontend %#v is not an object or array", f))
}
}
}
return nil
}
// GithubLogin performs a standard Github authentication and initializes the vaultClient with the resulting token.
func (s *VaultService) GithubLogin(data GithubLoginData) (*AuthenticatedVaultClient, error) {
// Perform login
vaultClient, address, err := s.newUnsealedClient()
if err != nil {
return nil, maskAny(err)
}
vaultClient.ClearToken()
logical := vaultClient.Logical()
loginData := make(map[string]interface{})
loginData["token"] = data.GithubToken
if data.Mount == "" {
data.Mount = "github"
}
path := fmt.Sprintf("auth/%s/login", data.Mount)
s.log.Debugf("write loginData at %s", address)
if loginSecret, err := logical.Write(path, loginData); err != nil {
return nil, maskAny(err)
} else if loginSecret.Auth == nil {
return nil, maskAny(errgo.WithCausef(nil, VaultError, "missing authentication in secret response"))
} else {
// Use token
vaultClient.SetToken(loginSecret.Auth.ClientToken)
}
// We're done
return s.newAuthenticatedClient(vaultClient), nil
}
// newMutex creates and initializes a new GlobalMutex.
func newMutex(name string, ttl time.Duration, service mutexService) (*GlobalMutex, error) {
if name == "" {
return nil, errgo.WithCausef(nil, InvalidArgumentError, "name empty")
}
if ttl <= 0 {
return nil, errgo.WithCausef(nil, InvalidArgumentError, "ttl <= 0")
}
if service == nil {
return nil, errgo.WithCausef(nil, InvalidArgumentError, "service nil")
}
return &GlobalMutex{
name: name,
ttl: ttl,
service: service,
}, nil
}
// getEnv loads an environment value and returns an error if it is empty.
func (jf *jobFunctions) getEnv(key string) (string, error) {
value := os.Getenv(key)
if value == "" {
return "", errgo.WithCausef(nil, ValidationError, "Missing environment variables '%s'", key)
}
return value, nil
}
// getOpt loads an option with given key and returns an error the option does not exist.
func (jf *jobFunctions) getOpt(key string) (string, error) {
value, ok := jf.options.Get(key)
if !ok {
value, ok = jf.cluster.DefaultOptions.Get(key)
if !ok {
switch key {
case "domain":
return jf.cluster.Domain, nil
case "stack":
return jf.cluster.Stack, nil
case "tunnel":
return jf.cluster.Tunnel, nil
case "instance-count":
return strconv.Itoa(jf.cluster.InstanceCount), nil
default:
return "", errgo.WithCausef(nil, ValidationError, "Missing option '%s'", key)
}
}
}
if result, err := formatOptionValue(value, false); err != nil {
return "", maskAny(err)
} else {
return result, nil
}
}
// GithubLogin performs a standard Github authentication and initializes the vaultClient with the resulting token.
func (s *Vault) GithubLogin(data GithubLoginData) error {
// Read token
var err error
data.GithubToken, err = s.readGithubToken(data)
if err != nil {
return maskAny(err)
}
// Perform login
s.vaultClient.ClearToken()
logical := s.vaultClient.Logical()
loginData := make(map[string]interface{})
loginData["token"] = data.GithubToken
if data.Mount == "" {
data.Mount = "github"
}
path := fmt.Sprintf("auth/%s/login", data.Mount)
if loginSecret, err := logical.Write(path, loginData); err != nil {
return maskAny(err)
} else if loginSecret.Auth == nil {
return maskAny(errgo.WithCausef(nil, VaultError, "missing authentication in secret response"))
} else {
// Use token
s.vaultClient.SetToken(loginSecret.Auth.ClientToken)
}
// We're done
return nil
}
// newUnsealedClient creates the first single vault client that resolves to an unsealed vault instance.
func (s *VaultService) newUnsealedClient() (*api.Client, string, error) {
clients, err := s.newClients()
if err != nil {
return nil, "", maskAny(err)
}
for _, client := range clients {
// Check seal status
status, err := client.Client.Sys().SealStatus()
if err != nil {
s.log.Debugf("vault at %s cannot be reached: %s", client.Address, Describe(err))
continue
} else if status.Sealed {
s.log.Warningf("Vault at %s is sealed", client.Address)
continue
}
// Check leader status
resp, err := client.Client.Sys().Leader()
if err != nil {
s.log.Debugf("vault at %s cannot be reached: %s", client.Address, Describe(err))
continue
} else if resp.HAEnabled && !resp.IsSelf {
s.log.Debugf("vault at %s is not the leader", client.Address)
continue
}
s.log.Debugf("found unsealed vault client at %s", client.Address)
return client.Client, client.Address, nil
}
return nil, "", maskAny(errgo.WithCausef(nil, VaultError, "no unsealed vault instance found"))
}
// MarshalJSON creates a json representation of a given volume
func (v Volume) MarshalJSON() ([]byte, error) {
str := v.String()
if str == "" {
return nil, maskAny(errgo.WithCausef(nil, ValidationError, "invalid type '%s'", v.Type))
}
return json.Marshal(str)
}
// newEngine creates a new Engine for the given task.
func newEngine(t *jobs.Task, ctx generatorContext) (engine.Engine, error) {
provider := extpoints.EngineProviders.Lookup(t.Engine.String())
if provider == nil {
return nil, maskAny(errgo.WithCausef(nil, ValidationError, "unknown engine type '%s'", t.Engine))
}
return provider.NewEngine(ctx.Cluster), nil
}
func formatOptionValue(value interface{}, quote bool) (string, error) {
if s, ok := value.(string); ok {
if quote {
return strconv.Quote(s), nil
}
return s, nil
}
if l, ok := value.([]interface{}); ok {
var result []string
for _, e := range l {
fe, err := formatOptionValue(e, true)
if err != nil {
return "", maskAny(err)
}
result = append(result, fe)
}
return "[" + strings.Join(result, ", ") + "]", nil
}
if m, ok := value.(map[string]interface{}); ok {
var result []string
for k, v := range m {
fv, err := formatOptionValue(v, true)
if err != nil {
return "", maskAny(err)
}
result = append(result, fmt.Sprintf("%s = %s", k, fv))
}
return "{\n" + strings.Join(result, "\n") + "}", nil
}
return "", maskAny(errgo.WithCausef(nil, ValidationError, "Unknown value type: %v", value))
}
// parse a QuarkOptions
func (options *QuarkOptions) parse(obj *ast.ObjectType, c Cluster) error {
// Parse the object
excludeList := []string{
"profile",
}
values, err := decodeIntoMap(obj, excludeList, nil)
if err != nil {
return maskAny(err)
}
options.DefaultValues = values
// Parse profiles
if o := obj.List.Filter("profile"); len(o.Items) > 0 {
for _, o := range o.Children().Items {
if obj, ok := o.Val.(*ast.ObjectType); ok {
p := Profile{}
n := o.Keys[0].Token.Value().(string)
if err := p.parse(obj); err != nil {
return maskAny(err)
}
p.Name = n
options.Profiles = append(options.Profiles, p)
} else {
return maskAny(errgo.WithCausef(nil, ValidationError, "profile is not an object"))
}
}
}
return nil
}
// Add adds a given frontend record with given ID to the list of frontends.
// If the given ID already exists, a DuplicateIDError is returned.
func (eb *etcdBackend) Add(id string, record api.FrontendRecord) error {
if err := validateID(id); err != nil {
return maskAny(err)
}
if err := record.Validate(); err != nil {
return maskAny(err)
}
etcdPath := path.Join(eb.prefix, frontEndPrefix, id)
kAPI := client.NewKeysAPI(eb.client)
options := &client.SetOptions{
PrevExist: client.PrevNoExist,
}
rawJSON, err := json.Marshal(record)
if err != nil {
return maskAny(err)
}
if _, err := kAPI.Set(context.Background(), etcdPath, string(rawJSON), options); isEtcdError(err, client.ErrorCodeNodeExist) {
return maskAny(errgo.WithCausef(nil, api.DuplicateIDError, "Duplicate ID '%s'", id))
} else if err != nil {
eb.Logger.Warningf("ETCD error in Add: %#v", err)
return maskAny(err)
}
return nil
}
// ParseJob takes input from a given reader and parses it into a Job.
func parseJob(input []byte, jf *jobFunctions) (*Job, error) {
// Create a template, add the function map, and parse the text.
tmpl, err := template.New("job").Funcs(jf.Functions()).Parse(string(input))
if err != nil {
return nil, maskAny(err)
}
// Run the template to verify the output.
buffer := &bytes.Buffer{}
err = tmpl.Execute(buffer, jf.Options())
if err != nil {
return nil, maskAny(err)
}
// Parse the input
root, err := hcl.Parse(buffer.String())
if err != nil {
return nil, maskAny(err)
}
// Top-level item should be a list
list, ok := root.Node.(*ast.ObjectList)
if !ok {
return nil, errgo.New("error parsing: root should be an object")
}
// Parse hcl into Job
job := &Job{}
matches := list.Filter("job")
if len(matches.Items) == 0 {
return nil, maskAny(errgo.WithCausef(nil, ValidationError, "'job' stanza not found"))
}
if err := job.parse(matches); err != nil {
return nil, maskAny(err)
}
// Link internal structures
job.prelink()
// Set defaults
job.setDefaults(jf.cluster)
// Replace variables
if err := job.replaceVariables(); err != nil {
return nil, maskAny(err)
}
// Sort internal structures and make final links
job.link()
// Optimize job for cluster
job.optimizeFor(jf.cluster)
// Validate the job
if err := job.Validate(); err != nil {
return nil, maskAny(err)
}
return job, nil
}
// Validate returns an error if the given network type is invalid.
// Returns nil on ok.
func (nt NetworkType) Validate() error {
switch nt {
case NetworkTypeDefault, NetworkTypeHost, NetworkTypeWeave:
return nil
default:
return maskAny(errgo.WithCausef(nil, ValidationError, "unknown network type '%s'", string(nt)))
}
}
// TaskGroup gets a taskgroup by the given name
func (j *Job) TaskGroup(name TaskGroupName) (*TaskGroup, error) {
for _, tg := range j.Groups {
if tg.Name == name {
return tg, nil
}
}
return nil, maskAny(errgo.WithCausef(nil, TaskGroupNotFoundError, name.String()))
}
// Validate checks if a link name follows a valid format
func (lt LinkType) Validate() error {
switch string(lt) {
case "http", "tcp", "":
return nil
default:
return maskAny(errgo.WithCausef(nil, ValidationError, "invalid link type '%s'", string(lt)))
}
}
// Dependency gets a dependency by the given name
func (j *Job) Dependency(name LinkName) (Dependency, error) {
for _, d := range j.Dependencies {
if d.Name == name {
return d, nil
}
}
return Dependency{}, maskAny(errgo.WithCausef(nil, DependencyNotFoundError, name.String()))
}
