func NewFioRunner(c FioConfiguration) (FioRunner, error) {
var err error
if !fioExists() {
return FioRunner{}, errgo.Newf("Cannot locate fio. Looks like it is not installed on your system.")
}
if c.GenerateBandwithLogs || c.GenerateIOPSLogs || c.GenerateLatencyLogs {
if len(c.LogsPrefix) < 1 {
return FioRunner{}, errgo.Newf("You have to set a log file prefix")
}
}
if c.JobDirectory, err = filepath.Abs(c.JobDirectory); err != nil {
return FioRunner{}, errgo.Mask(err)
}
if c.WorkingDirectory, err = filepath.Abs(c.WorkingDirectory); err != nil {
return FioRunner{}, errgo.Mask(err)
}
if c.LogsDirectory, err = filepath.Abs(c.LogsDirectory); err != nil {
return FioRunner{}, errgo.Mask(err)
}
return FioRunner{
conf: c,
}, nil
}
func parseDockerPort(input string, dp *Port) error {
s := strings.Split(input, "/")
switch len(s) {
case 1:
dp.Port = s[0]
dp.Protocol = protocolTCP
case 2:
dp.Port = s[0]
dp.Protocol = s[1]
default:
return errgo.Newf("Invalid format, must be either <port> or <port>/<prot>, got '%s'", input)
}
if parsedPort, err := strconv.Atoi(dp.Port); err != nil {
return errgo.Notef(err, "Port must be a number, got '%s'", dp.Port)
} else if parsedPort < 1 || parsedPort > 65535 {
return errgo.Notef(err, "Port must be a number between 1 and 65535, got '%s'", dp.Port)
}
switch dp.Protocol {
case "":
return errgo.Newf("Protocol must not be empty.")
case protocolUDP, protocolTCP:
return nil
default:
return errgo.Newf("Unknown protocol: '%s' in '%s'", dp.Protocol, input)
}
}
func (r FioRunner) createWorkingDirectory() error {
f, err := os.Open(r.conf.WorkingDirectory)
if err != nil {
if os.IsNotExist(err) {
os.Mkdir(r.conf.WorkingDirectory, 0755)
return nil
} else {
return errgo.Mask(err)
}
}
if fi, err := f.Stat(); err != nil {
return errgo.Mask(err)
} else if !fi.IsDir() {
return errgo.Newf("Working directory '%s' exists but appears to be a file.", r.conf.WorkingDirectory)
}
if fis, err := f.Readdir(0); err != nil {
return errgo.Mask(err)
} else if len(fis) > 0 {
return errgo.Newf("Working directory '%s' appears to be not empty.", r.conf.WorkingDirectory)
}
return nil
}
func runKillInstance(args []string) (exit int) {
if len(args) != 2 {
return exitError("wrong number of arguments. Usage: kocho kill-instance <swarm> <instance>")
}
swarmName := args[0]
instanceID := args[1]
s, err := swarmService.Get(swarmName, swarm.AWS)
if err != nil {
return exitError(fmt.Sprintf("couldn't get instances of swarm: %s", swarmName), err)
}
instances, err := s.GetInstances()
if err != nil {
return exitError(err)
}
killableInstance, err := swarmtypes.FindInstanceById(instances, instanceID)
if err != nil {
return exitError(errgo.WithCausef(err, nil, "failed to find provided instance: %s", instanceID))
}
runningInstances := swarmtypes.FilterInstanceById(instances, instanceID)
if len(runningInstances) == 0 {
return exitError(errgo.Newf("no more instances left in swarm %s. Cannot update Fleet DNS entry", swarmName))
}
if !ignoreQuorumCheck {
etcdQuorumID, err := ssh.GetEtcd2MemberName(killableInstance.PublicIPAddress)
if err != nil {
return exitError(errgo.WithCausef(err, nil, "ssh: failed to check quorum member list: %v", err))
}
if etcdQuorumID != "" {
return exitError(errgo.Newf("Instance %s seems to be part of the etcd quorum. Please remove it beforehand. See %s", killableInstance.Id, etcdDocsLink))
}
}
if err = s.KillInstance(killableInstance); err != nil {
return exitError(errgo.WithCausef(err, nil, "failed to kill instance: %s", instanceID))
}
if changed, err := dns.Update(dnsService, viperConfig.getDNSNamingPattern(), s, runningInstances); err != nil {
return exitError(errgo.WithCausef(err, nil, "failed to update dns records"))
} else if !changed {
return exitError(errgo.Newf("DNS not changed. Couldn't find valid publid DNS name"))
}
fmt.Printf(killInstanceSuccessMessage, killableInstance.Id, etcdDocsLink)
fireNotification()
return 0
}
// getAttributeValuesForWebsites creates a map where the key is the attribute ID and
// each part of the StringEntities slice are the full attribute values for a website ID.
func getAttributeValuesForWebsites(ctx *context) map[string][]codegen.StringEntities {
var tws store.TableWebsiteSlice
tws.Load(ctx.dbrConn.NewSession(nil), func(sb *dbr.SelectBuilder) *dbr.SelectBuilder {
return sb.Where("website_id > 0")
})
// key contains the attributeID as a string
var aws = make(map[string][]codegen.StringEntities)
tew, err := ctx.aat.TableEavWebsite()
codegen.LogFatal(err)
if tew != nil { // only for those who have a wbesite specific table
for _, w := range tws {
aCollection, err := codegen.LoadStringEntities(ctx.db, getAttrSelect(ctx, w.WebsiteID))
codegen.LogFatal(err)
for _, row := range aCollection {
if aid, ok := row["attribute_id"]; ok {
if nil == aws[aid] {
aws[aid] = make([]codegen.StringEntities, 0, 200) // up to 200 websites at once
}
aws[aid] = append(aws[aid], row)
} else {
codegen.LogFatal(errgo.Newf("Column attribute_id not found in collection %#v\n", aCollection))
}
}
}
}
return aws
}
// NewConnection instantiates a Connection for a given database/sql connection
// and event receiver
func NewConnection(opts ...ConnectionOption) (*Connection, error) {
c := &Connection{
dn: DriverNameMySQL,
EventReceiver: nullReceiver,
}
c.ApplyOpts(opts...)
switch c.dn {
case DriverNameMySQL:
default:
return nil, errgo.Newf("unsupported driver: %s", c.dn)
}
if c.DB != nil {
return c, nil
}
if c.dsn != "" {
var err error
if c.DB, err = sql.Open(c.dn, c.dsn); err != nil {
return nil, err
}
}
return c, nil
}
// Create creates and returns a Swarm, given a name for the swarm, a ProviderType, and CreateFlags.
func (srv *Service) Create(name string, providerType ProviderType, flags swarmtypes.CreateFlags) (*Swarm, error) {
p, err := srv.providers.GetByType(providerType)
if err != nil {
return nil, err
}
switch providerType {
case AWS:
if flags.AWSCreateFlags == nil {
return nil, errgo.Newf("AWSCreateFlags must be provided")
}
}
var cfg string
if flags.UseIgnition {
cfg, err = createIgnitionConfig(flags)
} else {
cfg, err = createCloudConfig(flags)
}
if err != nil {
return nil, err
}
swarm, err := p.CreateSwarm(name, flags, cfg)
if err != nil {
return nil, err
}
return createSwarm(swarm), nil
}
// GenerateToken creates a new JSON web token. The claims argument will be
// assigned after the registered claim names exp and iat have been set.
// If EnableJTI is false the returned argument jti is empty.
// For details of the registered claim names please see
// http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html#rfc.section.4.1
func (a *AuthManager) GenerateToken(claims map[string]interface{}) (token, jti string, err error) {
now := time.Now()
t := jwt.New(a.SigningMethod)
t.Claims["exp"] = now.Add(a.Expire).Unix()
t.Claims["iat"] = now.Unix()
for k, v := range claims {
t.Claims[k] = v
}
if a.EnableJTI && a.JTI != nil {
jti = a.JTI.Get()
t.Claims["jti"] = jti
}
switch t.Method.Alg() {
case jwt.SigningMethodRS256.Alg(), jwt.SigningMethodRS384.Alg(), jwt.SigningMethodRS512.Alg():
token, err = t.SignedString(a.rsapk)
case jwt.SigningMethodES256.Alg(), jwt.SigningMethodES384.Alg(), jwt.SigningMethodES512.Alg():
token, err = t.SignedString(a.ecdsapk)
case jwt.SigningMethodHS256.Alg(), jwt.SigningMethodHS384.Alg(), jwt.SigningMethodHS512.Alg():
token, err = t.SignedString(a.password)
default:
return "", "", errgo.Newf("GenerateToken: Unknown algorithm %s", t.Method.Alg())
}
return
}
// UnmarshalJSON decodes a pair from JSON
func (p *Pair) UnmarshalJSON(data []byte) error {
var rawPair = struct {
Value interface{}
Label string
}{}
if err := json.Unmarshal(data, &rawPair); err != nil {
return errgo.Mask(err)
}
p.label = rawPair.Label
switch vt := rawPair.Value.(type) {
case string:
p.NotNull = NotNullString
p.String = vt
case float64: // due to the interface{} above int types do not exists
if math.Abs(vt) < float64(math.MaxInt32) && vt == float64(int64(vt)) { // is int
p.NotNull = NotNullInt
p.Int = int(vt)
} else { // is float
p.NotNull = NotNullFloat64
p.Float64 = vt
}
case bool:
p.NotNull = NotNullBool
p.Bool = vt
default:
return errgo.Newf("Cannot detect type for value '%s' in Pair: %#v", rawPair.Value, rawPair)
}
return nil
}
// Validate checks for duplicated configuration paths in all three hierarchy levels.
func (ss SectionSlice) Validate() error {
if len(ss) == 0 {
return errgo.New("SectionSlice is empty")
}
// @todo try to pick the right strategy between maps and slice depending on the overall size of a full SectionSlice
var pc = make(utils.StringSlice, ss.TotalFields()) // pc path checker
i := 0
for _, s := range ss {
for _, g := range s.Groups {
for _, f := range g.Fields {
arg, err := newArg(Path(s.ID, g.ID, f.ID))
if err != nil {
log.Error("config.SectionSlice.Validate.newArg", "err", err, "s", s, "g", g, "f", f)
}
p := arg.scopePath()
if pc.Include(p) {
return errgo.Newf("Duplicate entry for path %s :: %s", p, ss.ToJSON())
}
pc[i] = p
i++
}
}
}
return nil
}
// GetByCode returns a CSEntityType using the entity code
func (es CSEntityTypeSlice) GetByCode(code string) (*CSEntityType, error) {
for _, e := range es {
if e.EntityTypeCode == code {
return e, nil
}
}
return nil, errgo.Newf("Entity Code %s not found", code)
}
// GetByID returns a CSEntityType using the entity id
func (es CSEntityTypeSlice) GetByID(id int64) (*CSEntityType, error) {
for _, e := range es {
if e.EntityTypeID == id {
return e, nil
}
}
return nil, errgo.Newf("Entity ID %d not found", id)
}
// FindInstanceById searches for an instance in a slice by its id.
func FindInstanceById(instances []Instance, instanceID string) (Instance, error) {
for _, i := range instances {
if i.Id == instanceID {
return i, nil
}
}
return Instance{}, errgo.Newf("couldn't find instance %s", instanceID)
}
func (m entityStoreMap) Get(typeID, storeID int64) (*TableEntityStore, error) {
entityStoreMutex.RLock()
defer entityStoreMutex.RUnlock()
if es, ok := m[getKey(typeID, storeID)]; ok {
return es, nil
}
return nil, errgo.Newf("Key typeID %d storeID %d not found in entity_type map", typeID, storeID)
}
// Append adds a table. Overrides silently existing entries.
func (tm *TableManager) Append(i Index, ts *Table) error {
if ts == nil {
return errgo.Newf("Table pointer cannot be nil for Index %d", i)
}
tm.mu.Lock()
tm.ts[i] = ts
tm.mu.Unlock() // use defer once there are multiple returns
return nil
}
// Implements interface eav.EntityTypeAdditionalAttributeTabler
func (aa *AddAttrTables) TableEavWebsite() (*csdb.Table, error) {
if t, ok := ConfigEntityType[aa.EntityTypeCode]; ok {
if t.TempAdditionalAttributeTableWebsite != "" {
return aa.newTableStructure(t.TempAdditionalAttributeTableWebsite)
}
return nil, nil
}
return nil, errgo.Newf("Table for %s not found", aa.EntityTypeCode)
}
func NewFio2GNUPlotRunner(c Fio2GNUPlotConfiguration) (Fio2GNUPlotRunner, error) {
var err error
if !fio2GNUPlotExists() {
return Fio2GNUPlotRunner{}, errgo.Newf("Cannot locate fio2gnuplot. Looks like it is not installed on your system.")
}
if len(c.LogsPatterns) == 0 {
return Fio2GNUPlotRunner{}, errgo.Newf("You have to configure at least one log file pattern.")
}
if c.LogsDirectory, err = filepath.Abs(c.LogsDirectory); err != nil {
return Fio2GNUPlotRunner{}, errgo.Mask(err)
}
return Fio2GNUPlotRunner{
conf: c,
}, nil
}
func extractSplit(s string) (path string, tp []string, hasVersion bool, err error) {
hasVersion = false
pp := strings.Split(s, "/")
if len(pp) == 1 {
return path, tp, hasVersion, errgo.Newf("Path '%s' contains no slashes", s)
}
switch pp[0] { // add more domains here
case "gopkg.in":
hasVersion = true
break
}
path = strings.Join(pp[:len(pp)-1], "/")
tp = strings.Split(pp[len(pp)-1:][0], ".")
if len(tp) == 1 {
return path, tp, hasVersion, errgo.Newf("Missing . in package.Type")
}
return path, tp, hasVersion, nil
}
func (m entityStoreMap) SetLastIncrementID(typeID, storeID int64, lastIncrementID string) error {
if lastIncrementID == "" {
return ErrLastIncrementIDEmpty
}
entityStoreMutex.Lock()
defer entityStoreMutex.Unlock()
if es, ok := m[getKey(typeID, storeID)]; ok {
es.IncrementLastID.String = lastIncrementID
// @todo now use a goroutine to permanently save that data
}
return errgo.Newf("Failed to save! Key typeID %d storeID %d not found in entity_type map", typeID, storeID)
}
// CreateSwarm creates and returns a Swarm, given a name, CreateFlags and cloud config text.
func (aws AwsProvider) CreateSwarm(name string, flags swarmtypes.CreateFlags, cloudconfigText string) (provider.ProviderSwarm, error) {
if flags.AWSCreateFlags == nil {
return nil, errgo.Newf("invalid arguments to create the swarm: AWSCreateFlags must be provided")
}
var (
cloudformationTmpl string
parametersTmpl string
err error
)
switch flags.Type {
case swarmPrimaryTemplate:
cloudformationTmpl, err = createPrimaryCloudformationTemplate(name, flags.ClusterSize, flags.TemplateDir, flags.AWSCreateFlags.VPCCIDR)
if err != nil {
return nil, errgo.Mask(err)
}
parametersTmpl, err = createPrimaryParametersTemplate(flags.ImageURI, cloudconfigText, flags.MachineType, flags.ClusterSize, flags.TemplateDir, flags.AWSCreateFlags)
if err != nil {
return nil, errgo.Mask(err)
}
case swarmSecondaryTemplate:
cloudformationTmpl, err = createSecondaryCloudformationTemplate(flags.TemplateDir, flags.AWSCreateFlags.VPCCIDR)
if err != nil {
return nil, errgo.Mask(err)
}
parametersTmpl, err = createSecondaryParametersTemplate(flags.ImageURI, cloudconfigText, flags.MachineType, flags.CertificateURI, flags.ClusterSize, flags.TemplateDir, flags.AWSCreateFlags)
if err != nil {
return nil, errgo.Mask(err)
}
case swarmStandaloneTemplate:
cloudformationTmpl, err = createStandaloneCloudformationTemplate(flags.TemplateDir, flags.AWSCreateFlags.VPCCIDR)
if err != nil {
return nil, errgo.Mask(err)
}
parametersTmpl, err = createStandaloneParametersTemplate(flags.ImageURI, cloudconfigText, flags.MachineType, flags.CertificateURI, flags.ClusterSize, flags.TemplateDir, flags.AWSCreateFlags)
if err != nil {
return nil, errgo.Mask(err)
}
}
_, err = aws.cloudformation.CreateStack(name, flags.Type,
cloudformationTmpl,
parametersTmpl,
)
if err != nil {
return nil, errgo.Mask(err)
}
return aws.GetSwarm(name)
}
func (cli *CloudFlareDNS) findZone(ctx context.Context, domain string) (*cloudflare.Zone, error) {
client := cli.client()
zones, err := client.Zones.List(ctx)
if err != nil {
return nil, errgo.Mask(err, errgo.Any)
}
for _, z := range zones {
if z.Name == domain {
return z, nil
}
}
return nil, errgo.Newf("no zone for domain %s found", domain)
}
// Set parses a rawURL and adds it to the cache by its Type. Multiple calls
// with the same type will overwrite existing values.
func (uc *URLCache) Set(t URLType, rawURL string) (*url.URL, error) {
if t >= maxURLTypes {
return nil, errgo.Newf("Unknown Index %d", t)
}
if rawURL == "" {
return nil, errgo.Mask(ErrURLEmpty)
}
u, err := url.Parse(rawURL)
if err != nil {
return nil, errgo.Mask(err)
}
uc.urls[t] = u
return u, nil
}
func serveFile(dir, file string, p Print) error {
if p.FileSystem == nil {
p.FileSystem = http.Dir(dir)
}
f, err := p.FileSystem.Open(file)
if err != nil {
return errgo.Newf("File not found: %s => %s", dir, file)
}
defer f.Close()
fi, _ := f.Stat()
if fi.IsDir() {
file = filepath.Join(file, indexFile)
f, err = p.FileSystem.Open(file)
if err != nil {
return errgo.Newf("Cannot access file: %s", file) // http.StatusForbidden
}
fi, _ = f.Stat()
}
http.ServeContent(p.Response, p.Request, fi.Name(), fi.ModTime(), f)
return nil
}
func (r FioRunner) removeWorkingDirectory() error {
fi, err := os.Stat(r.conf.WorkingDirectory)
if err != nil {
return errgo.Mask(err)
}
if !fi.IsDir() {
return errgo.Newf("Working directory '%s' exists but appears to be a file.", r.conf.WorkingDirectory)
} else {
return os.RemoveAll(r.conf.WorkingDirectory)
}
return nil
}
// SetRSA reads PEM byte data and decodes it and parses the private key.
// Applies the private and the public key to the AuthManager. Password as second
// argument is only required when the private key is encrypted.
// Checks for io.Close and closes the resource. Public key will be derived from
// the private key. Default Signing bits 256.
func SetRSA(privateKey io.Reader, password ...[]byte) OptionFunc {
if cl, ok := privateKey.(io.Closer); ok {
defer func() {
if err := cl.Close(); err != nil { // close file
log.Error("userjwt.RSAKey.ioCloser", "err", err)
}
}()
}
prKeyData, errRA := ioutil.ReadAll(privateKey)
if errRA != nil {
return func(a *AuthManager) {
a.lastError = errgo.Mask(errRA)
}
}
var prKeyPEM *pem.Block
if prKeyPEM, _ = pem.Decode(prKeyData); prKeyPEM == nil {
return func(a *AuthManager) {
a.lastError = errgo.New("Private Key from io.Reader no found")
}
}
var rsaPrivateKey *rsa.PrivateKey
var err error
if x509.IsEncryptedPEMBlock(prKeyPEM) {
if len(password) != 1 || len(password[0]) == 0 {
return func(a *AuthManager) {
a.lastError = errgo.New("Private Key is encrypted but password was not set")
}
}
var dd []byte
var errPEM error
if dd, errPEM = x509.DecryptPEMBlock(prKeyPEM, password[0]); errPEM != nil {
return func(a *AuthManager) {
a.lastError = errgo.Newf("Private Key decryption failed: %s", errPEM.Error())
}
}
rsaPrivateKey, err = x509.ParsePKCS1PrivateKey(dd)
} else {
rsaPrivateKey, err = x509.ParsePKCS1PrivateKey(prKeyPEM.Bytes)
}
return func(a *AuthManager) {
a.SigningMethod = jwt.SigningMethodRS256
a.rsapk = rsaPrivateKey
a.hasKey = true
a.lastError = errgo.Mask(err)
}
}
// keyFunc runs parallel and concurrent
func (a *AuthManager) keyFunc(t *jwt.Token) (interface{}, error) {
if t.Method.Alg() != a.SigningMethod.Alg() {
return nil, log.Error("userjwt.AuthManager.Authenticate.SigningMethod", "err", ErrUnexpectedSigningMethod, "token", t, "method", a.SigningMethod.Alg())
} else {
switch t.Method.Alg() {
case jwt.SigningMethodRS256.Alg(), jwt.SigningMethodRS384.Alg(), jwt.SigningMethodRS512.Alg():
return &a.rsapk.PublicKey, nil
case jwt.SigningMethodES256.Alg(), jwt.SigningMethodES384.Alg(), jwt.SigningMethodES512.Alg():
return &a.ecdsapk.PublicKey, nil
case jwt.SigningMethodHS256.Alg(), jwt.SigningMethodHS384.Alg(), jwt.SigningMethodHS512.Alg():
return a.password, nil
default:
return nil, errgo.Newf("Authenticate: Unknown algorithm %s", t.Method.Alg())
}
}
}
// Test_Log_LogControl_ResetLevels_002 checks for LogControl.ResetLevels to handle
// errors properly on valid error responses using api.WithError.
func Test_Log_LogControl_ResetLevels_002(t *testing.T) {
newLogControl, ts := testMaybeNewLogControlAndServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
response := api.WithError(errgo.Newf("test error"))
if err := json.NewEncoder(w).Encode(response); err != nil {
t.Fatalf("json.NewEncoder returned error: %#v", err)
}
}))
defer ts.Close()
ctx := context.Background()
err := newLogControl.ResetLevels(ctx)
if !IsInvalidAPIResponse(err) {
t.Fatalf("LogControl.ResetLevels NOT returned proper error")
}
}
func sendMsgRecoverable(id int, sl MessageReceiver, a arg) (err error) {
defer func() { // protect ... you'll never know
if r := recover(); r != nil {
if recErr, ok := r.(error); ok {
PkgLog.Debug("config.pubSub.publish.recover.err", "err", recErr)
err = recErr
} else {
PkgLog.Debug("config.pubSub.publish.recover.r", "recover", r)
err = errgo.Newf("%#v", r)
}
// the overall trick here is, that defer will assign a new error to err
// and therefore will overwrite the returned nil value!
}
}()
err = sl.MessageConfig(a.pathLevelAll(), a.scope, a.scopeID)
return
}
// Are the no uncommited changes in this repo?
func checkRepoClean(log *log.Logger, branch string) error {
if st, err := git.Status(log, true); err != nil {
return maskAny(err)
} else if st != "" {
return maskAny(errgo.New("There are uncommited changes"))
}
if err := git.Fetch(log, "origin"); err != nil {
return maskAny(err)
}
if diff, err := git.Diff(log, branch, path.Join("origin", branch)); err != nil {
return maskAny(err)
} else if diff != "" {
return maskAny(errgo.Newf("%s is not in sync with origin", branch))
}
return nil
}
// Parse a Cluster
func (c *Cluster) parse(list *ast.ObjectList) error {
list = list.Children()
if len(list.Items) != 1 {
return fmt.Errorf("only one 'cluster' block allowed")
}
// Get our cluster object
obj := list.Items[0]
// Decode the object
excludeList := []string{
"default-options",
"docker",
"fleet",
"quark",
}
if err := Decode(obj.Val, excludeList, nil, c); err != nil {
return maskAny(err)
}
c.Stack = obj.Keys[0].Token.Value().(string)
// Value should be an object
var listVal *ast.ObjectList
if ot, ok := obj.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return errgo.Newf("cluster '%s' value: should be an object", c.Stack)
}
// If we have quark object, parse it
if o := listVal.Filter("quark"); len(o.Items) > 0 {
for _, o := range o.Elem().Items {
if obj, ok := o.Val.(*ast.ObjectType); ok {
if err := c.QuarkOptions.parse(obj, *c); err != nil {
return maskAny(err)
}
} else {
return maskAny(errgo.WithCausef(nil, ValidationError, "docker of cluster '%s' is not an object", c.Stack))
}
}
}
return nil
}