func (engineProvider) NewEngine(options engines.EngineOptions) (engines.Engine, error) {
var c config
if schematypes.MustMap(configSchema, options.Config, &c) != nil {
return nil, engines.ErrContractViolation
}
// Load user-groups
groups := []*system.Group{}
for _, name := range c.Groups {
group, err := system.FindGroup(name)
if err != nil {
return nil, fmt.Errorf(
"Unable to find system user-group: %s from engine config, error: %s",
name, err,
)
}
groups = append(groups, group)
}
return &engine{
environment: options.Environment,
log: options.Log,
config: c,
groups: groups,
}, nil
}
func (p *plugin) NewTaskPlugin(options plugins.TaskPluginOptions) (
plugins.TaskPlugin, error,
) {
var P payload
if schematypes.MustMap(p.PayloadSchema(), options.Payload, &P) != nil {
return nil, engines.ErrContractViolation
}
// If not always enabled or no options are given then this is disabled
if P.Interactive == nil && !p.config.AlwaysEnabled {
return nil, nil
}
// Extract options
o := opts{}
if P.Interactive != nil {
o = *P.Interactive
}
if o.ArtifactPrefix == "" || p.config.ForbidCustomArtifactPrefix {
o.ArtifactPrefix = p.config.ArtifactPrefix
}
return &taskPlugin{
opts: o,
log: options.Log,
parent: p,
}, nil
}
func (e engineProvider) NewEngine(options engines.EngineOptions) (engines.Engine, error) {
var c configType
if err := schematypes.MustMap(configSchema, options.Config, &c); err != nil {
options.Log.WithError(err).Error("Invalid configuration")
return nil, engines.ErrContractViolation
}
return &engine{
config: &c,
log: options.Log,
}, nil
}
func (e *engine) NewSandboxBuilder(options engines.SandboxOptions) (engines.SandboxBuilder, error) {
var p payload
if schematypes.MustMap(payloadSchema, options.Payload, &p) != nil {
return nil, engines.ErrContractViolation
}
b := &sandboxBuilder{
engine: e,
payload: p,
context: options.TaskContext,
env: make(map[string]string),
log: e.log.
WithField("taskId", options.TaskContext.TaskID).
WithField("runId", options.TaskContext.RunID),
}
go b.fetchContext()
return b, nil
}
func (e *engine) NewSandboxBuilder(options engines.SandboxOptions) (engines.SandboxBuilder, error) {
var p payloadType
if schematypes.MustMap(payloadSchema, options.Payload, &p) != nil {
return nil, engines.ErrContractViolation
}
// Get an idle network
net, err := e.networkPool.Network()
if err == network.ErrAllNetworksInUse {
return nil, engines.ErrMaxConcurrencyExceeded
}
if err != nil {
return nil, err
}
// Create sandboxBuilder, it'll handle image downloading
return newSandboxBuilder(&p, net, options.TaskContext, e), nil
}
func (provider) NewPlugin(options plugins.PluginOptions) (plugins.Plugin, error) {
var c config
if schematypes.MustMap(configSchema, options.Config, &c) != nil {
return nil, engines.ErrContractViolation
}
if c.ArtifactPrefix == "" {
c.ArtifactPrefix = defaultArtifactPrefix
}
if c.ShellToolURL == "" {
c.ShellToolURL = defaultShellToolURL
}
if c.DisplayToolURL == "" {
c.DisplayToolURL = defaultDisplayToolURL
}
return &plugin{
config: c,
log: options.Log,
}, nil
}
func (engineProvider) NewEngine(options engines.EngineOptions) (engines.Engine, error) {
var config configType
if schematypes.MustMap(configSchema, options.Config, &config) != nil {
return nil, engines.ErrContractViolation
}
// Construct payload schema as schematypes.Object using schema.properties
properties := schematypes.Properties{}
for k, s := range config.Schema.Properties {
schema, err := schematypes.NewSchema(s)
if err != nil {
return nil, fmt.Errorf("Error loading schema: %s", err)
}
properties[k] = schema
}
return &engine{
Log: options.Log,
config: config,
schema: schematypes.Object{
Properties: properties,
},
environment: options.Environment,
}, nil
}
// New will create a worker given configuration matching the schema from
// ConfigSchema(). The log parameter is optional and if nil is given a default
// logrus logger will be used.
func New(config interface{}, log *logrus.Logger) (*Worker, error) {
// Validate and map configuration to c
var c configType
if err := schematypes.MustMap(ConfigSchema(), config, &c); err != nil {
return nil, fmt.Errorf("Invalid configuration: %s", err)
}
// Create temporary folder
err := os.RemoveAll(c.TemporaryFolder)
if err != nil {
return nil, fmt.Errorf("Failed to remove temporaryFolder: %s, error: %s",
c.TemporaryFolder, err)
}
tempStorage, err := runtime.NewTemporaryStorage(c.TemporaryFolder)
if err != nil {
return nil, fmt.Errorf("Failed to create temporary folder, error: %s", err)
}
// Create logger
if log == nil {
log = logrus.New()
}
log.Level, _ = logrus.ParseLevel(c.LogLevel)
// Setup WebHookServer
localServer, err := webhookserver.NewLocalServer(
net.ParseIP(c.ServerIP), c.ServerPort,
c.NetworkInterface, c.ExposedPort,
c.DNSDomain,
c.DNSSecret,
c.TLSCertificate,
c.TLSKey,
time.Duration(c.MaxLifeCycle)*time.Second,
)
if err != nil {
return nil, err
}
// Create environment
gc := gc.New(c.TemporaryFolder, c.MinimumDiskSpace, c.MinimumMemory)
env := &runtime.Environment{
GarbageCollector: gc,
Log: log,
TemporaryStorage: tempStorage,
WebHookServer: localServer,
}
// Ensure that engine confiuguration was provided for the engine selected
if _, ok := c.Engines[c.Engine]; !ok {
return nil, fmt.Errorf("Invalid configuration: The key 'engines.%s' must "+
"be specified when engine '%s' is selected", c.Engine, c.Engine)
}
// Find engine provider (schema should ensure it exists)
provider := engines.Engines()[c.Engine]
engine, err := provider.NewEngine(engines.EngineOptions{
Environment: env,
Log: env.Log.WithField("engine", c.Engine),
Config: c.Engines[c.Engine],
})
if err != nil {
return nil, fmt.Errorf("Engine initialization failed, error: %s", err)
}
// Initialize plugin manager
pm, err := plugins.NewPluginManager(plugins.PluginOptions{
Environment: env,
Engine: engine,
Log: env.Log.WithField("plugin", "plugin-manager"),
Config: c.Plugins,
})
if err != nil {
return nil, fmt.Errorf("Plugin initialization failed, error: %s", err)
}
tm, err := newTaskManager(
&c, engine, pm, env,
env.Log.WithField("component", "task-manager"), gc,
)
if err != nil {
return nil, err
}
return &Worker{
log: env.Log.WithField("component", "worker"),
tm: tm,
sm: runtime.NewShutdownManager("local"),
env: env,
server: localServer,
done: make(chan struct{}),
}, nil
}
// Load configuration from YAML config object.
func Load(data []byte) (map[string]interface{}, error) {
// Parse config file
var config interface{}
err := yaml.Unmarshal(data, &config)
if err != nil {
return nil, fmt.Errorf("Failed to parse YAML config, error: %s", err)
}
// This fixes obscurities in yaml.Unmarshal where it generates
// map[interface{}]interface{} instead of map[string]interface{}
// credits: https://github.com/go-yaml/yaml/issues/139#issuecomment-220072190
config = convertSimpleJSONTypes(config)
// Extract transforms and config
c, ok := config.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("Expected top-level config value to be an object")
}
result, ok := c["config"].(map[string]interface{})
if !ok {
return nil, fmt.Errorf("Expected 'config' property to be an object")
}
// Ensure that we have a simple JSON compatible structure
if err := jsonCompatTypes(result); err != nil {
panic(fmt.Sprintf("YAML loaded wrong types, error: %s", err))
}
// Apply transforms
if ct, ok := c["transforms"]; ok {
var transforms []string
err := schematypes.MustMap(Schema().Properties["transforms"], ct, &transforms)
if err != nil {
return nil, fmt.Errorf("'transforms' schema violated, error: %s", err)
}
providers := Providers()
for _, t := range transforms {
provider, ok := providers[t]
if !ok {
return nil, fmt.Errorf("Unknown config transformation: %s", t)
}
if err := provider.Transform(result); err != nil {
return nil, fmt.Errorf("Config transformation: %s failed error: %s",
t, err)
}
// Ensure that transform only injects simple JSON compatible types
if err := jsonCompatTypes(result); err != nil {
panic(fmt.Sprintf("%s injected wrong types, error: %s", t, err))
}
}
}
// Filter out keys that aren't in the config schema...
// This way extra keys can be used to provide options for the
// transformations, like "secrets" which will use the secretsBaseUrl if
// present in the configuration.
worker.ConfigSchema().Filter(result)
// Validate against worker schema
if err := worker.ConfigSchema().Validate(result); err != nil {
return nil, err
}
return result, nil
}
