コード例 #1
0
func (sb *sandboxBuilder) AttachProxy(hostname string, handler http.Handler) error {
	// Validate hostname against allowed patterns
	if !proxyNamePattern.MatchString(hostname) {
		return engines.NewMalformedPayloadError("Proxy hostname: '", hostname, "'",
			" is not allowed for QEMU engine. The hostname must match: ",
			proxyNamePattern.String())
	}
	// Ensure that we're not using the magic "engine" hostname
	if hostname == "engine" {
		return engines.NewMalformedPayloadError("Proxy hostname: 'engine' is " +
			"reserved for internal use (meta-data service)")
	}

	// Acquire the lock
	sb.m.Lock()
	defer sb.m.Unlock()

	// Check that the hostname isn't already in use
	if _, ok := sb.proxies[hostname]; ok {
		return engines.ErrNamingConflict
	}

	// Otherwise set the handler
	sb.proxies[hostname] = handler
	return nil
}
コード例 #2
0
// LoadMachine will load machine definition from file
func LoadMachine(machineFile string) (*Machine, error) {
	// Load the machine configuration
	machineData, err := ioext.BoundedReadFile(machineFile, 1024*1024)
	if err == ioext.ErrFileTooBig {
		return nil, engines.NewMalformedPayloadError(
			"The file 'machine.json' larger than 1MiB. JSON files must be small.")
	}
	if err != nil {
		return nil, err
	}

	// Parse json
	m := &Machine{}
	err = json.Unmarshal(machineData, m)
	if err != nil {
		return nil, engines.NewMalformedPayloadError(
			"Invalid JSON in 'machine.json', error: ", err)
	}

	// Validate the definition
	if err := m.Validate(); err != nil {
		return nil, err
	}

	return m, nil
}
コード例 #3
0
// DownloadImage returns a Downloader that will download the image from the
// given url. This will attempt multiple retries if necessary.
//
// If there is a non-200 response this will return a MalformedPayloadError.
func DownloadImage(url string) Downloader {
	// TODO: Add some logging, I really want to abstract away Logger
	// TODO: This method could probably exist somewhere else, in say runtime
	//       downloading from a URL to a file or stream with retries, etc. is a
	//       common thing. Using range headers for retries and getting integrity
	//       checks right is hard.
	return func(target string) error {
		// Create output file
		out, err := os.Create(target)
		if err != nil {
			return err
		}
		defer out.Close()

		attempt := 1
		for {
			// Move to start of file and truncate the file
			_, err = out.Seek(0, 0)
			if err != nil {
				panic("Unable to seek to file start")
			}
			err = out.Truncate(0)
			if err != nil {
				panic("Unable to truncate file")
			}

			// Create a GET request
			res, err := http.Get(url)
			if err != nil {
				goto retry
			}
			if 500 <= res.StatusCode && res.StatusCode < 600 {
				err = fmt.Errorf("Image download failed with status code: %d", res.StatusCode)
				goto retry
			}
			if res.StatusCode != 200 {
				return engines.NewMalformedPayloadError(
					"Image download failed with status code: ", res.StatusCode,
				)
			}

			// Copy response to file
			// TODO: Make integrity check with x-amx-meta-content-sha256 (if present)
			// TODO: Use range headers for retry, if supported and checksum for
			//       integrity check is present (otherwise request from start)
			_, err = io.Copy(out, res.Body)
			res.Body.Close()
			if err == nil {
				return nil
			}
		retry:
			if attempt >= maxRetries {
				return err
			}
			attempt++
			got.DefaultBackOff.Delay(attempt)
		}
	}
}
コード例 #4
0
func (tp *taskPlugin) Stopped(result engines.ResultSet) (bool, error) {
	nonFatalErrs := []engines.MalformedPayloadError{}

	for _, artifact := range tp.artifacts {
		// If expires is set to this time it's the default value
		if artifact.Expires.IsZero() {
			artifact.Expires = time.Time(tp.context.TaskInfo.Expires)
		}
		switch artifact.Type {
		case "directory":
			err := result.ExtractFolder(artifact.Path, tp.createUploadHandler(artifact.Name, artifact.Expires))
			if err != nil {
				if tp.errorHandled(artifact.Name, artifact.Expires, err) {
					nonFatalErrs = append(nonFatalErrs, engines.NewMalformedPayloadError(err.Error()))
					continue
				}
				return false, err
			}
		case "file":
			fileReader, err := result.ExtractFile(artifact.Path)
			if err != nil {
				if tp.errorHandled(artifact.Name, artifact.Expires, err) {
					nonFatalErrs = append(nonFatalErrs, engines.NewMalformedPayloadError(err.Error()))
					continue
				}
				return false, err
			}
			err = tp.attemptUpload(fileReader, artifact.Path, artifact.Name, artifact.Expires)
			if err != nil {
				return false, err
			}
		}
	}

	if len(nonFatalErrs) > 0 {
		return false, engines.MergeMalformedPayload(nonFatalErrs...)
	}
	return true, nil
}
コード例 #5
0
func (b *sandboxBuilder) SetEnvironmentVariable(name string, value string) error {
	if !envVarPattern.MatchString(name) {
		return engines.NewMalformedPayloadError(
			"Environment variables name: '", name, "' doesn't match: ",
			envVarPattern.String(),
		)
	}
	if _, ok := b.env[name]; ok {
		return engines.ErrNamingConflict
	}
	b.env[name] = value
	return nil
}
コード例 #6
0
ファイル: env.go プロジェクト: taskcluster/taskcluster-worker
func (p taskPlugin) BuildSandbox(sandboxBuilder engines.SandboxBuilder) error {
	for k, v := range p.variables {
		err := sandboxBuilder.SetEnvironmentVariable(k, v)

		// We can only return MalFormedPayloadError
		switch err {
		case engines.ErrNamingConflict:
			return engines.NewMalformedPayloadError("Environment variable ", k, " has already been set.")
		case engines.ErrFeatureNotSupported:
			return engines.NewMalformedPayloadError(
				"Cannot set environment variable ",
				k,
				". Engine does not support this operation")
		case nil:
			// break
		default:
			return err
		}
	}

	return nil
}
コード例 #7
0
func (s *sandbox) SetEnvironmentVariable(name string, value string) error {
	s.Lock()
	defer s.Unlock()
	if strings.Contains(name, " ") {
		return engines.NewMalformedPayloadError(
			"MockEngine environment variable names cannot contain space.",
			"Was given environment variable name: '", name, "' which isn't allowed!",
		)
	}
	if _, ok := s.env[name]; ok {
		return engines.ErrNamingConflict
	}
	s.env[name] = value
	return nil
}
コード例 #8
0
func (s *sandbox) AttachProxy(name string, handler http.Handler) error {
	// Lock before we access proxies as this method may be called concurrently
	s.Lock()
	defer s.Unlock()
	if strings.ContainsAny(name, " ") {
		return engines.NewMalformedPayloadError(
			"MockEngine proxy names cannot contain space.",
			"Was given proxy name: '", name, "' which isn't allowed!",
		)
	}
	if s.proxies[name] != nil {
		return engines.ErrNamingConflict
	}
	s.proxies[name] = handler
	return nil
}
コード例 #9
0
// SetDefaults will validate limitations and set defaults from options
func (m *Machine) SetDefaults(options MachineOptions) error {
	// Set defaults
	if m.Memory == 0 {
		m.Memory = options.MaxMemory
	}

	// Validate limitations
	if m.Memory > options.MaxMemory {
		return engines.NewMalformedPayloadError(
			"Image memory ", m.Memory, " MiB is larger than allowed machine memory ",
			options.MaxMemory, " MiB",
		)
	}

	return nil
}
コード例 #10
0
func (s *sandbox) WaitForResult() (engines.ResultSet, error) {
	// No need to lock access to payload, as it can't be mutated at this point
	select {
	case <-s.done:
		s.result = false
		return s, errors.New("Task execution has been aborted")
	case <-time.After(time.Duration(s.payload.Delay) * time.Millisecond):
		// No need to lock access mounts and proxies either
		f := functions[s.payload.Function]
		if f == nil {
			return nil, engines.NewMalformedPayloadError("Unknown function")
		}
		result := f(s, s.payload.Argument)
		s.Lock()
		defer s.Unlock()
		s.result = result
		return s, nil
	}
}
コード例 #11
0
func (e engine) NewSandboxBuilder(options engines.SandboxOptions) (engines.SandboxBuilder, error) {
	var taskPayload payloadType
	err := payloadSchema.Map(options.Payload, &taskPayload)
	if err == schematypes.ErrTypeMismatch {
		panic("Type mismatch")
	} else if err != nil {
		return nil, engines.NewMalformedPayloadError("Invalid payload: ", err)
	}

	var m sync.Mutex
	return sandboxbuilder{
		SandboxBuilderBase: engines.SandboxBuilderBase{},
		env:                map[string]string{},
		taskPayload:        &taskPayload,
		context:            options.TaskContext,
		envMutex:           &m,
		engine:             &e,
	}, nil
}
コード例 #12
0
func (r resultset) ExtractFile(path string) (ioext.ReadSeekCloser, error) {
	cwd := getWorkingDir(r.taskUser, r.context)
	if !r.validPath(cwd, path) {
		return nil, engines.NewMalformedPayloadError(path + " is invalid")
	}

	if !filepath.IsAbs(path) {
		path = filepath.Join(cwd, path)
	}

	file, err := os.Open(path)

	if err != nil {
		r.context.LogError(err)
		return nil, engines.ErrResourceNotFound
	}

	return file, nil
}
コード例 #13
0
func (sb *sandboxBuilder) SetEnvironmentVariable(name, value string) error {
	// Simple sanity check of environment variable names
	if !envVarPattern.MatchString(name) {
		return engines.NewMalformedPayloadError("Environment variable name: '",
			name, "' is not allowed for QEMU engine. Environment variable names",
			" must be on the form: ", envVarPattern.String())
	}

	// Acquire the lock
	sb.m.Lock()
	defer sb.m.Unlock()

	// Check if the name is already used
	if _, ok := sb.env[name]; ok {
		return engines.ErrNamingConflict
	}

	// Set the env var
	sb.env[name] = value
	return nil
}
コード例 #14
0
func (r *resultSet) ExtractFile(path string) (ioext.ReadSeekCloser, error) {
	// Evaluate symlinks
	p, err := filepath.EvalSymlinks(filepath.Join(r.homeFolder.Path(), path))
	if err != nil {
		if _, ok := err.(*os.PathError); ok {
			return nil, engines.ErrResourceNotFound
		}
		return nil, engines.NewMalformedPayloadError(
			"Unable to evaluate path: ", path,
		)
	}

	// Cleanup the path
	p = filepath.Clean(p)

	// Check that p is inside homeFolder
	if !strings.HasPrefix(p, r.homeFolder.Path()+string(filepath.Separator)) {
		return nil, engines.ErrResourceNotFound
	}

	// Stat the file to make sure it's a file
	info, err := os.Lstat(p)
	if err != nil {
		return nil, engines.ErrResourceNotFound
	}
	// Don't allow anything that isn't a plain file
	if !ioext.IsPlainFileInfo(info) {
		return nil, engines.ErrResourceNotFound
	}

	// Open file
	f, err := os.Open(p)
	if err != nil {
		return nil, engines.ErrResourceNotFound
	}

	return f, nil
}
コード例 #15
0
func (r resultset) ExtractFolder(path string, handler engines.FileHandler) error {
	cwd := getWorkingDir(r.taskUser, r.context)
	if !r.validPath(cwd, path) {
		return engines.NewMalformedPayloadError(path + " is invalid")
	}

	if !filepath.IsAbs(path) {
		path = filepath.Join(cwd, path)
	}

	return filepath.Walk(path, func(p string, info os.FileInfo, e error) error {
		if e != nil {
			r.context.LogError(e)
			return engines.ErrResourceNotFound
		}

		if !info.IsDir() {
			file, err := os.Open(p)
			if err != nil {
				r.context.LogError(err)
				return engines.ErrResourceNotFound
			}

			p, err = filepath.Rel(path, p)
			if err != nil {
				r.context.LogError(err)
				return engines.ErrResourceNotFound
			}

			err = handler(p, file)
			if err != nil {
				return err
			}
		}

		return nil
	})
}
コード例 #16
0
// Validate returns a MalformedPayloadError if the Machine definition isn't
// valid and legal.
func (m *Machine) Validate() error {
	hasError := false
	errs := "Invalid machine definition in 'machine.json'"
	msg := func(a ...interface{}) {
		errs += "\n" + fmt.Sprint(a...)
		hasError = true
	}

	// Render to JSON so we can validate with gojsonschema
	// (this isn't efficient, but we'll rarely do this so who cares)
	data, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		panic(fmt.Sprintln(
			"json.Marshal should never fail for vm.Machine, error: ", err,
		))
	}

	// Validate against JSON schema
	result, err := machineSchema.Validate(
		gojsonschema.NewStringLoader(string(data)),
	)
	if err != nil {
		panic(fmt.Sprintln(
			"machineSchema.Validate should always be able to validate, error: ", err,
		))
	}
	if !result.Valid() {
		for _, err := range result.Errors() {
			msg(err.(*gojsonschema.ResultErrorFields).String())
		}
	}

	// Return any errors collected
	if hasError {
		return engines.NewMalformedPayloadError(errs)
	}
	return nil
}
コード例 #17
0
func (s *sandbox) AttachVolume(mountpoint string, v engines.Volume, readOnly bool) error {
	// We can type cast Volume to our internal type as we know the volume was
	// created by NewCacheFolder() or NewMemoryDisk(), this is a contract.
	vol, valid := v.(*volume)
	if !valid {
		// TODO: Write to some sort of log if the type assertion fails
		return engines.ErrContractViolation
	}
	// Lock before we access mounts as this method may be called concurrently
	s.Lock()
	defer s.Unlock()
	if strings.ContainsAny(mountpoint, " ") {
		return engines.NewMalformedPayloadError("MockEngine mountpoints cannot contain space")
	}
	if s.mounts[mountpoint] != nil {
		return engines.ErrNamingConflict
	}
	s.mounts[mountpoint] = &mount{
		volume:   vol,
		readOnly: readOnly,
	}
	return nil
}
コード例 #18
0
// prepareStage is where task plugins are prepared and a sandboxbuilder is created.
func (t *TaskRun) prepareStage() error {
	t.log.Debug("Preparing task run")

	// Parse payload
	payload := map[string]interface{}{}
	err := json.Unmarshal([]byte(t.definition.Payload), &payload)
	if err != nil {
		err = engines.NewMalformedPayloadError(fmt.Sprintf("Invalid task payload. %s", err))
		t.context.LogError(err.Error())
		return err
	}

	// Construct payload schema
	payloadSchema, err := schematypes.Merge(
		t.engine.PayloadSchema(),
		t.plugin.PayloadSchema(),
	)
	if err != nil {
		panic(fmt.Sprintf(
			"Conflicting plugin and engine payload properties, error: %s", err,
		))
	}

	// Validate payload against schema
	err = payloadSchema.Validate(payload)
	if err != nil {
		err = engines.NewMalformedPayloadError("Schema violation: ", err)
		t.context.LogError(err.Error())
		return err
	}

	// Create TaskPlugin
	t.taskPlugin, err = t.plugin.NewTaskPlugin(plugins.TaskPluginOptions{
		TaskInfo: &runtime.TaskInfo{
			TaskID: t.TaskID,
			RunID:  t.RunID,
		},
		Payload: t.plugin.PayloadSchema().Filter(payload),
		Log:     t.log.WithField("plugin", "pluginManager"),
	})
	if err != nil {
		// TODO: We need to review all this... t.context.LogError is for task errors
		// hence MalformedPayloadError only, not for internal-errors!!!
		t.context.LogError(err.Error())
		return err
	}

	// Prepare TaskPlugin
	err = t.taskPlugin.Prepare(t.context)
	if err != nil {
		t.context.LogError(fmt.Sprintf("Could not prepare task plugins. %s", err))
		return err
	}

	sandboxBuilder, err := t.engine.NewSandboxBuilder(engines.SandboxOptions{
		TaskContext: t.context,
		Payload:     t.engine.PayloadSchema().Filter(payload),
	})
	t.m.Lock()
	t.sandboxBuilder = sandboxBuilder
	t.m.Unlock()
	if err != nil {
		t.context.LogError(fmt.Sprintf("Could not create task execution environment. %s", err))
		return err
	}

	return nil
}
コード例 #19
0
func (r *resultSet) ExtractFolder(path string, handler engines.FileHandler) error {
	// Evaluate symlinks
	p, err := filepath.EvalSymlinks(filepath.Join(r.homeFolder.Path(), path))
	if err != nil {
		if _, ok := err.(*os.PathError); ok {
			return engines.ErrResourceNotFound
		}
		return engines.NewMalformedPayloadError(
			"Unable to evaluate path: ", path,
		)
	}

	// Cleanup the path
	p = filepath.Clean(p)

	// Check that p is inside homeFolder
	if !strings.HasPrefix(p, r.homeFolder.Path()+string(filepath.Separator)) {
		return engines.ErrResourceNotFound
	}

	first := true
	return filepath.Walk(p, func(abspath string, info os.FileInfo, err error) error {
		// If there is a path error, on the first call then the folder is missing
		if _, ok := err.(*os.PathError); ok && first {
			return engines.ErrResourceNotFound
		}
		first = false

		// Ignore folder we can't walk (probably a permission issues)
		if err != nil {
			return nil
		}

		// Skip anything that isn't a plain file
		if !ioext.IsPlainFileInfo(info) {
			return nil
		}

		// If we can't construct relative file path this internal error, we'll skip
		relpath, err := filepath.Rel(p, abspath)
		if err != nil {
			// TODO: Send error to sentry
			r.log.Errorf(
				"ExtractFolder from %s, filepath.Rel('%s', '%s') returns error: %s",
				path, p, abspath, err,
			)
			return nil
		}

		f, err := os.Open(abspath)
		if err != nil {
			// file must have been deleted as we tried to open it
			// that makes no sense, but who knows...
			return nil
		}

		// If handler returns an error we return ErrHandlerInterrupt
		if handler(filepath.ToSlash(relpath), f) != nil {
			return engines.ErrHandlerInterrupt
		}
		return nil
	})
}
コード例 #20
0
// extractImage will extract the "disk.img", "layer.qcow2" and "machine.json"
// files from a tar archive using GNU tar ensuring that sparse entries will be
// extracted as sparse files.
//
// This also validates that files aren't symlinks and are in correct format,
// with legal backing_file parameters.
//
// Returns a MalformedPayloadError if we believe extraction failed due to a
// badly formatted image.
func extractImage(imageFile, imageFolder string) (*vm.Machine, error) {
	// Restrict file to some maximum size
	if !ioext.IsPlainFile(imageFile) {
		return nil, fmt.Errorf("extractImage: imageFile is not a file")
	}
	if !ioext.IsFileLessThan(imageFile, maxImageSize) {
		return nil, engines.NewMalformedPayloadError("Image file is larger than ", maxImageSize, " bytes")
	}

	// Using zstd | tar so we get sparse files (sh to get OS pipes)
	tar := exec.Command("sh", "-fec", "zstd -dqc '"+imageFile+"' | "+
		"tar -xoC '"+imageFolder+"' --no-same-permissions -- "+
		"disk.img layer.qcow2 machine.json",
	)
	_, err := tar.Output()
	if err != nil {
		if ee, ok := err.(*exec.ExitError); ok {
			return nil, engines.NewMalformedPayloadError(
				"Failed to extract image archieve, error: ", string(ee.Stderr),
			)
		}
		// If this wasn't GNU tar exiting non-zero then it must be some internal
		// error. Perhaps tar is missing from the PATH.
		return nil, fmt.Errorf("Failed to extract image archieve, error: %s", err)
	}

	// Check files exist, are plain files and not larger than maxImageSize
	for _, name := range []string{"disk.img", "layer.qcow2", "machine.json"} {
		f := filepath.Join(imageFolder, name)
		if !ioext.IsPlainFile(f) {
			return nil, engines.NewMalformedPayloadError("Image file is missing '", name, "'")
		}
		if !ioext.IsFileLessThan(f, maxImageSize) {
			return nil, engines.NewMalformedPayloadError("Image file contains '",
				name, "' larger than ", maxImageSize, " bytes")
		}
	}

	// Load the machine configuration
	machineFile := filepath.Join(imageFolder, "machine.json")
	machine, err := vm.LoadMachine(machineFile)
	if err != nil {
		return nil, err
	}

	// Inspect the raw disk file
	diskFile := filepath.Join(imageFolder, "disk.img")
	diskInfo := inspectImageFile(diskFile, imageRawFormat)
	if diskInfo == nil || diskInfo.Format != formatRaw {
		return nil, engines.NewMalformedPayloadError("Image file contains ",
			"'disk.img' which is not a RAW image file")
	}
	if diskInfo.VirtualSize > maxImageSize {
		return nil, engines.NewMalformedPayloadError("Image file contains ",
			"'disk.img' has virtual size larger than ", maxImageSize, " bytes")
	}
	if diskInfo.DirtyFlag {
		return nil, engines.NewMalformedPayloadError("Image file contains ",
			"'disk.img' which has the dirty-flag set")
	}
	if diskInfo.BackingFile != "" {
		return nil, engines.NewMalformedPayloadError("Image file contains ",
			"'disk.img' which has a backing file, this is not permitted")
	}

	// Inspect the QCOW2 layer file
	layerFile := filepath.Join(imageFolder, "layer.qcow2")
	layerInfo := inspectImageFile(layerFile, imageQCOW2Format)
	if layerInfo == nil || layerInfo.Format != formatQCOW2 {
		return nil, engines.NewMalformedPayloadError("Image file contains ",
			"'layer.qcow2' which is not a QCOW2 file")
	}
	if layerInfo.VirtualSize > maxImageSize {
		return nil, engines.NewMalformedPayloadError("Image file contains ",
			"'layer.qcow2' has virtual size larger than ", maxImageSize, " bytes")
	}
	if layerInfo.DirtyFlag {
		return nil, engines.NewMalformedPayloadError("Image file contains ",
			"'layer.qcow2' which has the dirty-flag set")
	}
	if layerInfo.BackingFile != "disk.img" {
		return nil, engines.NewMalformedPayloadError("Image file contains ",
			"'layer.qcow2' which has a backing file that isn't: 'disk.img'")
	}
	if layerInfo.BackingFormat != formatRaw {
		return nil, engines.NewMalformedPayloadError("Image file contains ",
			"'layer.qcow2' which has a backing file format that isn't 'raw'")
	}

	return machine, nil
}