Example #1
0
// Split implements of FileSplitter
// {Input}: a directory
// return an array of files in the input
func (f *FileSplitter) Split(input string) ([]string, error) {
	var files []string

	dir, err := ioutil.ReadDir(fs.NormalizePath(input))
	if err != nil {
		return nil, err
	}

	for _, fi := range dir {
		files = append(files, path.Join(input, fi.Name()))
	}

	return files, nil
}
Example #2
0
func (core *Core) addTask(job *registry.Job, scale int, inputs []string) {
	for _, task := range job.Tasks {
		randID, err := utils.Encode(6)
		if err != nil {
			log.Errorf("Error when generate id to task %s of job %v", task.ID, job.ID)
			continue
		}

		if task.Scale <= 0 {
			task.Scale = 1
		}

		if scale == 0 {
			scale = task.Scale
		}

		job.TotalTaskLen = scale
		for index := 1; index <= scale; index++ {
			// To avoid use same pointer of ports
			// Instantiate a new array
			var ports []*registry.Port
			for _, port := range task.Ports {
				ports = append(ports, &registry.Port{
					ContainerPort: port.ContainerPort,
					HostPort:      port.HostPort,
				})
			}

			taskInstance := &registry.Task{
				JobID:       job.ID,
				ID:          "task-" + job.ID + "-" + randID + "-" + strconv.Itoa(index),
				Name:        job.Name,
				DockerImage: job.Image,
				Cpus:        task.Cpus,
				Mem:         task.Mem,
				Disk:        task.Disk,
				Ports:       ports,
				Command:     task.Command,
				Volumes:     task.Volumes,
				Resources:   task.Resources,
				Attributes:  task.Attributes,
				CreateTime:  time.Now().UnixNano(),
				Type:        registry.TaskTypeTest,
				State:       "TASK_WAITING",
			}

			// mount input path
			if len(inputs) > 0 {
				taskInstance.Volumes = append(taskInstance.Volumes, &registry.Volume{
					HostPath:      fs.NormalizePath(inputs[index-1]),
					ContainerPath: "/input",
				})
			}

			// mount work directory
			taskInstance.Volumes = append(taskInstance.Volumes, &registry.Volume{
				HostPath:      fs.NormalizePath(job.WorkDirectory),
				ContainerPath: "/workspace",
			})

			// mount output
			taskInstance.Volumes = append(taskInstance.Volumes, &registry.Volume{
				HostPath:      fs.NormalizePath(job.OutputPath),
				ContainerPath: "/output",
			})

			// if task was build from dockerfile
			// add attribute to task
			if job.ContextDir != "" {
				taskInstance.Attributes = append(task.Attributes, &mesosproto.Attribute{
					Name: proto.String("Image"),
					Text: &mesosproto.Value_Text{
						Value: proto.String(job.Image),
					},
				})
			}

			// TODO bugfix: task point to one pointer
			var taskArguments []string
			for _, arg := range task.Arguments {
				taskArguments = append(taskArguments, arg)
			}
			taskInstance.Arguments = taskArguments

			err = core.AddTask(taskInstance.ID, taskInstance)
			job.PushTask(taskInstance)
			if err != nil {
				task.State = "TASK_FAILED"
				log.Errorf("Error when running task %v: %v", task.ID, err)
				job.PopLastTask()
				continue
			}
		}
	}
}