func (j *jobRunner) runJobInfo(persistJobInfo *persist.JobInfo) error {
switch {
case persistJobInfo.GetTransform() != nil:
return j.reallyRunJobInfo(
uuid.NewWithoutDashes(),
persistJobInfo.JobId,
persistJobInfo.GetTransform(),
persistJobInfo.Input,
persistJobInfo.OutputParent,
1,
)
case persistJobInfo.GetPipelineName() != "":
persistPipelineInfo, err := j.persistAPIClient.GetPipelineInfo(
context.Background(),
&pps.Pipeline{Name: persistJobInfo.GetPipelineName()},
)
if err != nil {
return err
}
if persistPipelineInfo.GetTransform() == nil {
return fmt.Errorf("pachyderm.pps.server: transform not set on pipeline info %v", persistPipelineInfo)
}
return j.reallyRunJobInfo(
persistPipelineInfo.PipelineName,
persistJobInfo.JobId,
persistPipelineInfo.GetTransform(),
persistJobInfo.Input,
persistJobInfo.OutputParent,
1,
)
default:
return fmt.Errorf("pachyderm.pps.server: neither transform or pipeline name set on job info %v", persistJobInfo)
}
}
func (a *apiServer) persistJobInfoToJobInfo(ctx context.Context, persistJobInfo *persist.JobInfo) (*pps.JobInfo, error) {
job := &pps.Job{Id: persistJobInfo.JobId}
jobInfo := &pps.JobInfo{
Job: job,
Shards: persistJobInfo.Shards,
InputCommit: persistJobInfo.InputCommit,
}
if persistJobInfo.GetTransform() != nil {
jobInfo.Spec = &pps.JobInfo_Transform{
Transform: persistJobInfo.GetTransform(),
}
}
if persistJobInfo.GetPipelineName() != "" {
jobInfo.Spec = &pps.JobInfo_Pipeline{
Pipeline: &pps.Pipeline{
Name: persistJobInfo.GetPipelineName(),
},
}
}
persistJobOutput, err := a.persistAPIClient.GetJobOutput(ctx, job)
if err != nil {
return nil, err
}
if persistJobOutput != nil {
jobInfo.OutputCommit = persistJobOutput.OutputCommit
}
return jobInfo, nil
}
// job_id cannot be set
// timestamp cannot be set
func (a *rethinkAPIServer) CreateJobInfo(ctx context.Context, request *persist.JobInfo) (response *persist.JobInfo, err error) {
defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
if request.JobId != "" {
return nil, ErrIDSet
}
if request.CreatedAt != nil {
return nil, ErrTimestampSet
}
request.JobId = uuid.New()
request.CreatedAt = a.now()
if err := a.insertMessage(jobInfosTable, request); err != nil {
return nil, err
}
return request, nil
}
// JobId cannot be set
// Timestamp cannot be set
func (a *rethinkAPIServer) CreateJobInfo(ctx context.Context, request *persist.JobInfo) (response *persist.JobInfo, err error) {
defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
if request.JobId != "" {
return nil, fmt.Errorf("request.JobId should be unset")
}
if request.CreatedAt != nil {
return nil, fmt.Errorf("request.CreatedAt should be unset")
}
if request.CommitIndex != "" {
return nil, fmt.Errorf("request.CommitIndex should be unset")
}
request.JobId = uuid.NewWithoutDashes()
request.CreatedAt = prototime.TimeToTimestamp(time.Now())
request.CommitIndex = genCommitIndex(request.InputCommit)
if err := a.insertMessage(jobInfosTable, request); err != nil {
return nil, err
}
return request, nil
}
func job(jobInfo *persist.JobInfo) *extensions.Job {
app := jobInfo.JobId
shards := int(jobInfo.Shards)
return &extensions.Job{
TypeMeta: unversioned.TypeMeta{
Kind: "Job",
APIVersion: "v1",
},
ObjectMeta: api.ObjectMeta{
Name: jobInfo.JobId,
Labels: labels(app),
},
Spec: extensions.JobSpec{
Selector: &extensions.PodSelector{
MatchLabels: labels(app),
},
Parallelism: &shards,
Completions: &shards,
Template: api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Name: jobInfo.JobId,
Labels: labels(app),
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "user",
Image: "pachyderm/job-shim",
Command: append([]string{"/job-shim", jobInfo.JobId}, jobInfo.GetTransform().Cmd...),
SecurityContext: &api.SecurityContext{
Privileged: &trueVal, // god is this dumb
},
},
},
RestartPolicy: "Never",
},
},
},
}
}
func job(jobInfo *persist.JobInfo) *extensions.Job {
app := jobInfo.JobId
return &extensions.Job{
TypeMeta: unversioned.TypeMeta{
Kind: "Job",
APIVersion: "v1",
},
ObjectMeta: api.ObjectMeta{
Name: jobInfo.JobId,
Labels: map[string]string{
"app": app,
},
},
Spec: extensions.JobSpec{
Selector: &extensions.PodSelector{
MatchLabels: map[string]string{
"app": app,
},
},
Template: api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Name: jobInfo.JobId,
Labels: map[string]string{
"app": app,
},
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "user",
Image: jobInfo.GetTransform().Image,
Command: jobInfo.GetTransform().Cmd,
},
},
},
},
},
}
}
// TODO: bulk get
func (a *apiServer) persistJobInfoToJobInfo(ctx context.Context, persistJobInfo *persist.JobInfo) (*pps.JobInfo, error) {
job := &pps.Job{Id: persistJobInfo.JobId}
persistJobStatuses, err := a.persistAPIClient.GetJobStatuses(ctx, job)
if err != nil {
return nil, err
}
persistJobOutput, err := a.persistAPIClient.GetJobOutput(ctx, job)
if err != nil {
return nil, err
}
jobInfo := &pps.JobInfo{
Job: job,
Input: persistJobInfo.Input,
}
if persistJobInfo.GetTransform() != nil {
jobInfo.Spec = &pps.JobInfo_Transform{
Transform: persistJobInfo.GetTransform(),
}
}
if persistJobInfo.GetPipelineName() != "" {
jobInfo.Spec = &pps.JobInfo_Pipeline{
Pipeline: &pps.Pipeline{
Name: persistJobInfo.GetPipelineName(),
},
}
}
jobInfo.JobStatus = make([]*pps.JobStatus, len(persistJobStatuses.JobStatus))
for i, persistJobStatus := range persistJobStatuses.JobStatus {
jobInfo.JobStatus[i] = &pps.JobStatus{
Type: persistJobStatus.Type,
Timestamp: persistJobStatus.Timestamp,
Message: persistJobStatus.Message,
}
}
if persistJobOutput != nil {
jobInfo.Output = persistJobOutput.Output
}
return jobInfo, nil
}