func (s *simpleTest) setup() {
if s.inputRepoName == "" {
s.inputRepoName = uuid.NewWithoutDashes()
}
if s.outputRepoName == "" {
s.outputRepoName = uuid.NewWithoutDashes()
}
if s.pipelineName == "" {
s.pipelineName = uuid.NewWithoutDashes()
}
if s.jobTimeoutSec == 0 {
s.jobTimeoutSec = defaultJobTimeoutSec
}
}
func (a *apiServer) StartCommit(ctx context.Context, request *pfs.StartCommitRequest) (response *pfs.Commit, retErr error) {
defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
if request.Commit == nil {
if request.Parent == nil {
return nil, fmt.Errorf("one of Parent or Commit must be non nil")
}
request.Commit = &pfs.Commit{
Repo: request.Parent.Repo,
Id: uuid.NewWithoutDashes(),
}
}
for _, clientConn := range clientConns {
if _, err := pfs.NewInternalAPIClient(clientConn).StartCommit(ctx, request); err != nil {
return nil, err
}
}
return request.Commit, nil
}
func (a *apiServer) CreateJob(ctx context.Context, request *pps.CreateJobRequest) (response *pps.Job, err error) {
defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
persistJobInfo := &persist.JobInfo{
Input: request.Input,
OutputParent: request.OutputParent,
}
if request.GetTransform() != nil {
persistJobInfo.Spec = &persist.JobInfo_Transform{
Transform: request.GetTransform(),
}
} else if request.GetPipeline() != nil {
persistJobInfo.Spec = &persist.JobInfo_PipelineName{
PipelineName: request.GetPipeline().Name,
}
} else {
return nil, fmt.Errorf("pachyderm.pps.jobserver: both transform and pipeline are not set on %v", request)
}
persistJobInfo.JobId = uuid.NewWithoutDashes()
persistJobInfo.CreatedAt = prototime.TimeToTimestamp(time.Now())
if a.kubeClient == nil {
return nil, fmt.Errorf("pachyderm.pps.jobserver: no job backend")
}
if _, err := a.kubeClient.Jobs(api.NamespaceDefault).Create(job(persistJobInfo)); err != nil {
return nil, err
}
_, err = a.persistAPIClient.CreateJobInfo(ctx, persistJobInfo)
if err != nil {
return nil, err
}
return &pps.Job{
Id: persistJobInfo.JobId,
}, nil
}
func (j *jobRunner) runJobInfo(persistJobInfo *persist.JobInfo) error {
switch {
case persistJobInfo.GetTransform() != nil:
return j.reallyRunJobInfo(
uuid.NewWithoutDashes(),
persistJobInfo.JobId,
persistJobInfo.GetTransform(),
persistJobInfo.Input,
persistJobInfo.OutputParent,
1,
)
case persistJobInfo.GetPipelineName() != "":
persistPipelineInfo, err := j.persistAPIClient.GetPipelineInfo(
context.Background(),
&pps.Pipeline{Name: persistJobInfo.GetPipelineName()},
)
if err != nil {
return err
}
if persistPipelineInfo.GetTransform() == nil {
return fmt.Errorf("pachyderm.pps.server: transform not set on pipeline info %v", persistPipelineInfo)
}
return j.reallyRunJobInfo(
persistPipelineInfo.PipelineName,
persistJobInfo.JobId,
persistPipelineInfo.GetTransform(),
persistJobInfo.Input,
persistJobInfo.OutputParent,
1,
)
default:
return fmt.Errorf("pachyderm.pps.server: neither transform or pipeline name set on job info %v", persistJobInfo)
}
}
func (a *apiServer) createDisks(ctx context.Context, nodes uint64) ([]string, error) {
var names []string
for i := uint64(0); i < nodes; i++ {
name := "disk-" + uuid.NewWithoutDashes()
if err := a.provider.CreateDisk(name, defaultDiskSizeGb); err != nil {
return nil, err
}
names = append(names, name)
}
return names, nil
}
func NewTestRethinkAPIServer() (server.APIServer, error) {
address, err := getTestRethinkAddress()
if err != nil {
return nil, err
}
databaseName := uuid.NewWithoutDashes()
if err := server.InitDBs(address, databaseName); err != nil {
return nil, err
}
return server.NewRethinkAPIServer(address, databaseName)
}
// JobId cannot be set
// Timestamp cannot be set
func (a *rethinkAPIServer) CreateJobInfo(ctx context.Context, request *persist.JobInfo) (response *persist.JobInfo, err error) {
defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
if request.JobId != "" {
return nil, fmt.Errorf("request.JobId should be unset")
}
if request.CreatedAt != nil {
return nil, fmt.Errorf("request.CreatedAt should be unset")
}
if request.CommitIndex != "" {
return nil, fmt.Errorf("request.CommitIndex should be unset")
}
request.JobId = uuid.NewWithoutDashes()
request.CreatedAt = prototime.TimeToTimestamp(time.Now())
request.CommitIndex = genCommitIndex(request.InputCommit)
if err := a.insertMessage(jobInfosTable, request); err != nil {
return nil, err
}
return request, nil
}
func (a *apiServer) StartCommit(ctx context.Context, request *pfs.StartCommitRequest) (*pfs.Commit, error) {
a.versionLock.RLock()
defer a.versionLock.RUnlock()
ctx = versionToContext(a.version, ctx)
clientConns, err := a.router.GetAllClientConns(a.version)
if err != nil {
return nil, err
}
if request.Commit == nil {
request.Commit = &pfs.Commit{
Repo: request.Parent.Repo,
Id: uuid.NewWithoutDashes(),
}
}
for _, clientConn := range clientConns {
if _, err := pfs.NewInternalAPIClient(clientConn).StartCommit(ctx, request); err != nil {
return nil, err
}
}
return request.Commit, nil
}
func uniqueString(prefix string) string {
return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
func NewLocalServer() LocalServer {
return &localServer{
server: grpc.NewServer(),
path: filepath.Join("/tmp", uuid.NewWithoutDashes()),
}
}
func testBasicRethink(t *testing.T, apiServer persist.APIServer) {
_, err := apiServer.CreatePipelineInfo(
context.Background(),
&persist.PipelineInfo{
PipelineName: "foo",
},
)
require.NoError(t, err)
pipelineInfo, err := apiServer.GetPipelineInfo(
context.Background(),
&pps.Pipeline{Name: "foo"},
)
require.NoError(t, err)
require.Equal(t, pipelineInfo.PipelineName, "foo")
inputCommits := []*pfs.Commit{pfsutil.NewCommit("bar", uuid.NewWithoutDashes())}
jobInfo, err := apiServer.CreateJobInfo(
context.Background(),
&persist.JobInfo{
PipelineName: "foo",
InputCommit: inputCommits,
},
)
jobID := jobInfo.JobId
inputCommits2 := []*pfs.Commit{pfsutil.NewCommit("fizz", uuid.NewWithoutDashes())}
_, err = apiServer.CreateJobInfo(
context.Background(),
&persist.JobInfo{
PipelineName: "buzz",
InputCommit: inputCommits2,
},
)
require.NoError(t, err)
jobInfo, err = apiServer.InspectJob(
context.Background(),
&pps.InspectJobRequest{
Job: &pps.Job{
Id: jobInfo.JobId,
},
},
)
require.NoError(t, err)
require.Equal(t, jobInfo.JobId, jobID)
require.Equal(t, "foo", jobInfo.PipelineName)
jobInfos, err := apiServer.ListJobInfos(
context.Background(),
&pps.ListJobRequest{
Pipeline: &pps.Pipeline{Name: "foo"},
},
)
require.NoError(t, err)
require.Equal(t, len(jobInfos.JobInfo), 1)
require.Equal(t, jobInfos.JobInfo[0].JobId, jobID)
jobInfos, err = apiServer.ListJobInfos(
context.Background(),
&pps.ListJobRequest{
InputCommit: inputCommits,
},
)
require.NoError(t, err)
require.Equal(t, len(jobInfos.JobInfo), 1)
require.Equal(t, jobInfos.JobInfo[0].JobId, jobID)
jobInfos, err = apiServer.ListJobInfos(
context.Background(),
&pps.ListJobRequest{
Pipeline: &pps.Pipeline{Name: "foo"},
InputCommit: inputCommits,
},
)
require.NoError(t, err)
require.Equal(t, len(jobInfos.JobInfo), 1)
require.Equal(t, jobInfos.JobInfo[0].JobId, jobID)
}
func (c *mockClient) CreateInDir(dir string, value string, ttl uint64) error {
c.lock.Lock()
defer c.lock.Unlock()
key := path.Join(dir, uuid.NewWithoutDashes())
return c.unsafeSet(key, value, ttl)
}