func (s *simpleTest) setup() { if s.inputRepoName == "" { s.inputRepoName = uuid.NewWithoutDashes() } if s.outputRepoName == "" { s.outputRepoName = uuid.NewWithoutDashes() } if s.pipelineName == "" { s.pipelineName = uuid.NewWithoutDashes() } if s.jobTimeoutSec == 0 { s.jobTimeoutSec = defaultJobTimeoutSec } }
func (a *apiServer) StartCommit(ctx context.Context, request *pfs.StartCommitRequest) (response *pfs.Commit, retErr error) { defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now()) a.versionLock.RLock() defer a.versionLock.RUnlock() ctx = versionToContext(a.version, ctx) clientConns, err := a.router.GetAllClientConns(a.version) if err != nil { return nil, err } if request.Commit == nil { if request.Parent == nil { return nil, fmt.Errorf("one of Parent or Commit must be non nil") } request.Commit = &pfs.Commit{ Repo: request.Parent.Repo, Id: uuid.NewWithoutDashes(), } } for _, clientConn := range clientConns { if _, err := pfs.NewInternalAPIClient(clientConn).StartCommit(ctx, request); err != nil { return nil, err } } return request.Commit, nil }
func (a *apiServer) CreateJob(ctx context.Context, request *pps.CreateJobRequest) (response *pps.Job, err error) { defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now()) persistJobInfo := &persist.JobInfo{ Input: request.Input, OutputParent: request.OutputParent, } if request.GetTransform() != nil { persistJobInfo.Spec = &persist.JobInfo_Transform{ Transform: request.GetTransform(), } } else if request.GetPipeline() != nil { persistJobInfo.Spec = &persist.JobInfo_PipelineName{ PipelineName: request.GetPipeline().Name, } } else { return nil, fmt.Errorf("pachyderm.pps.jobserver: both transform and pipeline are not set on %v", request) } persistJobInfo.JobId = uuid.NewWithoutDashes() persistJobInfo.CreatedAt = prototime.TimeToTimestamp(time.Now()) if a.kubeClient == nil { return nil, fmt.Errorf("pachyderm.pps.jobserver: no job backend") } if _, err := a.kubeClient.Jobs(api.NamespaceDefault).Create(job(persistJobInfo)); err != nil { return nil, err } _, err = a.persistAPIClient.CreateJobInfo(ctx, persistJobInfo) if err != nil { return nil, err } return &pps.Job{ Id: persistJobInfo.JobId, }, nil }
func (j *jobRunner) runJobInfo(persistJobInfo *persist.JobInfo) error { switch { case persistJobInfo.GetTransform() != nil: return j.reallyRunJobInfo( uuid.NewWithoutDashes(), persistJobInfo.JobId, persistJobInfo.GetTransform(), persistJobInfo.Input, persistJobInfo.OutputParent, 1, ) case persistJobInfo.GetPipelineName() != "": persistPipelineInfo, err := j.persistAPIClient.GetPipelineInfo( context.Background(), &pps.Pipeline{Name: persistJobInfo.GetPipelineName()}, ) if err != nil { return err } if persistPipelineInfo.GetTransform() == nil { return fmt.Errorf("pachyderm.pps.server: transform not set on pipeline info %v", persistPipelineInfo) } return j.reallyRunJobInfo( persistPipelineInfo.PipelineName, persistJobInfo.JobId, persistPipelineInfo.GetTransform(), persistJobInfo.Input, persistJobInfo.OutputParent, 1, ) default: return fmt.Errorf("pachyderm.pps.server: neither transform or pipeline name set on job info %v", persistJobInfo) } }
func (a *apiServer) createDisks(ctx context.Context, nodes uint64) ([]string, error) { var names []string for i := uint64(0); i < nodes; i++ { name := "disk-" + uuid.NewWithoutDashes() if err := a.provider.CreateDisk(name, defaultDiskSizeGb); err != nil { return nil, err } names = append(names, name) } return names, nil }
func NewTestRethinkAPIServer() (server.APIServer, error) { address, err := getTestRethinkAddress() if err != nil { return nil, err } databaseName := uuid.NewWithoutDashes() if err := server.InitDBs(address, databaseName); err != nil { return nil, err } return server.NewRethinkAPIServer(address, databaseName) }
// JobId cannot be set // Timestamp cannot be set func (a *rethinkAPIServer) CreateJobInfo(ctx context.Context, request *persist.JobInfo) (response *persist.JobInfo, err error) { defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now()) if request.JobId != "" { return nil, fmt.Errorf("request.JobId should be unset") } if request.CreatedAt != nil { return nil, fmt.Errorf("request.CreatedAt should be unset") } if request.CommitIndex != "" { return nil, fmt.Errorf("request.CommitIndex should be unset") } request.JobId = uuid.NewWithoutDashes() request.CreatedAt = prototime.TimeToTimestamp(time.Now()) request.CommitIndex = genCommitIndex(request.InputCommit) if err := a.insertMessage(jobInfosTable, request); err != nil { return nil, err } return request, nil }
func (a *apiServer) StartCommit(ctx context.Context, request *pfs.StartCommitRequest) (*pfs.Commit, error) { a.versionLock.RLock() defer a.versionLock.RUnlock() ctx = versionToContext(a.version, ctx) clientConns, err := a.router.GetAllClientConns(a.version) if err != nil { return nil, err } if request.Commit == nil { request.Commit = &pfs.Commit{ Repo: request.Parent.Repo, Id: uuid.NewWithoutDashes(), } } for _, clientConn := range clientConns { if _, err := pfs.NewInternalAPIClient(clientConn).StartCommit(ctx, request); err != nil { return nil, err } } return request.Commit, nil }
func uniqueString(prefix string) string { return prefix + "." + uuid.NewWithoutDashes()[0:12] }
func NewLocalServer() LocalServer { return &localServer{ server: grpc.NewServer(), path: filepath.Join("/tmp", uuid.NewWithoutDashes()), } }
func testBasicRethink(t *testing.T, apiServer persist.APIServer) { _, err := apiServer.CreatePipelineInfo( context.Background(), &persist.PipelineInfo{ PipelineName: "foo", }, ) require.NoError(t, err) pipelineInfo, err := apiServer.GetPipelineInfo( context.Background(), &pps.Pipeline{Name: "foo"}, ) require.NoError(t, err) require.Equal(t, pipelineInfo.PipelineName, "foo") inputCommits := []*pfs.Commit{pfsutil.NewCommit("bar", uuid.NewWithoutDashes())} jobInfo, err := apiServer.CreateJobInfo( context.Background(), &persist.JobInfo{ PipelineName: "foo", InputCommit: inputCommits, }, ) jobID := jobInfo.JobId inputCommits2 := []*pfs.Commit{pfsutil.NewCommit("fizz", uuid.NewWithoutDashes())} _, err = apiServer.CreateJobInfo( context.Background(), &persist.JobInfo{ PipelineName: "buzz", InputCommit: inputCommits2, }, ) require.NoError(t, err) jobInfo, err = apiServer.InspectJob( context.Background(), &pps.InspectJobRequest{ Job: &pps.Job{ Id: jobInfo.JobId, }, }, ) require.NoError(t, err) require.Equal(t, jobInfo.JobId, jobID) require.Equal(t, "foo", jobInfo.PipelineName) jobInfos, err := apiServer.ListJobInfos( context.Background(), &pps.ListJobRequest{ Pipeline: &pps.Pipeline{Name: "foo"}, }, ) require.NoError(t, err) require.Equal(t, len(jobInfos.JobInfo), 1) require.Equal(t, jobInfos.JobInfo[0].JobId, jobID) jobInfos, err = apiServer.ListJobInfos( context.Background(), &pps.ListJobRequest{ InputCommit: inputCommits, }, ) require.NoError(t, err) require.Equal(t, len(jobInfos.JobInfo), 1) require.Equal(t, jobInfos.JobInfo[0].JobId, jobID) jobInfos, err = apiServer.ListJobInfos( context.Background(), &pps.ListJobRequest{ Pipeline: &pps.Pipeline{Name: "foo"}, InputCommit: inputCommits, }, ) require.NoError(t, err) require.Equal(t, len(jobInfos.JobInfo), 1) require.Equal(t, jobInfos.JobInfo[0].JobId, jobID) }
func (c *mockClient) CreateInDir(dir string, value string, ttl uint64) error { c.lock.Lock() defer c.lock.Unlock() key := path.Join(dir, uuid.NewWithoutDashes()) return c.unsafeSet(key, value, ttl) }