Esempio n. 1
0
func (s *simpleTest) setup() {
	if s.inputRepoName == "" {
		s.inputRepoName = uuid.NewWithoutDashes()
	}
	if s.outputRepoName == "" {
		s.outputRepoName = uuid.NewWithoutDashes()
	}
	if s.pipelineName == "" {
		s.pipelineName = uuid.NewWithoutDashes()
	}
	if s.jobTimeoutSec == 0 {
		s.jobTimeoutSec = defaultJobTimeoutSec
	}
}
Esempio n. 2
0
func (a *apiServer) StartCommit(ctx context.Context, request *pfs.StartCommitRequest) (response *pfs.Commit, retErr error) {
	defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
	a.versionLock.RLock()
	defer a.versionLock.RUnlock()
	ctx = versionToContext(a.version, ctx)
	clientConns, err := a.router.GetAllClientConns(a.version)
	if err != nil {
		return nil, err
	}
	if request.Commit == nil {
		if request.Parent == nil {
			return nil, fmt.Errorf("one of Parent or Commit must be non nil")
		}
		request.Commit = &pfs.Commit{
			Repo: request.Parent.Repo,
			Id:   uuid.NewWithoutDashes(),
		}
	}
	for _, clientConn := range clientConns {
		if _, err := pfs.NewInternalAPIClient(clientConn).StartCommit(ctx, request); err != nil {
			return nil, err
		}
	}
	return request.Commit, nil
}
Esempio n. 3
0
func (a *apiServer) CreateJob(ctx context.Context, request *pps.CreateJobRequest) (response *pps.Job, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	persistJobInfo := &persist.JobInfo{
		Input:        request.Input,
		OutputParent: request.OutputParent,
	}
	if request.GetTransform() != nil {
		persistJobInfo.Spec = &persist.JobInfo_Transform{
			Transform: request.GetTransform(),
		}
	} else if request.GetPipeline() != nil {
		persistJobInfo.Spec = &persist.JobInfo_PipelineName{
			PipelineName: request.GetPipeline().Name,
		}
	} else {
		return nil, fmt.Errorf("pachyderm.pps.jobserver: both transform and pipeline are not set on %v", request)
	}
	persistJobInfo.JobId = uuid.NewWithoutDashes()
	persistJobInfo.CreatedAt = prototime.TimeToTimestamp(time.Now())
	if a.kubeClient == nil {
		return nil, fmt.Errorf("pachyderm.pps.jobserver: no job backend")
	}
	if _, err := a.kubeClient.Jobs(api.NamespaceDefault).Create(job(persistJobInfo)); err != nil {
		return nil, err
	}
	_, err = a.persistAPIClient.CreateJobInfo(ctx, persistJobInfo)
	if err != nil {
		return nil, err
	}
	return &pps.Job{
		Id: persistJobInfo.JobId,
	}, nil
}
Esempio n. 4
0
func (j *jobRunner) runJobInfo(persistJobInfo *persist.JobInfo) error {
	switch {
	case persistJobInfo.GetTransform() != nil:
		return j.reallyRunJobInfo(
			uuid.NewWithoutDashes(),
			persistJobInfo.JobId,
			persistJobInfo.GetTransform(),
			persistJobInfo.Input,
			persistJobInfo.OutputParent,
			1,
		)
	case persistJobInfo.GetPipelineName() != "":
		persistPipelineInfo, err := j.persistAPIClient.GetPipelineInfo(
			context.Background(),
			&pps.Pipeline{Name: persistJobInfo.GetPipelineName()},
		)
		if err != nil {
			return err
		}
		if persistPipelineInfo.GetTransform() == nil {
			return fmt.Errorf("pachyderm.pps.server: transform not set on pipeline info %v", persistPipelineInfo)
		}
		return j.reallyRunJobInfo(
			persistPipelineInfo.PipelineName,
			persistJobInfo.JobId,
			persistPipelineInfo.GetTransform(),
			persistJobInfo.Input,
			persistJobInfo.OutputParent,
			1,
		)
	default:
		return fmt.Errorf("pachyderm.pps.server: neither transform or pipeline name set on job info %v", persistJobInfo)
	}
}
Esempio n. 5
0
func (a *apiServer) createDisks(ctx context.Context, nodes uint64) ([]string, error) {
	var names []string
	for i := uint64(0); i < nodes; i++ {
		name := "disk-" + uuid.NewWithoutDashes()
		if err := a.provider.CreateDisk(name, defaultDiskSizeGb); err != nil {
			return nil, err
		}
		names = append(names, name)
	}
	return names, nil
}
Esempio n. 6
0
func NewTestRethinkAPIServer() (server.APIServer, error) {
	address, err := getTestRethinkAddress()
	if err != nil {
		return nil, err
	}
	databaseName := uuid.NewWithoutDashes()
	if err := server.InitDBs(address, databaseName); err != nil {
		return nil, err
	}
	return server.NewRethinkAPIServer(address, databaseName)
}
Esempio n. 7
0
// JobId cannot be set
// Timestamp cannot be set
func (a *rethinkAPIServer) CreateJobInfo(ctx context.Context, request *persist.JobInfo) (response *persist.JobInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if request.JobId != "" {
		return nil, fmt.Errorf("request.JobId should be unset")
	}
	if request.CreatedAt != nil {
		return nil, fmt.Errorf("request.CreatedAt should be unset")
	}
	if request.CommitIndex != "" {
		return nil, fmt.Errorf("request.CommitIndex should be unset")
	}
	request.JobId = uuid.NewWithoutDashes()
	request.CreatedAt = prototime.TimeToTimestamp(time.Now())
	request.CommitIndex = genCommitIndex(request.InputCommit)
	if err := a.insertMessage(jobInfosTable, request); err != nil {
		return nil, err
	}
	return request, nil
}
Esempio n. 8
0
func (a *apiServer) StartCommit(ctx context.Context, request *pfs.StartCommitRequest) (*pfs.Commit, error) {
	a.versionLock.RLock()
	defer a.versionLock.RUnlock()
	ctx = versionToContext(a.version, ctx)
	clientConns, err := a.router.GetAllClientConns(a.version)
	if err != nil {
		return nil, err
	}
	if request.Commit == nil {
		request.Commit = &pfs.Commit{
			Repo: request.Parent.Repo,
			Id:   uuid.NewWithoutDashes(),
		}
	}
	for _, clientConn := range clientConns {
		if _, err := pfs.NewInternalAPIClient(clientConn).StartCommit(ctx, request); err != nil {
			return nil, err
		}
	}
	return request.Commit, nil
}
Esempio n. 9
0
func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
Esempio n. 10
0
func NewLocalServer() LocalServer {
	return &localServer{
		server: grpc.NewServer(),
		path:   filepath.Join("/tmp", uuid.NewWithoutDashes()),
	}
}
func testBasicRethink(t *testing.T, apiServer persist.APIServer) {
	_, err := apiServer.CreatePipelineInfo(
		context.Background(),
		&persist.PipelineInfo{
			PipelineName: "foo",
		},
	)
	require.NoError(t, err)
	pipelineInfo, err := apiServer.GetPipelineInfo(
		context.Background(),
		&pps.Pipeline{Name: "foo"},
	)
	require.NoError(t, err)
	require.Equal(t, pipelineInfo.PipelineName, "foo")
	inputCommits := []*pfs.Commit{pfsutil.NewCommit("bar", uuid.NewWithoutDashes())}
	jobInfo, err := apiServer.CreateJobInfo(
		context.Background(),
		&persist.JobInfo{
			PipelineName: "foo",
			InputCommit:  inputCommits,
		},
	)
	jobID := jobInfo.JobId
	inputCommits2 := []*pfs.Commit{pfsutil.NewCommit("fizz", uuid.NewWithoutDashes())}
	_, err = apiServer.CreateJobInfo(
		context.Background(),
		&persist.JobInfo{
			PipelineName: "buzz",
			InputCommit:  inputCommits2,
		},
	)
	require.NoError(t, err)
	jobInfo, err = apiServer.InspectJob(
		context.Background(),
		&pps.InspectJobRequest{
			Job: &pps.Job{
				Id: jobInfo.JobId,
			},
		},
	)
	require.NoError(t, err)
	require.Equal(t, jobInfo.JobId, jobID)
	require.Equal(t, "foo", jobInfo.PipelineName)
	jobInfos, err := apiServer.ListJobInfos(
		context.Background(),
		&pps.ListJobRequest{
			Pipeline: &pps.Pipeline{Name: "foo"},
		},
	)
	require.NoError(t, err)
	require.Equal(t, len(jobInfos.JobInfo), 1)
	require.Equal(t, jobInfos.JobInfo[0].JobId, jobID)
	jobInfos, err = apiServer.ListJobInfos(
		context.Background(),
		&pps.ListJobRequest{
			InputCommit: inputCommits,
		},
	)
	require.NoError(t, err)
	require.Equal(t, len(jobInfos.JobInfo), 1)
	require.Equal(t, jobInfos.JobInfo[0].JobId, jobID)
	jobInfos, err = apiServer.ListJobInfos(
		context.Background(),
		&pps.ListJobRequest{
			Pipeline:    &pps.Pipeline{Name: "foo"},
			InputCommit: inputCommits,
		},
	)
	require.NoError(t, err)
	require.Equal(t, len(jobInfos.JobInfo), 1)
	require.Equal(t, jobInfos.JobInfo[0].JobId, jobID)
}
Esempio n. 12
0
func (c *mockClient) CreateInDir(dir string, value string, ttl uint64) error {
	c.lock.Lock()
	defer c.lock.Unlock()
	key := path.Join(dir, uuid.NewWithoutDashes())
	return c.unsafeSet(key, value, ttl)
}