Example #1
0
func (n *nodeRunner) run() error {
	var err error
	for name, parentChan := range n.parentChans {
		protolog.Debug(&NodeWaiting{Node: n.nodeName, ParentNode: name})
		select {
		case parentErr := <-parentChan:
			if parentErr != nil {
				err = parentErr
			}
			continue
		case <-n.cancel:
			return err
		}
	}
	protolog.Debug(&NodeFinishedWaiting{Node: n.nodeName, ParentError: errorString(err)})
	if err == nil {
		protolog.Info(&NodeStarting{Node: n.nodeName})
		err = n.f()
		protolog.Info(&NodeFinished{Node: n.nodeName, Error: errorString(err)})
	}
	for name, childChan := range n.childrenChans {
		protolog.Debug(&NodeSending{Node: n.nodeName, ChildNode: name, Error: errorString(err)})
		childChan <- err
		close(childChan)
	}
	return err
}
Example #2
0
func (f *file) Open(ctx context.Context, request *fuse.OpenRequest, response *fuse.OpenResponse) (_ fs.Handle, retErr error) {
	defer func() {
		protolog.Debug(&FileRead{&f.Node, errorToString(retErr)})
	}()
	atomic.AddInt32(&f.handles, 1)
	return f, nil
}
Example #3
0
func execSubvolumeExists(path string) (result bool) {
	defer func() {
		protolog.Debug(&SubvolumeExists{path, result})
	}()
	_, err := os.Stat(path)
	return err == nil
}
Example #4
0
func (a *sharder) announceFrontend(
	address string,
	frontend Frontend,
	versionChan chan int64,
	cancel chan bool,
) error {
	frontendState := &FrontendState{
		Address: address,
		Version: InvalidVersion,
	}
	for {
		encodedFrontendState, err := marshaler.MarshalToString(frontendState)
		if err != nil {
			return err
		}
		if err := a.discoveryClient.Set(a.frontendStateKey(address), encodedFrontendState, holdTTL); err != nil {
			protolog.Printf("Error setting server state: %s", err.Error())
		}
		protolog.Debug(&SetFrontendState{frontendState})
		select {
		case <-cancel:
			return nil
		case version := <-versionChan:
			frontendState.Version = version
		case <-time.After(time.Second * time.Duration(holdTTL/2)):
		}
	}
}
Example #5
0
// RunIO runs the command with the given IO and arguments.
func RunIO(ioObj IO, args ...string) error {
	if len(args) == 0 {
		return ErrNoArgs
	}
	var debugStderr io.ReadWriter
	stderr := ioObj.Stderr
	if globalDebug {
		debugStderr = bytes.NewBuffer(nil)
		if stderr == nil {
			stderr = debugStderr
		} else {
			stderr = io.MultiWriter(stderr, debugStderr)
		}
	}
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdin = ioObj.Stdin
	cmd.Stdout = ioObj.Stdout
	cmd.Stderr = stderr
	if globalDebug {
		protolog.Debug(&RunningCommand{Args: strings.Join(args, " ")})
	}
	if err := cmd.Run(); err != nil {
		if debugStderr != nil {
			data, _ := ioutil.ReadAll(debugStderr)
			if data != nil && len(data) > 0 {
				return fmt.Errorf("%s: %s\n\t%s", strings.Join(args, " "), err.Error(), string(data))
			}
		}
		return fmt.Errorf("%s: %s", strings.Join(args, " "), err.Error())
	}
	return nil
}
Example #6
0
func RunWithOptions(runOptions RunOptions, args ...string) error {
	if len(args) == 0 {
		return errors.New("run called with no args")
	}
	var debugStderr io.ReadWriter
	stderr := runOptions.stderr
	if debug {
		debugStderr = bytes.NewBuffer(nil)
		if stderr == nil {
			stderr = debugStderr
		} else {
			stderr = io.MultiWriter(stderr, debugStderr)
		}
	}
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdin = runOptions.stdin
	cmd.Stdout = runOptions.stdout
	cmd.Stderr = stderr
	argsString := strings.Join(args, " ")
	protolog.Debug(&RunningCommand{Args: argsString})
	if err := cmd.Run(); err != nil {
		if debugStderr != nil {
			data, _ := ioutil.ReadAll(debugStderr)
			if data != nil && len(data) > 0 {
				return fmt.Errorf("%s: %s\n\t%s", argsString, err.Error(), string(data))
			}
		}
		return fmt.Errorf("%s: %s", argsString, err.Error())
	}
	return nil
}
Example #7
0
func (a *discoveryAddresser) announceState(
	id string,
	address string,
	server Server,
	versionChan chan int64,
	cancel chan bool,
) error {
	serverState := &proto.ServerState{
		Id:      id,
		Address: address,
		Version: InvalidVersion,
	}
	for {
		shards, err := server.LocalShards()
		if err != nil {
			return err
		}
		serverState.Shards = shards
		encodedServerState, err := marshaler.MarshalToString(serverState)
		if err != nil {
			return err
		}
		if err := a.discoveryClient.Set(a.serverStateKey(id), encodedServerState, holdTTL); err != nil {
			return err
		}
		protolog.Debug(&log.SetServerState{serverState})
		select {
		case <-cancel:
			return nil
		case version := <-versionChan:
			serverState.Version = version
		case <-time.After(time.Second * time.Duration(holdTTL/2)):
		}
	}
}
Example #8
0
func execTransID(path string) (result string, retErr error) {
	defer func() {
		protolog.Debug(&TransID{path, result, errorToString(retErr)})
	}()
	//  "9223372036854775810" == 2 ** 63 we use a very big number there so that
	//  we get the transid of the from path. According to the internet this is
	//  the nicest way to get it from btrfs.
	var buffer bytes.Buffer
	if err := pkgexec.RunStdout(&buffer, "btrfs", "subvolume", "find-new", path, "9223372036854775808"); err != nil {
		return "", err
	}
	scanner := bufio.NewScanner(&buffer)
	for scanner.Scan() {
		// scanner.Text() looks like this:
		// transid marker was 907
		// 0       1      2   3
		tokens := strings.Split(scanner.Text(), " ")
		if len(tokens) != 4 {
			return "", fmt.Errorf("pachyderm: failed to parse find-new output")
		}
		return tokens[3], nil
	}
	if scanner.Err() != nil {
		return "", scanner.Err()
	}
	return "", fmt.Errorf("pachyderm: empty output from find-new")
}
Example #9
0
func execSubvolumeExists(path string) (result bool) {
	defer func() {
		protolog.Debug(&SubvolumeExists{path, result})
	}()
	if err := pkgexec.Run("btrfs", "subvolume", "show", path); err != nil {
		return false
	}
	return true
}
Example #10
0
func execSend(path string, parent string, diff io.Writer) (retErr error) {
	defer func() {
		protolog.Debug(&Send{path, parent, errorToString(retErr)})
	}()
	if parent == "" {
		return pkgexec.RunStdout(diff, "btrfs", "send", path)
	}
	return pkgexec.RunStdout(diff, "btrfs", "send", "-p", parent, path)
}
Example #11
0
func (d *directory) Lookup(ctx context.Context, name string) (result fs.Node, retErr error) {
	defer func() {
		protolog.Debug(&DirectoryLookup{&d.Node, name, getNode(result), errorToString(retErr)})
	}()
	if d.File.Commit.Repo.Name == "" {
		return d.lookUpRepo(ctx, name)
	}
	if d.File.Commit.Id == "" {
		return d.lookUpCommit(ctx, name)
	}
	return d.lookUpFile(ctx, name)
}
Example #12
0
func (d *directory) Attr(ctx context.Context, a *fuse.Attr) (retErr error) {
	defer func() {
		protolog.Debug(&DirectoryAttr{&d.Node, &Attr{uint32(a.Mode)}, errorToString(retErr)})
	}()
	a.Valid = time.Nanosecond
	if d.Write {
		a.Mode = os.ModeDir | 0775
	} else {
		a.Mode = os.ModeDir | 0555
	}
	a.Inode = d.fs.inode(d.File)
	return nil
}
Example #13
0
func execSubvolumeFindNew(commit string, fromCommit string, out io.Writer) (retErr error) {
	defer func() {
		protolog.Debug(&SubvolumeFindNew{commit, fromCommit, errorToString(retErr)})
	}()
	if fromCommit == "" {
		return pkgexec.RunStdout(out, "btrfs", "subvolume", "find-new", commit, "0")
	}
	transid, err := execTransID(fromCommit)
	if err != nil {
		return err
	}
	return pkgexec.RunStdout(out, "btrfs", "subvolume", "find-new", commit, transid)
}
Example #14
0
func (d *directory) Mkdir(ctx context.Context, request *fuse.MkdirRequest) (result fs.Node, retErr error) {
	defer func() {
		protolog.Debug(&DirectoryMkdir{&d.Node, getNode(result), errorToString(retErr)})
	}()
	if d.File.Commit.Id == "" {
		return nil, fuse.EPERM
	}
	if err := pfsutil.MakeDirectory(d.fs.apiClient, d.File.Commit.Repo.Name, d.File.Commit.Id, path.Join(d.File.Path, request.Name)); err != nil {
		return nil, err
	}
	localResult := d.copy()
	localResult.File.Path = path.Join(localResult.File.Path, request.Name)
	return localResult, nil
}
Example #15
0
func (f *file) Write(ctx context.Context, request *fuse.WriteRequest, response *fuse.WriteResponse) (retErr error) {
	defer func() {
		protolog.Debug(&FileWrite{&f.Node, errorToString(retErr)})
	}()
	written, err := pfsutil.PutFile(f.fs.apiClient, f.File.Commit.Repo.Name, f.File.Commit.Id, f.File.Path, request.Offset, bytes.NewReader(request.Data))
	if err != nil {
		return err
	}
	response.Size = written
	if f.size < request.Offset+int64(written) {
		f.size = request.Offset + int64(written)
	}
	return nil
}
Example #16
0
func (a *discoveryAddresser) GetShardToMasterAddress(version int64) (result map[uint64]string, retErr error) {
	defer func() {
		protolog.Debug(&log.GetShardToMasterAddress{version, result, errorToString(retErr)})
	}()
	addresses, err := a.getAddresses(version)
	if err != nil {
		return nil, err
	}
	_result := make(map[uint64]string)
	for shard, shardAddresses := range addresses.Addresses {
		_result[shard] = shardAddresses.Master
	}
	return _result, nil
}
Example #17
0
func (a *discoveryAddresser) GetMasterAddress(shard uint64, version int64) (result string, ok bool, retErr error) {
	defer func() {
		protolog.Debug(&log.GetMasterAddress{shard, version, result, ok, errorToString(retErr)})
	}()
	addresses, err := a.getAddresses(version)
	if err != nil {
		return "", false, err
	}
	shardAddresses, ok := addresses.Addresses[shard]
	if !ok {
		return "", false, nil
	}
	return shardAddresses.Master, true, nil
}
Example #18
0
func (a *discoveryAddresser) GetReplicaAddresses(shard uint64, version int64) (result map[string]bool, retErr error) {
	defer func() {
		protolog.Debug(&log.GetReplicaAddresses{shard, version, result, errorToString(retErr)})
	}()
	addresses, err := a.getAddresses(version)
	if err != nil {
		return nil, err
	}
	shardAddresses, ok := addresses.Addresses[shard]
	if !ok {
		return nil, fmt.Errorf("shard %d not found", shard)
	}
	return shardAddresses.Replicas, nil
}
func (f *filesystem) Root() (result fs.Node, retErr error) {
	defer func() {
		protolog.Debug(&Root{&f.Filesystem, getNode(result), errorToString(retErr)})
	}()
	return &directory{
		f,
		Node{
			File: &pfs.File{
				Commit: &pfs.Commit{
					Repo: &pfs.Repo{},
				},
			},
		},
	}, nil
}
Example #20
0
func (d *directory) Create(ctx context.Context, request *fuse.CreateRequest, response *fuse.CreateResponse) (result fs.Node, _ fs.Handle, retErr error) {
	defer func() {
		protolog.Debug(&DirectoryCreate{&d.Node, getNode(result), errorToString(retErr)})
	}()
	if d.File.Commit.Id == "" {
		return nil, 0, fuse.EPERM
	}
	directory := d.copy()
	directory.File.Path = path.Join(directory.File.Path, request.Name)
	localResult := &file{*directory, 0, 0}
	handle, err := localResult.Open(ctx, nil, nil)
	if err != nil {
		return nil, nil, err
	}
	return localResult, handle, nil
}
Example #21
0
func (a *discoveryAddresser) GetShardToReplicaAddresses(version int64) (result map[uint64]map[string]bool, retErr error) {
	defer func() {
		// We need resultPrime is because proto3 can't do maps of maps.
		resultPrime := make(map[uint64]*log.ReplicaAddresses)
		for shard, addresses := range result {
			resultPrime[shard] = &log.ReplicaAddresses{addresses}
		}
		protolog.Debug(&log.GetShardToReplicaAddresses{version, resultPrime, errorToString(retErr)})
	}()
	addresses, err := a.getAddresses(version)
	if err != nil {
		return nil, err
	}
	_result := make(map[uint64]map[string]bool)
	for shard, shardAddresses := range addresses.Addresses {
		_result[shard] = shardAddresses.Replicas
	}
	return _result, nil
}
Example #22
0
func execSubvolumeList(path string, fromPath string, ascending bool, out io.Writer) (retErr error) {
	defer func() {
		protolog.Debug(&SubvolumeList{path, fromPath, ascending, errorToString(retErr)})
	}()
	var sort string
	if ascending {
		sort = "+ogen"
	} else {
		sort = "-ogen"
	}

	if fromPath == "" {
		return pkgexec.RunStdout(out, "btrfs", "subvolume", "list", "-a", "--sort", sort, path)
	}
	transid, err := execTransID(fromPath)
	if err != nil {
		return err
	}
	return pkgexec.RunStdout(out, "btrfs", "subvolume", "list", "-aC", "+"+transid, "--sort", sort, path)
}
Example #23
0
func (a *discoveryAddresser) Version() (result int64, retErr error) {
	defer func() {
		protolog.Debug(&log.Version{result, errorToString(retErr)})
	}()
	minVersion := int64(math.MaxInt64)
	encodedServerStates, err := a.discoveryClient.GetAll(a.serverStateDir())
	if err != nil {
		return 0, err
	}
	for _, encodedServerState := range encodedServerStates {
		serverState, err := decodeServerState(encodedServerState)
		if err != nil {
			return 0, err
		}
		if serverState.Version < minVersion {
			minVersion = serverState.Version
		}
	}
	return minVersion, nil
}
Example #24
0
func (f *file) Read(ctx context.Context, request *fuse.ReadRequest, response *fuse.ReadResponse) (retErr error) {
	defer func() {
		protolog.Debug(&FileRead{&f.Node, errorToString(retErr)})
	}()
	var buffer bytes.Buffer
	if err := pfsutil.GetFile(
		f.fs.apiClient,
		f.File.Commit.Repo.Name,
		f.File.Commit.Id,
		f.File.Path,
		request.Offset,
		int64(request.Size),
		f.fs.Shard,
		&buffer,
	); err != nil {
		return err
	}
	response.Data = buffer.Bytes()
	return nil
}
Example #25
0
func (d *directory) ReadDirAll(ctx context.Context) (result []fuse.Dirent, retErr error) {
	defer func() {
		var dirents []*Dirent
		for _, dirent := range result {
			dirents = append(dirents, &Dirent{dirent.Inode, dirent.Name})
		}
		protolog.Debug(&DirectoryReadDirAll{&d.Node, dirents, errorToString(retErr)})
	}()
	if d.File.Commit.Repo.Name == "" {
		return d.readRepos(ctx)
	}
	if d.File.Commit.Id == "" {
		commitMount := d.fs.getCommitMount(d.File.Commit.Repo.Name)
		if commitMount != nil && commitMount.Commit.Id != "" {
			d.File.Commit.Id = commitMount.Commit.Id
			return d.readFiles(ctx)
		}
		return d.readCommits(ctx)
	}
	return d.readFiles(ctx)
}
Example #26
0
func (f *file) Attr(ctx context.Context, a *fuse.Attr) (retErr error) {
	defer func() {
		protolog.Debug(&FileAttr{&f.Node, &Attr{uint32(a.Mode)}, errorToString(retErr)})
	}()
	fileInfo, err := pfsutil.InspectFile(
		f.fs.apiClient,
		f.File.Commit.Repo.Name,
		f.File.Commit.Id,
		f.File.Path,
		f.fs.Shard,
	)
	if err != nil && !f.local {
		return err
	}
	if fileInfo != nil {
		a.Size = fileInfo.SizeBytes
	}
	a.Mode = 0666
	a.Inode = f.fs.inode(f.File)
	return nil
}
Example #27
0
// Debug logs an RPC call at the debug level.
func Debug(serviceName string, methodName string, request proto.Message, response proto.Message, err error, duration time.Duration) {
	protolog.Debug(event(serviceName, methodName, request, response, err, duration))
}
Example #28
0
func execRecv(path string, diff io.Reader) (retErr error) {
	defer func() {
		protolog.Debug(&Recv{path, errorToString(retErr)})
	}()
	return pkgexec.RunStdin(diff, "btrfs", "receive", path)
}
Example #29
0
func (c *commitScanner) Commit() string {
	commit, _ := c.parseCommit()
	protolog.Debug(&SubvolumeListLine{c.textScanner.Text()})
	return commit
}
Example #30
0
func (c *changeScanner) Change() *pfs.Change {
	change, _ := c.parseChange()
	protolog.Debug(&SubvolumeFindNewLine{c.textScanner.Text()})
	return change
}