func (n *nodeRunner) run() error {
var err error
for name, parentChan := range n.parentChans {
protolog.Debug(&NodeWaiting{Node: n.nodeName, ParentNode: name})
select {
case parentErr := <-parentChan:
if parentErr != nil {
err = parentErr
}
continue
case <-n.cancel:
return err
}
}
protolog.Debug(&NodeFinishedWaiting{Node: n.nodeName, ParentError: errorString(err)})
if err == nil {
protolog.Info(&NodeStarting{Node: n.nodeName})
err = n.f()
protolog.Info(&NodeFinished{Node: n.nodeName, Error: errorString(err)})
}
for name, childChan := range n.childrenChans {
protolog.Debug(&NodeSending{Node: n.nodeName, ChildNode: name, Error: errorString(err)})
childChan <- err
close(childChan)
}
return err
}
func (f *file) Open(ctx context.Context, request *fuse.OpenRequest, response *fuse.OpenResponse) (_ fs.Handle, retErr error) {
defer func() {
protolog.Debug(&FileRead{&f.Node, errorToString(retErr)})
}()
atomic.AddInt32(&f.handles, 1)
return f, nil
}
func execSubvolumeExists(path string) (result bool) {
defer func() {
protolog.Debug(&SubvolumeExists{path, result})
}()
_, err := os.Stat(path)
return err == nil
}
func (a *sharder) announceFrontend(
address string,
frontend Frontend,
versionChan chan int64,
cancel chan bool,
) error {
frontendState := &FrontendState{
Address: address,
Version: InvalidVersion,
}
for {
encodedFrontendState, err := marshaler.MarshalToString(frontendState)
if err != nil {
return err
}
if err := a.discoveryClient.Set(a.frontendStateKey(address), encodedFrontendState, holdTTL); err != nil {
protolog.Printf("Error setting server state: %s", err.Error())
}
protolog.Debug(&SetFrontendState{frontendState})
select {
case <-cancel:
return nil
case version := <-versionChan:
frontendState.Version = version
case <-time.After(time.Second * time.Duration(holdTTL/2)):
}
}
}
// RunIO runs the command with the given IO and arguments.
func RunIO(ioObj IO, args ...string) error {
if len(args) == 0 {
return ErrNoArgs
}
var debugStderr io.ReadWriter
stderr := ioObj.Stderr
if globalDebug {
debugStderr = bytes.NewBuffer(nil)
if stderr == nil {
stderr = debugStderr
} else {
stderr = io.MultiWriter(stderr, debugStderr)
}
}
cmd := exec.Command(args[0], args[1:]...)
cmd.Stdin = ioObj.Stdin
cmd.Stdout = ioObj.Stdout
cmd.Stderr = stderr
if globalDebug {
protolog.Debug(&RunningCommand{Args: strings.Join(args, " ")})
}
if err := cmd.Run(); err != nil {
if debugStderr != nil {
data, _ := ioutil.ReadAll(debugStderr)
if data != nil && len(data) > 0 {
return fmt.Errorf("%s: %s\n\t%s", strings.Join(args, " "), err.Error(), string(data))
}
}
return fmt.Errorf("%s: %s", strings.Join(args, " "), err.Error())
}
return nil
}
func RunWithOptions(runOptions RunOptions, args ...string) error {
if len(args) == 0 {
return errors.New("run called with no args")
}
var debugStderr io.ReadWriter
stderr := runOptions.stderr
if debug {
debugStderr = bytes.NewBuffer(nil)
if stderr == nil {
stderr = debugStderr
} else {
stderr = io.MultiWriter(stderr, debugStderr)
}
}
cmd := exec.Command(args[0], args[1:]...)
cmd.Stdin = runOptions.stdin
cmd.Stdout = runOptions.stdout
cmd.Stderr = stderr
argsString := strings.Join(args, " ")
protolog.Debug(&RunningCommand{Args: argsString})
if err := cmd.Run(); err != nil {
if debugStderr != nil {
data, _ := ioutil.ReadAll(debugStderr)
if data != nil && len(data) > 0 {
return fmt.Errorf("%s: %s\n\t%s", argsString, err.Error(), string(data))
}
}
return fmt.Errorf("%s: %s", argsString, err.Error())
}
return nil
}
func (a *discoveryAddresser) announceState(
id string,
address string,
server Server,
versionChan chan int64,
cancel chan bool,
) error {
serverState := &proto.ServerState{
Id: id,
Address: address,
Version: InvalidVersion,
}
for {
shards, err := server.LocalShards()
if err != nil {
return err
}
serverState.Shards = shards
encodedServerState, err := marshaler.MarshalToString(serverState)
if err != nil {
return err
}
if err := a.discoveryClient.Set(a.serverStateKey(id), encodedServerState, holdTTL); err != nil {
return err
}
protolog.Debug(&log.SetServerState{serverState})
select {
case <-cancel:
return nil
case version := <-versionChan:
serverState.Version = version
case <-time.After(time.Second * time.Duration(holdTTL/2)):
}
}
}
func execTransID(path string) (result string, retErr error) {
defer func() {
protolog.Debug(&TransID{path, result, errorToString(retErr)})
}()
// "9223372036854775810" == 2 ** 63 we use a very big number there so that
// we get the transid of the from path. According to the internet this is
// the nicest way to get it from btrfs.
var buffer bytes.Buffer
if err := pkgexec.RunStdout(&buffer, "btrfs", "subvolume", "find-new", path, "9223372036854775808"); err != nil {
return "", err
}
scanner := bufio.NewScanner(&buffer)
for scanner.Scan() {
// scanner.Text() looks like this:
// transid marker was 907
// 0 1 2 3
tokens := strings.Split(scanner.Text(), " ")
if len(tokens) != 4 {
return "", fmt.Errorf("pachyderm: failed to parse find-new output")
}
return tokens[3], nil
}
if scanner.Err() != nil {
return "", scanner.Err()
}
return "", fmt.Errorf("pachyderm: empty output from find-new")
}
func execSubvolumeExists(path string) (result bool) {
defer func() {
protolog.Debug(&SubvolumeExists{path, result})
}()
if err := pkgexec.Run("btrfs", "subvolume", "show", path); err != nil {
return false
}
return true
}
func execSend(path string, parent string, diff io.Writer) (retErr error) {
defer func() {
protolog.Debug(&Send{path, parent, errorToString(retErr)})
}()
if parent == "" {
return pkgexec.RunStdout(diff, "btrfs", "send", path)
}
return pkgexec.RunStdout(diff, "btrfs", "send", "-p", parent, path)
}
func (d *directory) Lookup(ctx context.Context, name string) (result fs.Node, retErr error) {
defer func() {
protolog.Debug(&DirectoryLookup{&d.Node, name, getNode(result), errorToString(retErr)})
}()
if d.File.Commit.Repo.Name == "" {
return d.lookUpRepo(ctx, name)
}
if d.File.Commit.Id == "" {
return d.lookUpCommit(ctx, name)
}
return d.lookUpFile(ctx, name)
}
func (d *directory) Attr(ctx context.Context, a *fuse.Attr) (retErr error) {
defer func() {
protolog.Debug(&DirectoryAttr{&d.Node, &Attr{uint32(a.Mode)}, errorToString(retErr)})
}()
a.Valid = time.Nanosecond
if d.Write {
a.Mode = os.ModeDir | 0775
} else {
a.Mode = os.ModeDir | 0555
}
a.Inode = d.fs.inode(d.File)
return nil
}
func execSubvolumeFindNew(commit string, fromCommit string, out io.Writer) (retErr error) {
defer func() {
protolog.Debug(&SubvolumeFindNew{commit, fromCommit, errorToString(retErr)})
}()
if fromCommit == "" {
return pkgexec.RunStdout(out, "btrfs", "subvolume", "find-new", commit, "0")
}
transid, err := execTransID(fromCommit)
if err != nil {
return err
}
return pkgexec.RunStdout(out, "btrfs", "subvolume", "find-new", commit, transid)
}
func (d *directory) Mkdir(ctx context.Context, request *fuse.MkdirRequest) (result fs.Node, retErr error) {
defer func() {
protolog.Debug(&DirectoryMkdir{&d.Node, getNode(result), errorToString(retErr)})
}()
if d.File.Commit.Id == "" {
return nil, fuse.EPERM
}
if err := pfsutil.MakeDirectory(d.fs.apiClient, d.File.Commit.Repo.Name, d.File.Commit.Id, path.Join(d.File.Path, request.Name)); err != nil {
return nil, err
}
localResult := d.copy()
localResult.File.Path = path.Join(localResult.File.Path, request.Name)
return localResult, nil
}
func (f *file) Write(ctx context.Context, request *fuse.WriteRequest, response *fuse.WriteResponse) (retErr error) {
defer func() {
protolog.Debug(&FileWrite{&f.Node, errorToString(retErr)})
}()
written, err := pfsutil.PutFile(f.fs.apiClient, f.File.Commit.Repo.Name, f.File.Commit.Id, f.File.Path, request.Offset, bytes.NewReader(request.Data))
if err != nil {
return err
}
response.Size = written
if f.size < request.Offset+int64(written) {
f.size = request.Offset + int64(written)
}
return nil
}
func (a *discoveryAddresser) GetShardToMasterAddress(version int64) (result map[uint64]string, retErr error) {
defer func() {
protolog.Debug(&log.GetShardToMasterAddress{version, result, errorToString(retErr)})
}()
addresses, err := a.getAddresses(version)
if err != nil {
return nil, err
}
_result := make(map[uint64]string)
for shard, shardAddresses := range addresses.Addresses {
_result[shard] = shardAddresses.Master
}
return _result, nil
}
func (a *discoveryAddresser) GetMasterAddress(shard uint64, version int64) (result string, ok bool, retErr error) {
defer func() {
protolog.Debug(&log.GetMasterAddress{shard, version, result, ok, errorToString(retErr)})
}()
addresses, err := a.getAddresses(version)
if err != nil {
return "", false, err
}
shardAddresses, ok := addresses.Addresses[shard]
if !ok {
return "", false, nil
}
return shardAddresses.Master, true, nil
}
func (a *discoveryAddresser) GetReplicaAddresses(shard uint64, version int64) (result map[string]bool, retErr error) {
defer func() {
protolog.Debug(&log.GetReplicaAddresses{shard, version, result, errorToString(retErr)})
}()
addresses, err := a.getAddresses(version)
if err != nil {
return nil, err
}
shardAddresses, ok := addresses.Addresses[shard]
if !ok {
return nil, fmt.Errorf("shard %d not found", shard)
}
return shardAddresses.Replicas, nil
}
func (f *filesystem) Root() (result fs.Node, retErr error) {
defer func() {
protolog.Debug(&Root{&f.Filesystem, getNode(result), errorToString(retErr)})
}()
return &directory{
f,
Node{
File: &pfs.File{
Commit: &pfs.Commit{
Repo: &pfs.Repo{},
},
},
},
}, nil
}
func (d *directory) Create(ctx context.Context, request *fuse.CreateRequest, response *fuse.CreateResponse) (result fs.Node, _ fs.Handle, retErr error) {
defer func() {
protolog.Debug(&DirectoryCreate{&d.Node, getNode(result), errorToString(retErr)})
}()
if d.File.Commit.Id == "" {
return nil, 0, fuse.EPERM
}
directory := d.copy()
directory.File.Path = path.Join(directory.File.Path, request.Name)
localResult := &file{*directory, 0, 0}
handle, err := localResult.Open(ctx, nil, nil)
if err != nil {
return nil, nil, err
}
return localResult, handle, nil
}
func (a *discoveryAddresser) GetShardToReplicaAddresses(version int64) (result map[uint64]map[string]bool, retErr error) {
defer func() {
// We need resultPrime is because proto3 can't do maps of maps.
resultPrime := make(map[uint64]*log.ReplicaAddresses)
for shard, addresses := range result {
resultPrime[shard] = &log.ReplicaAddresses{addresses}
}
protolog.Debug(&log.GetShardToReplicaAddresses{version, resultPrime, errorToString(retErr)})
}()
addresses, err := a.getAddresses(version)
if err != nil {
return nil, err
}
_result := make(map[uint64]map[string]bool)
for shard, shardAddresses := range addresses.Addresses {
_result[shard] = shardAddresses.Replicas
}
return _result, nil
}
func execSubvolumeList(path string, fromPath string, ascending bool, out io.Writer) (retErr error) {
defer func() {
protolog.Debug(&SubvolumeList{path, fromPath, ascending, errorToString(retErr)})
}()
var sort string
if ascending {
sort = "+ogen"
} else {
sort = "-ogen"
}
if fromPath == "" {
return pkgexec.RunStdout(out, "btrfs", "subvolume", "list", "-a", "--sort", sort, path)
}
transid, err := execTransID(fromPath)
if err != nil {
return err
}
return pkgexec.RunStdout(out, "btrfs", "subvolume", "list", "-aC", "+"+transid, "--sort", sort, path)
}
func (a *discoveryAddresser) Version() (result int64, retErr error) {
defer func() {
protolog.Debug(&log.Version{result, errorToString(retErr)})
}()
minVersion := int64(math.MaxInt64)
encodedServerStates, err := a.discoveryClient.GetAll(a.serverStateDir())
if err != nil {
return 0, err
}
for _, encodedServerState := range encodedServerStates {
serverState, err := decodeServerState(encodedServerState)
if err != nil {
return 0, err
}
if serverState.Version < minVersion {
minVersion = serverState.Version
}
}
return minVersion, nil
}
func (f *file) Read(ctx context.Context, request *fuse.ReadRequest, response *fuse.ReadResponse) (retErr error) {
defer func() {
protolog.Debug(&FileRead{&f.Node, errorToString(retErr)})
}()
var buffer bytes.Buffer
if err := pfsutil.GetFile(
f.fs.apiClient,
f.File.Commit.Repo.Name,
f.File.Commit.Id,
f.File.Path,
request.Offset,
int64(request.Size),
f.fs.Shard,
&buffer,
); err != nil {
return err
}
response.Data = buffer.Bytes()
return nil
}
func (d *directory) ReadDirAll(ctx context.Context) (result []fuse.Dirent, retErr error) {
defer func() {
var dirents []*Dirent
for _, dirent := range result {
dirents = append(dirents, &Dirent{dirent.Inode, dirent.Name})
}
protolog.Debug(&DirectoryReadDirAll{&d.Node, dirents, errorToString(retErr)})
}()
if d.File.Commit.Repo.Name == "" {
return d.readRepos(ctx)
}
if d.File.Commit.Id == "" {
commitMount := d.fs.getCommitMount(d.File.Commit.Repo.Name)
if commitMount != nil && commitMount.Commit.Id != "" {
d.File.Commit.Id = commitMount.Commit.Id
return d.readFiles(ctx)
}
return d.readCommits(ctx)
}
return d.readFiles(ctx)
}
func (f *file) Attr(ctx context.Context, a *fuse.Attr) (retErr error) {
defer func() {
protolog.Debug(&FileAttr{&f.Node, &Attr{uint32(a.Mode)}, errorToString(retErr)})
}()
fileInfo, err := pfsutil.InspectFile(
f.fs.apiClient,
f.File.Commit.Repo.Name,
f.File.Commit.Id,
f.File.Path,
f.fs.Shard,
)
if err != nil && !f.local {
return err
}
if fileInfo != nil {
a.Size = fileInfo.SizeBytes
}
a.Mode = 0666
a.Inode = f.fs.inode(f.File)
return nil
}
// Debug logs an RPC call at the debug level.
func Debug(serviceName string, methodName string, request proto.Message, response proto.Message, err error, duration time.Duration) {
protolog.Debug(event(serviceName, methodName, request, response, err, duration))
}
func execRecv(path string, diff io.Reader) (retErr error) {
defer func() {
protolog.Debug(&Recv{path, errorToString(retErr)})
}()
return pkgexec.RunStdin(diff, "btrfs", "receive", path)
}
func (c *commitScanner) Commit() string {
commit, _ := c.parseCommit()
protolog.Debug(&SubvolumeListLine{c.textScanner.Text()})
return commit
}
func (c *changeScanner) Change() *pfs.Change {
change, _ := c.parseChange()
protolog.Debug(&SubvolumeFindNewLine{c.textScanner.Text()})
return change
}