func (d *driver) inspectFile(file *pfs.File, shard uint64) (*pfs.FileInfo, []*drive.BlockRef, error) { fileInfo := &pfs.FileInfo{File: file} var blockRefs []*drive.BlockRef children := make(map[string]bool) commit := file.Commit for commit != nil { diffInfo, _, ok := d.getDiffInfo(&drive.Diff{ Commit: commit, Shard: shard, }) if !ok { return nil, nil, fmt.Errorf("diff %s/%s not found", commit.Repo.Name, commit.Id) } log.Printf("Appends: %+v", diffInfo.Appends) if _append, ok := diffInfo.Appends[path.Clean(file.Path)]; ok { log.Printf("using: %+v", _append) if len(_append.BlockRefs) > 0 { if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR { return nil, nil, fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path) } fileInfo.FileType = pfs.FileType_FILE_TYPE_REGULAR blockRefs = append(_append.BlockRefs, blockRefs...) for _, blockRef := range _append.BlockRefs { fileInfo.SizeBytes += (blockRef.Range.Upper - blockRef.Range.Lower) } } else if len(_append.Children) > 0 { if fileInfo.FileType == pfs.FileType_FILE_TYPE_REGULAR { return nil, nil, fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path) } fileInfo.FileType = pfs.FileType_FILE_TYPE_DIR for child := range _append.Children { if !children[child] { fileInfo.Children = append( fileInfo.Children, pfsutil.NewFile(commit.Repo.Name, commit.Id, child), ) } children[child] = true } } if fileInfo.CommitModified == nil { fileInfo.CommitModified = commit fileInfo.Modified = diffInfo.Finished } commit = _append.LastRef continue } commit = diffInfo.ParentCommit } if fileInfo.FileType == pfs.FileType_FILE_TYPE_NONE { return nil, nil, pfs.ErrFileNotFound } log.Printf("returning: %+v", fileInfo) return fileInfo, blockRefs, nil }
func (d *driver) PutFile(file *pfs.File, shard uint64, offset int64, reader io.Reader) (retErr error) { d.lock.RLock() diffInfo, ok := d.started.get(&pfs.Diff{ Commit: file.Commit, Shard: shard, }) d.lock.RUnlock() if !ok { return fmt.Errorf("commit %s/%s not found", file.Commit.Repo.Name, file.Commit.Id) } blockClient, err := d.getBlockClient() if err != nil { return err } blockRefs, err := pfsutil.PutBlock(blockClient, reader) if err != nil { return err } d.lock.Lock() defer d.lock.Unlock() diffInfo, ok = d.started.get(&pfs.Diff{ Commit: file.Commit, Shard: shard, }) if !ok { // This is a weird case since the commit existed above, it means someone // deleted the commit while the above code was running return fmt.Errorf("commit %s/%s not found", file.Commit.Repo.Name, file.Commit.Id) } addDirs(diffInfo, file) _append, ok := diffInfo.Appends[path.Clean(file.Path)] if !ok { _append = &pfs.Append{} if diffInfo.ParentCommit != nil { _append.LastRef = d.lastRef( pfsutil.NewFile( diffInfo.ParentCommit.Repo.Name, diffInfo.ParentCommit.Id, file.Path, ), shard, ) } diffInfo.Appends[path.Clean(file.Path)] = _append } _append.BlockRefs = append(_append.BlockRefs, blockRefs.BlockRef...) for _, blockRef := range blockRefs.BlockRef { diffInfo.SizeBytes += blockRef.Range.Upper - blockRef.Range.Lower } return nil }
func (d *driver) inspectFile(file *pfs.File, filterShard *pfs.Shard, shard uint64) (*pfs.FileInfo, []*drive.BlockRef, error) { fileInfo := &pfs.FileInfo{File: file} var blockRefs []*drive.BlockRef children := make(map[string]bool) commit := file.Commit for commit != nil { diffInfo, _, ok := d.getDiffInfo(&drive.Diff{ Commit: commit, Shard: shard, }) if !ok { return nil, nil, fmt.Errorf("diff %s/%s not found", commit.Repo.Name, commit.Id) } if _append, ok := diffInfo.Appends[path.Clean(file.Path)]; ok { if len(_append.BlockRefs) > 0 { if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR { return nil, nil, fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path) } if fileInfo.FileType == pfs.FileType_FILE_TYPE_NONE { // the first time we find out it's a regular file we check // the file shard, dirs get returned regardless of sharding, // since they might have children from any shard if !route.FileInShard(filterShard, file) { return nil, nil, pfs.ErrFileNotFound } } fileInfo.FileType = pfs.FileType_FILE_TYPE_REGULAR filtered := filterBlockRefs(filterShard, _append.BlockRefs) blockRefs = append(filtered, blockRefs...) for _, blockRef := range filtered { fileInfo.SizeBytes += (blockRef.Range.Upper - blockRef.Range.Lower) } } else if len(_append.Children) > 0 { if fileInfo.FileType == pfs.FileType_FILE_TYPE_REGULAR { return nil, nil, fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path) } fileInfo.FileType = pfs.FileType_FILE_TYPE_DIR for child := range _append.Children { if !children[child] { fileInfo.Children = append( fileInfo.Children, pfsutil.NewFile(commit.Repo.Name, commit.Id, child), ) } children[child] = true } } if fileInfo.CommitModified == nil { fileInfo.CommitModified = commit fileInfo.Modified = diffInfo.Finished } commit = _append.LastRef continue } commit = diffInfo.ParentCommit } if fileInfo.FileType == pfs.FileType_FILE_TYPE_NONE || (fileInfo.FileType == pfs.FileType_FILE_TYPE_REGULAR && len(blockRefs) == 0) { return nil, nil, pfs.ErrFileNotFound } return fileInfo, blockRefs, nil }
func (d *driver) PutFile(file *pfs.File, shard uint64, offset int64, reader io.Reader) (retErr error) { d.lock.RLock() diffInfo, ok := d.started.get(&drive.Diff{ Commit: file.Commit, Shard: shard, }) d.lock.RUnlock() if !ok { return fmt.Errorf("commit %s/%s not found", file.Commit.Repo.Name, file.Commit.Id) } var blockRefs []*drive.BlockRef scanner := bufio.NewScanner(reader) scanner.Split(blockSplitFunc) var wg sync.WaitGroup var loopErr error var sizeBytes uint64 for scanner.Scan() { data := scanner.Bytes() blockRef := &drive.BlockRef{ Range: &drive.ByteRange{ Lower: 0, Upper: uint64(len(data)), }, } blockRefs = append(blockRefs, blockRef) sizeBytes += blockRef.Range.Upper - blockRef.Range.Lower wg.Add(1) go func() { defer wg.Done() block, err := pfsutil.PutBlock(d.driveClient, bytes.NewReader(data)) if err != nil && loopErr == nil { loopErr = err return } blockRef.Block = block }() } wg.Wait() if err := scanner.Err(); err != nil { return err } if loopErr != nil { return loopErr } d.lock.Lock() defer d.lock.Unlock() diffInfo, ok = d.started.get(&drive.Diff{ Commit: file.Commit, Shard: shard, }) if !ok { // This is a weird case since the commit existed above, it means someone // deleted the commit while the above code was running return fmt.Errorf("commit %s/%s not found", file.Commit.Repo.Name, file.Commit.Id) } addDirs(diffInfo, file) _append, ok := diffInfo.Appends[path.Clean(file.Path)] if !ok { _append = &drive.Append{} if diffInfo.ParentCommit != nil { _append.LastRef = d.lastRef( pfsutil.NewFile( diffInfo.ParentCommit.Repo.Name, diffInfo.ParentCommit.Id, file.Path, ), shard, ) } diffInfo.Appends[path.Clean(file.Path)] = _append } _append.BlockRefs = append(_append.BlockRefs, blockRefs...) diffInfo.SizeBytes += sizeBytes return nil }
func (d *driver) inspectFile(file *pfs.File, shard uint64) (*pfs.FileInfo, []*drive.BlockRef, error) { fileInfo := &pfs.FileInfo{File: file} var blockRefs []*drive.BlockRef children := make(map[string]bool) commit := file.Commit writeable := false for commit != nil { diffInfo, read, ok := d.getDiffInfo(&drive.Diff{ Commit: commit, Shard: shard, }) if !ok { return nil, nil, fmt.Errorf("diff %s/%s not found", commit.Repo.Name, commit.Id) } if !read { // should only be possible the hit this the first time through the // loop writeable = true } if _append, ok := diffInfo.Appends[path.Clean(file.Path)]; ok { if len(_append.BlockRefs) > 0 { if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR { return nil, nil, fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path) } fileInfo.FileType = pfs.FileType_FILE_TYPE_REGULAR blockRefs = append(_append.BlockRefs, blockRefs...) for _, blockRef := range _append.BlockRefs { fileInfo.SizeBytes += (blockRef.Range.Upper - blockRef.Range.Lower) } } else if len(_append.Children) > 0 { if fileInfo.FileType == pfs.FileType_FILE_TYPE_REGULAR { return nil, nil, fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path) } fileInfo.FileType = pfs.FileType_FILE_TYPE_DIR for child := range _append.Children { if !children[child] { fileInfo.Children = append( fileInfo.Children, pfsutil.NewFile(commit.Repo.Name, commit.Id, child), ) } children[child] = true } } if fileInfo.CommitModified == nil { fileInfo.CommitModified = commit fileInfo.Modified = diffInfo.Finished } commit = _append.LastRef continue } commit = diffInfo.ParentCommit } if fileInfo.FileType == pfs.FileType_FILE_TYPE_NONE { if writeable { fileInfo.FileType = pfs.FileType_FILE_TYPE_REGULAR } else { return nil, nil, fmt.Errorf("file %s/%s/%s not found", file.Commit.Repo.Name, file.Commit.Id, file.Path) } } return fileInfo, blockRefs, nil }