func (d *driver) inspectFile(file *pfs.File, shard uint64) (*pfs.FileInfo, []*drive.BlockRef, error) {
fileInfo := &pfs.FileInfo{File: file}
var blockRefs []*drive.BlockRef
children := make(map[string]bool)
commit := file.Commit
for commit != nil {
diffInfo, _, ok := d.getDiffInfo(&drive.Diff{
Commit: commit,
Shard: shard,
})
if !ok {
return nil, nil, fmt.Errorf("diff %s/%s not found", commit.Repo.Name, commit.Id)
}
log.Printf("Appends: %+v", diffInfo.Appends)
if _append, ok := diffInfo.Appends[path.Clean(file.Path)]; ok {
log.Printf("using: %+v", _append)
if len(_append.BlockRefs) > 0 {
if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR {
return nil, nil,
fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path)
}
fileInfo.FileType = pfs.FileType_FILE_TYPE_REGULAR
blockRefs = append(_append.BlockRefs, blockRefs...)
for _, blockRef := range _append.BlockRefs {
fileInfo.SizeBytes += (blockRef.Range.Upper - blockRef.Range.Lower)
}
} else if len(_append.Children) > 0 {
if fileInfo.FileType == pfs.FileType_FILE_TYPE_REGULAR {
return nil, nil,
fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path)
}
fileInfo.FileType = pfs.FileType_FILE_TYPE_DIR
for child := range _append.Children {
if !children[child] {
fileInfo.Children = append(
fileInfo.Children,
pfsutil.NewFile(commit.Repo.Name, commit.Id, child),
)
}
children[child] = true
}
}
if fileInfo.CommitModified == nil {
fileInfo.CommitModified = commit
fileInfo.Modified = diffInfo.Finished
}
commit = _append.LastRef
continue
}
commit = diffInfo.ParentCommit
}
if fileInfo.FileType == pfs.FileType_FILE_TYPE_NONE {
return nil, nil, pfs.ErrFileNotFound
}
log.Printf("returning: %+v", fileInfo)
return fileInfo, blockRefs, nil
}
func (d *driver) PutFile(file *pfs.File, shard uint64, offset int64, reader io.Reader) (retErr error) {
d.lock.RLock()
diffInfo, ok := d.started.get(&pfs.Diff{
Commit: file.Commit,
Shard: shard,
})
d.lock.RUnlock()
if !ok {
return fmt.Errorf("commit %s/%s not found", file.Commit.Repo.Name, file.Commit.Id)
}
blockClient, err := d.getBlockClient()
if err != nil {
return err
}
blockRefs, err := pfsutil.PutBlock(blockClient, reader)
if err != nil {
return err
}
d.lock.Lock()
defer d.lock.Unlock()
diffInfo, ok = d.started.get(&pfs.Diff{
Commit: file.Commit,
Shard: shard,
})
if !ok {
// This is a weird case since the commit existed above, it means someone
// deleted the commit while the above code was running
return fmt.Errorf("commit %s/%s not found", file.Commit.Repo.Name, file.Commit.Id)
}
addDirs(diffInfo, file)
_append, ok := diffInfo.Appends[path.Clean(file.Path)]
if !ok {
_append = &pfs.Append{}
if diffInfo.ParentCommit != nil {
_append.LastRef = d.lastRef(
pfsutil.NewFile(
diffInfo.ParentCommit.Repo.Name,
diffInfo.ParentCommit.Id,
file.Path,
),
shard,
)
}
diffInfo.Appends[path.Clean(file.Path)] = _append
}
_append.BlockRefs = append(_append.BlockRefs, blockRefs.BlockRef...)
for _, blockRef := range blockRefs.BlockRef {
diffInfo.SizeBytes += blockRef.Range.Upper - blockRef.Range.Lower
}
return nil
}
func (d *driver) inspectFile(file *pfs.File, filterShard *pfs.Shard, shard uint64) (*pfs.FileInfo, []*drive.BlockRef, error) {
fileInfo := &pfs.FileInfo{File: file}
var blockRefs []*drive.BlockRef
children := make(map[string]bool)
commit := file.Commit
for commit != nil {
diffInfo, _, ok := d.getDiffInfo(&drive.Diff{
Commit: commit,
Shard: shard,
})
if !ok {
return nil, nil, fmt.Errorf("diff %s/%s not found", commit.Repo.Name, commit.Id)
}
if _append, ok := diffInfo.Appends[path.Clean(file.Path)]; ok {
if len(_append.BlockRefs) > 0 {
if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR {
return nil, nil,
fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path)
}
if fileInfo.FileType == pfs.FileType_FILE_TYPE_NONE {
// the first time we find out it's a regular file we check
// the file shard, dirs get returned regardless of sharding,
// since they might have children from any shard
if !route.FileInShard(filterShard, file) {
return nil, nil, pfs.ErrFileNotFound
}
}
fileInfo.FileType = pfs.FileType_FILE_TYPE_REGULAR
filtered := filterBlockRefs(filterShard, _append.BlockRefs)
blockRefs = append(filtered, blockRefs...)
for _, blockRef := range filtered {
fileInfo.SizeBytes += (blockRef.Range.Upper - blockRef.Range.Lower)
}
} else if len(_append.Children) > 0 {
if fileInfo.FileType == pfs.FileType_FILE_TYPE_REGULAR {
return nil, nil,
fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path)
}
fileInfo.FileType = pfs.FileType_FILE_TYPE_DIR
for child := range _append.Children {
if !children[child] {
fileInfo.Children = append(
fileInfo.Children,
pfsutil.NewFile(commit.Repo.Name, commit.Id, child),
)
}
children[child] = true
}
}
if fileInfo.CommitModified == nil {
fileInfo.CommitModified = commit
fileInfo.Modified = diffInfo.Finished
}
commit = _append.LastRef
continue
}
commit = diffInfo.ParentCommit
}
if fileInfo.FileType == pfs.FileType_FILE_TYPE_NONE ||
(fileInfo.FileType == pfs.FileType_FILE_TYPE_REGULAR && len(blockRefs) == 0) {
return nil, nil, pfs.ErrFileNotFound
}
return fileInfo, blockRefs, nil
}
func (d *driver) PutFile(file *pfs.File, shard uint64, offset int64, reader io.Reader) (retErr error) {
d.lock.RLock()
diffInfo, ok := d.started.get(&drive.Diff{
Commit: file.Commit,
Shard: shard,
})
d.lock.RUnlock()
if !ok {
return fmt.Errorf("commit %s/%s not found", file.Commit.Repo.Name, file.Commit.Id)
}
var blockRefs []*drive.BlockRef
scanner := bufio.NewScanner(reader)
scanner.Split(blockSplitFunc)
var wg sync.WaitGroup
var loopErr error
var sizeBytes uint64
for scanner.Scan() {
data := scanner.Bytes()
blockRef := &drive.BlockRef{
Range: &drive.ByteRange{
Lower: 0,
Upper: uint64(len(data)),
},
}
blockRefs = append(blockRefs, blockRef)
sizeBytes += blockRef.Range.Upper - blockRef.Range.Lower
wg.Add(1)
go func() {
defer wg.Done()
block, err := pfsutil.PutBlock(d.driveClient, bytes.NewReader(data))
if err != nil && loopErr == nil {
loopErr = err
return
}
blockRef.Block = block
}()
}
wg.Wait()
if err := scanner.Err(); err != nil {
return err
}
if loopErr != nil {
return loopErr
}
d.lock.Lock()
defer d.lock.Unlock()
diffInfo, ok = d.started.get(&drive.Diff{
Commit: file.Commit,
Shard: shard,
})
if !ok {
// This is a weird case since the commit existed above, it means someone
// deleted the commit while the above code was running
return fmt.Errorf("commit %s/%s not found", file.Commit.Repo.Name, file.Commit.Id)
}
addDirs(diffInfo, file)
_append, ok := diffInfo.Appends[path.Clean(file.Path)]
if !ok {
_append = &drive.Append{}
if diffInfo.ParentCommit != nil {
_append.LastRef = d.lastRef(
pfsutil.NewFile(
diffInfo.ParentCommit.Repo.Name,
diffInfo.ParentCommit.Id,
file.Path,
),
shard,
)
}
diffInfo.Appends[path.Clean(file.Path)] = _append
}
_append.BlockRefs = append(_append.BlockRefs, blockRefs...)
diffInfo.SizeBytes += sizeBytes
return nil
}
func (d *driver) inspectFile(file *pfs.File, shard uint64) (*pfs.FileInfo, []*drive.BlockRef, error) {
fileInfo := &pfs.FileInfo{File: file}
var blockRefs []*drive.BlockRef
children := make(map[string]bool)
commit := file.Commit
writeable := false
for commit != nil {
diffInfo, read, ok := d.getDiffInfo(&drive.Diff{
Commit: commit,
Shard: shard,
})
if !ok {
return nil, nil, fmt.Errorf("diff %s/%s not found", commit.Repo.Name, commit.Id)
}
if !read {
// should only be possible the hit this the first time through the
// loop
writeable = true
}
if _append, ok := diffInfo.Appends[path.Clean(file.Path)]; ok {
if len(_append.BlockRefs) > 0 {
if fileInfo.FileType == pfs.FileType_FILE_TYPE_DIR {
return nil, nil,
fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path)
}
fileInfo.FileType = pfs.FileType_FILE_TYPE_REGULAR
blockRefs = append(_append.BlockRefs, blockRefs...)
for _, blockRef := range _append.BlockRefs {
fileInfo.SizeBytes += (blockRef.Range.Upper - blockRef.Range.Lower)
}
} else if len(_append.Children) > 0 {
if fileInfo.FileType == pfs.FileType_FILE_TYPE_REGULAR {
return nil, nil,
fmt.Errorf("mixed dir and regular file %s/%s/%s, (this is likely a bug)", file.Commit.Repo.Name, file.Commit.Id, file.Path)
}
fileInfo.FileType = pfs.FileType_FILE_TYPE_DIR
for child := range _append.Children {
if !children[child] {
fileInfo.Children = append(
fileInfo.Children,
pfsutil.NewFile(commit.Repo.Name, commit.Id, child),
)
}
children[child] = true
}
}
if fileInfo.CommitModified == nil {
fileInfo.CommitModified = commit
fileInfo.Modified = diffInfo.Finished
}
commit = _append.LastRef
continue
}
commit = diffInfo.ParentCommit
}
if fileInfo.FileType == pfs.FileType_FILE_TYPE_NONE {
if writeable {
fileInfo.FileType = pfs.FileType_FILE_TYPE_REGULAR
} else {
return nil, nil, fmt.Errorf("file %s/%s/%s not found", file.Commit.Repo.Name, file.Commit.Id, file.Path)
}
}
return fileInfo, blockRefs, nil
}