func (n *node) load() {
defer n.g.wg.Done()
rc, err := fetch(n.g.src, n.br)
check(err)
defer rc.Close()
sniff := index.NewBlobSniffer(n.br)
n.size, err = io.Copy(sniff, rc)
check(err)
sniff.Parse()
blob, ok := sniff.SchemaBlob()
if !ok {
return
}
n.blob = blob
for _, part := range blob.ByteParts() {
n.addEdge(part.BlobRef)
n.addEdge(part.BytesRef)
}
}
// smartFetch the things that blobs point to, not just blobs.
func smartFetch(src blobref.StreamingFetcher, targ string, br *blobref.BlobRef) error {
rc, err := fetch(src, br)
if err != nil {
return err
}
defer rc.Close()
sniffer := index.NewBlobSniffer(br)
_, err = io.CopyN(sniffer, rc, sniffSize)
if err != nil && err != io.EOF {
return err
}
sniffer.Parse()
blob, ok := sniffer.SchemaBlob()
if !ok {
if *flagVerbose {
log.Printf("Fetching opaque data %v into %q", br, targ)
}
// opaque data - put it in a file
f, err := os.Create(targ)
if err != nil {
return fmt.Errorf("opaque: %v", err)
}
defer f.Close()
body, _ := sniffer.Body()
r := io.MultiReader(bytes.NewReader(body), rc)
_, err = io.Copy(f, r)
return err
}
switch blob.Type() {
case "directory":
dir := filepath.Join(targ, blob.FileName())
if *flagVerbose {
log.Printf("Fetching directory %v into %s", br, dir)
}
if err := os.MkdirAll(dir, blob.FileMode()); err != nil {
return err
}
if err := setFileMeta(dir, blob); err != nil {
log.Print(err)
}
entries := blob.DirectoryEntries()
if entries == nil {
return fmt.Errorf("bad entries blobref in dir %v", blob.BlobRef())
}
return smartFetch(src, dir, entries)
case "static-set":
if *flagVerbose {
log.Printf("Fetching directory entries %v into %s", br, targ)
}
// directory entries
const numWorkers = 10
type work struct {
br *blobref.BlobRef
errc chan<- error
}
members := blob.StaticSetMembers()
workc := make(chan work, len(members))
defer close(workc)
for i := 0; i < numWorkers; i++ {
go func() {
for wi := range workc {
wi.errc <- smartFetch(src, targ, wi.br)
}
}()
}
var errcs []<-chan error
for _, mref := range members {
errc := make(chan error, 1)
errcs = append(errcs, errc)
workc <- work{mref, errc}
}
for _, errc := range errcs {
if err := <-errc; err != nil {
return err
}
}
return nil
case "file":
seekFetcher := blobref.SeekerFromStreamingFetcher(src)
fr, err := schema.NewFileReader(seekFetcher, br)
if err != nil {
return fmt.Errorf("NewFileReader: %v", err)
}
fr.LoadAllChunks()
defer fr.Close()
name := filepath.Join(targ, blob.FileName())
if fi, err := os.Stat(name); err == nil && fi.Size() == fi.Size() {
if *flagVerbose {
log.Printf("Skipping %s; already exists.", name)
return nil
}
}
if *flagVerbose {
log.Printf("Writing %s to %s ...", br, name)
}
f, err := os.Create(name)
if err != nil {
return fmt.Errorf("file type: %v", err)
}
defer f.Close()
if _, err := io.Copy(f, fr); err != nil {
return fmt.Errorf("Copying %s to %s: %v", br, name, err)
}
if err := setFileMeta(name, blob); err != nil {
log.Print(err)
}
return nil
default:
return errors.New("unknown blob type: " + blob.Type())
}
panic("unreachable")
}
func parseSchema(ref blob.Ref, body io.Reader) (*schema.Blob, bool) {
sn := index.NewBlobSniffer(ref)
io.Copy(sn, body)
sn.Parse()
return sn.SchemaBlob()
}
// smartFetch the things that blobs point to, not just blobs.
func smartFetch(src blob.Fetcher, targ string, br blob.Ref) error {
rc, err := fetch(src, br)
if err != nil {
return err
}
rcc := types.NewOnceCloser(rc)
defer rcc.Close()
sniffer := index.NewBlobSniffer(br)
_, err = io.CopyN(sniffer, rc, sniffSize)
if err != nil && err != io.EOF {
return err
}
sniffer.Parse()
b, ok := sniffer.SchemaBlob()
if !ok {
if *flagVerbose {
log.Printf("Fetching opaque data %v into %q", br, targ)
}
// opaque data - put it in a file
f, err := os.Create(targ)
if err != nil {
return fmt.Errorf("opaque: %v", err)
}
defer f.Close()
body, _ := sniffer.Body()
r := io.MultiReader(bytes.NewReader(body), rc)
_, err = io.Copy(f, r)
return err
}
rcc.Close()
switch b.Type() {
case "directory":
dir := filepath.Join(targ, b.FileName())
if *flagVerbose {
log.Printf("Fetching directory %v into %s", br, dir)
}
if err := os.MkdirAll(dir, b.FileMode()); err != nil {
return err
}
if err := setFileMeta(dir, b); err != nil {
log.Print(err)
}
entries, ok := b.DirectoryEntries()
if !ok {
return fmt.Errorf("bad entries blobref in dir %v", b.BlobRef())
}
return smartFetch(src, dir, entries)
case "static-set":
if *flagVerbose {
log.Printf("Fetching directory entries %v into %s", br, targ)
}
// directory entries
const numWorkers = 10
type work struct {
br blob.Ref
errc chan<- error
}
members := b.StaticSetMembers()
workc := make(chan work, len(members))
defer close(workc)
for i := 0; i < numWorkers; i++ {
go func() {
for wi := range workc {
wi.errc <- smartFetch(src, targ, wi.br)
}
}()
}
var errcs []<-chan error
for _, mref := range members {
errc := make(chan error, 1)
errcs = append(errcs, errc)
workc <- work{mref, errc}
}
for _, errc := range errcs {
if err := <-errc; err != nil {
return err
}
}
return nil
case "file":
fr, err := schema.NewFileReader(src, br)
if err != nil {
return fmt.Errorf("NewFileReader: %v", err)
}
fr.LoadAllChunks()
defer fr.Close()
name := filepath.Join(targ, b.FileName())
if fi, err := os.Stat(name); err == nil && fi.Size() == fr.Size() {
if *flagVerbose {
log.Printf("Skipping %s; already exists.", name)
}
return nil
}
if *flagVerbose {
log.Printf("Writing %s to %s ...", br, name)
}
f, err := os.Create(name)
if err != nil {
return fmt.Errorf("file type: %v", err)
}
defer f.Close()
if _, err := io.Copy(f, fr); err != nil {
return fmt.Errorf("Copying %s to %s: %v", br, name, err)
}
if err := setFileMeta(name, b); err != nil {
log.Print(err)
}
return nil
case "symlink":
if *flagSkipIrregular {
return nil
}
sf, ok := b.AsStaticFile()
if !ok {
return errors.New("blob is not a static file")
}
sl, ok := sf.AsStaticSymlink()
if !ok {
return errors.New("blob is not a symlink")
}
name := filepath.Join(targ, sl.FileName())
if _, err := os.Lstat(name); err == nil {
if *flagVerbose {
log.Printf("Skipping creating symbolic link %s: A file with that name exists", name)
}
return nil
}
target := sl.SymlinkTargetString()
if target == "" {
return errors.New("symlink without target")
}
// On Windows, os.Symlink isn't yet implemented as of Go 1.3.
// See https://code.google.com/p/go/issues/detail?id=5750
err := os.Symlink(target, name)
// We won't call setFileMeta for a symlink because:
// the permissions of a symlink do not matter and Go's
// os.Chtimes always dereferences (does not act on the
// symlink but its target).
return err
case "fifo":
if *flagSkipIrregular {
return nil
}
name := filepath.Join(targ, b.FileName())
sf, ok := b.AsStaticFile()
if !ok {
return errors.New("blob is not a static file")
}
_, ok = sf.AsStaticFIFO()
if !ok {
return errors.New("blob is not a static FIFO")
}
if _, err := os.Lstat(name); err == nil {
log.Printf("Skipping FIFO %s: A file with that name already exists", name)
return nil
}
err = osutil.Mkfifo(name, 0600)
if err == osutil.ErrNotSupported {
log.Printf("Skipping FIFO %s: Unsupported filetype", name)
return nil
}
if err != nil {
return fmt.Errorf("%s: osutil.Mkfifo(): %v", name, err)
}
if err := setFileMeta(name, b); err != nil {
log.Print(err)
}
return nil
case "socket":
if *flagSkipIrregular {
return nil
}
name := filepath.Join(targ, b.FileName())
sf, ok := b.AsStaticFile()
if !ok {
return errors.New("blob is not a static file")
}
_, ok = sf.AsStaticSocket()
if !ok {
return errors.New("blob is not a static socket")
}
if _, err := os.Lstat(name); err == nil {
log.Printf("Skipping socket %s: A file with that name already exists", name)
return nil
}
err = osutil.Mksocket(name)
if err == osutil.ErrNotSupported {
log.Printf("Skipping socket %s: Unsupported filetype", name)
return nil
}
if err != nil {
return fmt.Errorf("%s: %v", name, err)
}
if err := setFileMeta(name, b); err != nil {
log.Print(err)
}
return nil
default:
return errors.New("unknown blob type: " + b.Type())
}
panic("unreachable")
}
// smartFetch the things that blobs point to, not just blobs.
func smartFetch(src blob.Fetcher, targ string, br blob.Ref) error {
rc, err := fetch(src, br)
if err != nil {
return err
}
defer rc.Close()
sniffer := index.NewBlobSniffer(br)
_, err = io.CopyN(sniffer, rc, sniffSize)
if err != nil && err != io.EOF {
return err
}
sniffer.Parse()
b, ok := sniffer.SchemaBlob()
if !ok {
if *flagVerbose {
log.Printf("Fetching opaque data %v into %q", br, targ)
}
// opaque data - put it in a file
f, err := os.Create(targ)
if err != nil {
return fmt.Errorf("opaque: %v", err)
}
defer f.Close()
body, _ := sniffer.Body()
r := io.MultiReader(bytes.NewReader(body), rc)
_, err = io.Copy(f, r)
return err
}
switch b.Type() {
case "directory":
dir := filepath.Join(targ, b.FileName())
if *flagVerbose {
log.Printf("Fetching directory %v into %s", br, dir)
}
if err := os.MkdirAll(dir, b.FileMode()); err != nil {
return err
}
if err := setFileMeta(dir, b); err != nil {
log.Print(err)
}
entries, ok := b.DirectoryEntries()
if !ok {
return fmt.Errorf("bad entries blobref in dir %v", b.BlobRef())
}
return smartFetch(src, dir, entries)
case "static-set":
if *flagVerbose {
log.Printf("Fetching directory entries %v into %s", br, targ)
}
// directory entries
const numWorkers = 10
type work struct {
br blob.Ref
errc chan<- error
}
members := b.StaticSetMembers()
workc := make(chan work, len(members))
defer close(workc)
for i := 0; i < numWorkers; i++ {
go func() {
for wi := range workc {
wi.errc <- smartFetch(src, targ, wi.br)
}
}()
}
var errcs []<-chan error
for _, mref := range members {
errc := make(chan error, 1)
errcs = append(errcs, errc)
workc <- work{mref, errc}
}
for _, errc := range errcs {
if err := <-errc; err != nil {
return err
}
}
return nil
case "file":
fr, err := schema.NewFileReader(src, br)
if err != nil {
return fmt.Errorf("NewFileReader: %v", err)
}
fr.LoadAllChunks()
defer fr.Close()
name := filepath.Join(targ, b.FileName())
if fi, err := os.Stat(name); err == nil && fi.Size() == fi.Size() {
if *flagVerbose {
log.Printf("Skipping %s; already exists.", name)
return nil
}
}
if *flagVerbose {
log.Printf("Writing %s to %s ...", br, name)
}
f, err := os.Create(name)
if err != nil {
return fmt.Errorf("file type: %v", err)
}
defer f.Close()
if _, err := io.Copy(f, fr); err != nil {
return fmt.Errorf("Copying %s to %s: %v", br, name, err)
}
if err := setFileMeta(name, b); err != nil {
log.Print(err)
}
return nil
case "symlink":
sf, ok := b.AsStaticFile()
if !ok {
return errors.New("blob is not a static file")
}
sl, ok := sf.AsStaticSymlink()
if !ok {
return errors.New("blob is not a symlink")
}
name := filepath.Join(targ, sl.FileName())
if _, err := os.Lstat(name); err == nil {
if *flagVerbose {
log.Printf("Skipping creating symbolic link %s: A file with that name exists", name)
}
return nil
}
target := sl.SymlinkTargetString()
if target == "" {
return errors.New("symlink without target")
}
// TODO (marete): The Go docs promise that everything
// in pkg os should work the same everywhere. Not true
// for os.Symlin() at the moment. See what to do for
// windows here.
err := os.Symlink(target, name)
// We won't call setFileMeta for a symlink because:
// the permissions of a symlink do not matter and Go's
// os.Chtimes always dereferences (does not act on the
// symlink but its target).
return err
default:
return errors.New("unknown blob type: " + b.Type())
}
panic("unreachable")
}