// NewRootedCamliFileSystem returns a CamliFileSystem with root as its
// base.
func NewRootedCamliFileSystem(fetcher blob.SeekFetcher, root blob.Ref) (*CamliFileSystem, error) {
fs := newCamliFileSystem(fetcher)
blob, err := fs.fetchSchemaMeta(root)
if err != nil {
return nil, err
}
if blob.Type() != "directory" {
return nil, fmt.Errorf("Blobref must be of a directory, got a %v", blob.Type())
}
n := &node{fs: fs, blobref: root, meta: blob}
n.populateAttr()
fs.root = n
return fs, nil
}
// populateMutationMap populates keys & values that will be committed
// into the returned map.
//
// the blobref can be trusted at this point (it's been fully consumed
// and verified to match), and the sniffer has been populated.
func (ix *Index) populateMutationMap(fetcher *missTrackFetcher, br blob.Ref, sniffer *BlobSniffer) (*mutationMap, error) {
// TODO(mpl): shouldn't we remove these two from the map (so they don't get committed) when
// e.g in populateClaim we detect a bogus claim (which does not yield an error)?
mm := &mutationMap{
kv: map[string]string{
"have:" + br.String(): fmt.Sprintf("%d", sniffer.Size()),
"meta:" + br.String(): fmt.Sprintf("%d|%s", sniffer.Size(), sniffer.MIMEType()),
},
}
if blob, ok := sniffer.SchemaBlob(); ok {
switch blob.Type() {
case "claim":
if err := ix.populateClaim(blob, mm); err != nil {
return nil, err
}
case "file":
if err := ix.populateFile(fetcher, blob, mm); err != nil {
return nil, err
}
case "directory":
if err := ix.populateDir(fetcher, blob, mm); err != nil {
return nil, err
}
}
}
return mm, nil
}
// populateMutation populates keys & values into the provided BatchMutation.
//
// the blobref can be trusted at this point (it's been fully consumed
// and verified to match), and the sniffer has been populated.
func (ix *Index) populateMutation(br blob.Ref, sniffer *BlobSniffer, bm BatchMutation) error {
bm.Set("have:"+br.String(), fmt.Sprintf("%d", sniffer.Size()))
bm.Set("meta:"+br.String(), fmt.Sprintf("%d|%s", sniffer.Size(), sniffer.MIMEType()))
if blob, ok := sniffer.SchemaBlob(); ok {
switch blob.Type() {
case "claim":
if err := ix.populateClaim(blob, bm); err != nil {
return err
}
case "permanode":
//if err := mi.populatePermanode(blobRef, camli, bm); err != nil {
//return err
//}
case "file":
if err := ix.populateFile(blob, bm); err != nil {
return err
}
case "directory":
if err := ix.populateDir(blob, bm); err != nil {
return err
}
}
}
return nil
}
func TestBlobFromReader(t *testing.T) {
br := blob.MustParse("sha1-f1d2d2f924e986ac86fdf7b36c94bcdf32beec15")
blob, err := BlobFromReader(br, strings.NewReader(`{"camliVersion": 1, "camliType": "foo"} `))
if err != nil {
t.Error(err)
} else if blob.Type() != "foo" {
t.Errorf("got type %q; want foo", blob.Type())
}
blob, err = BlobFromReader(br, strings.NewReader(`{"camliVersion": 1, "camliType": "foo"} X `))
if err == nil {
// TODO(bradfitz): fix this somehow. Currently encoding/json's
// decoder over-reads.
// See: https://code.google.com/p/go/issues/detail?id=1955 ,
// which was "fixed", but not really.
t.Logf("TODO(bradfitz): make sure bogus non-whitespace after the JSON object causes an error.")
}
}
// consolated logic for determining a node to mount based on an arbitrary blobref
func (fs *CamliFileSystem) newNodeFromBlobRef(root blob.Ref) (fusefs.Node, error) {
blob, err := fs.fetchSchemaMeta(root)
if err != nil {
return nil, err
}
switch blob.Type() {
case "directory":
n := &node{fs: fs, blobref: root, meta: blob}
n.populateAttr()
return n, nil
case "permanode":
// other mutDirs listed in the default fileystem have names and are displayed
return &mutDir{fs: fs, permanode: root, name: "-"}, nil
}
return nil, fmt.Errorf("Blobref must be of a directory or permanode got a %v", blob.Type())
}
func isSchemaPicker(thenSto, elseSto blobserver.Storage) storageFunc {
return func(br blob.Ref, src io.Reader) (dest blobserver.Storage, newSrc io.Reader, err error) {
var buf bytes.Buffer
blob, err := schema.BlobFromReader(br, io.TeeReader(src, &buf))
newSrc = io.MultiReader(bytes.NewReader(buf.Bytes()), src)
if err != nil || blob.Type() == "" {
return elseSto, newSrc, nil
}
return thenSto, newSrc, nil
}
}
func isSchemaPicker(thenSto, elseSto blobserver.Storage) storageFunc {
return func(src io.Reader) (dest blobserver.Storage, overRead []byte, err error) {
var buf bytes.Buffer
tee := io.TeeReader(src, &buf)
blob, err := schema.BlobFromReader(dummyRef, tee)
if err != nil || blob.Type() == "" {
return elseSto, buf.Bytes(), nil
}
return thenSto, buf.Bytes(), nil
}
}
// populateMutationMap populates keys & values that will be committed
// into the returned map.
//
// the blobref can be trusted at this point (it's been fully consumed
// and verified to match), and the sniffer has been populated.
func (ix *Index) populateMutationMap(fetcher *missTrackFetcher, br blob.Ref, sniffer *BlobSniffer) (*mutationMap, error) {
mm := &mutationMap{
kv: map[string]string{
"meta:" + br.String(): fmt.Sprintf("%d|%s", sniffer.Size(), sniffer.MIMEType()),
},
}
var err error
if blob, ok := sniffer.SchemaBlob(); ok {
switch blob.Type() {
case "claim":
err = ix.populateClaim(fetcher, blob, mm)
case "file":
err = ix.populateFile(fetcher, blob, mm)
case "directory":
err = ix.populateDir(fetcher, blob, mm)
}
}
if err != nil && err != errMissingDep {
return nil, err
}
var haveVal string
if err == errMissingDep {
haveVal = fmt.Sprintf("%d", sniffer.Size())
} else {
haveVal = fmt.Sprintf("%d|indexed", sniffer.Size())
}
mm.kv["have:"+br.String()] = haveVal
ix.mu.Lock()
defer ix.mu.Unlock()
if len(fetcher.missing) == 0 {
// If err == nil, we're good. Else (err == errMissingDep), we
// know the error did not come from a fetching miss (because
// len(fetcher.missing) == 0) , but from an index miss. Therefore
// we know the miss has already been noted and will be dealt with
// later, so we can also pretend everything's fine.
return mm, nil
}
return mm, err
}
// Errors returned are:
// os.ErrNotExist -- blob not found
// os.ErrInvalid -- not JSON or a camli schema blob
func (fs *CamliFileSystem) fetchSchemaMeta(br blob.Ref) (*schema.Blob, error) {
blobStr := br.String()
if blob, ok := fs.blobToSchema.Get(blobStr); ok {
return blob.(*schema.Blob), nil
}
rc, _, err := fs.fetcher.Fetch(br)
if err != nil {
return nil, err
}
defer rc.Close()
blob, err := schema.BlobFromReader(br, rc)
if err != nil {
log.Printf("Error parsing %s as schema blob: %v", br, err)
return nil, os.ErrInvalid
}
if blob.Type() == "" {
log.Printf("blob %s is JSON but lacks camliType", br)
return nil, os.ErrInvalid
}
fs.blobToSchema.Add(blobStr, blob)
return blob, nil
}
// Unauthenticated user. Be paranoid.
func handleGetViaSharing(rw http.ResponseWriter, req *http.Request,
blobRef blob.Ref, fetcher blob.Fetcher) error {
if !httputil.IsGet(req) {
return &shareError{code: invalidMethod, response: badRequest, message: "Invalid method"}
}
rw.Header().Set("Access-Control-Allow-Origin", "*")
viaPathOkay := false
startTime := time.Now()
defer func() {
if !viaPathOkay {
// Insert a delay, to hide timing attacks probing
// for the existence of blobs.
sleep := fetchFailureDelay - (time.Now().Sub(startTime))
timeSleep(sleep)
}
}()
viaBlobs := make([]blob.Ref, 0)
if via := req.FormValue("via"); via != "" {
for _, vs := range strings.Split(via, ",") {
if br, ok := blob.Parse(vs); ok {
viaBlobs = append(viaBlobs, br)
} else {
return &shareError{code: invalidVia, response: badRequest, message: "Malformed blobref in via param"}
}
}
}
fetchChain := make([]blob.Ref, 0)
fetchChain = append(fetchChain, viaBlobs...)
fetchChain = append(fetchChain, blobRef)
isTransitive := false
for i, br := range fetchChain {
switch i {
case 0:
file, size, err := fetcher.Fetch(br)
if err != nil {
return unauthorized(shareFetchFailed, "Fetch chain 0 of %s failed: %v", br, err)
}
defer file.Close()
if size > schema.MaxSchemaBlobSize {
return unauthorized(shareBlobTooLarge, "Fetch chain 0 of %s too large", br)
}
blob, err := schema.BlobFromReader(br, file)
if err != nil {
return unauthorized(shareReadFailed, "Can't create a blob from %v: %v", br, err)
}
share, ok := blob.AsShare()
if !ok {
return unauthorized(shareBlobInvalid, "Fetch chain 0 of %s wasn't a valid Share (is %q)", br, blob.Type())
}
if share.IsExpired() {
return unauthorized(shareExpired, "Share is expired")
}
if len(fetchChain) > 1 && fetchChain[1].String() != share.Target().String() {
return unauthorized(shareTargetInvalid,
"Fetch chain 0->1 (%s -> %q) unauthorized, expected hop to %q",
br, fetchChain[1], share.Target())
}
isTransitive = share.IsTransitive()
if len(fetchChain) > 2 && !isTransitive {
return unauthorized(shareNotTransitive, "Share is not transitive")
}
case len(fetchChain) - 1:
// Last one is fine (as long as its path up to here has been proven, and it's
// not the first thing in the chain)
continue
default:
rc, _, err := fetcher.Fetch(br)
if err != nil {
return unauthorized(viaChainFetchFailed, "Fetch chain %d of %s failed: %v", i, br, err)
}
defer rc.Close()
lr := io.LimitReader(rc, schema.MaxSchemaBlobSize)
slurpBytes, err := ioutil.ReadAll(lr)
if err != nil {
return unauthorized(viaChainReadFailed,
"Fetch chain %d of %s failed in slurp: %v", i, br, err)
}
saught := fetchChain[i+1]
if !bytesHaveSchemaLink(br, slurpBytes, saught) {
return unauthorized(viaChainInvalidLink,
"Fetch chain %d of %s failed; no reference to %s", i, br, saught)
}
}
}
if assemble, _ := strconv.ParseBool(req.FormValue("assemble")); assemble {
if !isTransitive {
return unauthorized(assembleNonTransitive, "Cannot assemble non-transitive share")
}
dh := &DownloadHandler{
Fetcher: fetcher,
// TODO(aa): It would be nice to specify a local cache here, as the UI handler does.
}
dh.ServeHTTP(rw, req, blobRef)
} else {
gethandler.ServeBlobRef(rw, req, blobRef, fetcher)
}
viaPathOkay = true
return nil
}
