func (ix *Index) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (retsb blobref.SizedBlobRef, err error) {
sniffer := new(BlobSniffer)
hash := blobRef.Hash()
var written int64
written, err = io.Copy(io.MultiWriter(hash, sniffer), source)
if err != nil {
return
}
if !blobRef.HashMatches(hash) {
err = blobserver.ErrCorruptBlob
return
}
sniffer.Parse()
bm := ix.s.BeginBatch()
err = ix.populateMutation(blobRef, sniffer, bm)
if err != nil {
return
}
err = ix.s.CommitBatch(bm)
if err != nil {
return
}
mimeType := sniffer.MimeType()
log.Printf("indexer: received %s; type=%v; truncated=%v", blobRef, mimeType, sniffer.IsTruncated())
return blobref.SizedBlobRef{blobRef, written}, nil
}
func (s *storage) FetchStreaming(plainBR *blobref.BlobRef) (file io.ReadCloser, size int64, err error) {
meta, err := s.fetchMeta(plainBR)
if err != nil {
return nil, 0, err
}
encData, _, err := s.blobs.FetchStreaming(meta.EncBlobRef)
if err != nil {
log.Printf("encrypt: plaintext %s's encrypted %v blob not found", plainBR, meta.EncBlobRef)
return
}
defer encData.Close()
// Quick sanity check that the blob begins with the same IV we
// have in our metadata.
blobIV := make([]byte, len(meta.IV))
_, err = io.ReadFull(encData, blobIV)
if err != nil {
return nil, 0, fmt.Errorf("Error reading off IV header from blob: %v", err)
}
if !bytes.Equal(blobIV, meta.IV) {
return nil, 0, fmt.Errorf("Blob and meta IV don't match")
}
// Slurp the whole blob into memory to validate its plaintext
// checksum (no tampered bits) before returning it. Clients
// should be the party doing this in the general case, but
// we'll be extra paranoid and always do it here, at the cost
// of sometimes having it be done twice.
var plain bytes.Buffer
plainHash := plainBR.Hash()
plainSize, err := io.Copy(io.MultiWriter(&plain, plainHash), cipher.StreamReader{
S: cipher.NewCTR(s.block, meta.IV),
R: encData,
})
if err != nil {
return nil, 0, err
}
if !plainBR.HashMatches(plainHash) {
return nil, 0, blobserver.ErrCorruptBlob
}
return struct {
*bytes.Reader
io.Closer
}{
bytes.NewReader(plain.Bytes()),
dummyCloser,
}, plainSize, nil
}
func (tf *Fetcher) ReceiveBlob(br *blobref.BlobRef, source io.Reader) (blobref.SizedBlobRef, error) {
sb := blobref.SizedBlobRef{}
h := br.Hash()
if h == nil {
return sb, fmt.Errorf("Unsupported blobref hash for %s", br)
}
all, err := ioutil.ReadAll(io.TeeReader(source, h))
if err != nil {
return sb, err
}
if !br.HashMatches(h) {
return sb, fmt.Errorf("Hash mismatch receiving blob %s", br)
}
blob := &Blob{Contents: string(all)}
tf.AddBlob(blob)
return blobref.SizedBlobRef{br, int64(len(all))}, nil
}
func (sto *s3Storage) ReceiveBlob(blob *blobref.BlobRef, source io.Reader) (outsb blobref.SizedBlobRef, outerr error) {
zero := outsb
slurper := newAmazonSlurper(blob)
defer slurper.Cleanup()
hash := blob.Hash()
size, err := io.Copy(io.MultiWriter(hash, slurper), source)
if err != nil {
return zero, err
}
if !blob.HashMatches(hash) {
return zero, blobserver.ErrCorruptBlob
}
err = sto.s3Client.PutObject(blob.String(), sto.bucket, slurper.md5, size, slurper)
if err != nil {
return zero, err
}
return blobref.SizedBlobRef{BlobRef: blob, Size: size}, nil
}
func (x *appengineIndex) ReceiveBlob(br *blobref.BlobRef, in io.Reader) (sb blobref.SizedBlobRef, err os.Error) {
if x.ctx == nil {
err = errNoContext
return
}
var b bytes.Buffer
hash := br.Hash()
written, err := io.Copy(io.MultiWriter(hash, &b), in)
if err != nil {
return
}
if !br.HashMatches(hash) {
err = blobserver.ErrCorruptBlob
return
}
// TODO(bradfitz): implement
return blobref.SizedBlobRef{br, written}, nil
}
func (s *storage) ReceiveBlob(plainBR *blobref.BlobRef, source io.Reader) (sb blobref.SizedBlobRef, err error) {
iv := s.randIV()
stream := cipher.NewCTR(s.block, iv)
hash := plainBR.Hash()
var buf bytes.Buffer
// TODO: compress before encrypting?
buf.Write(iv) // TODO: write more structured header w/ version & IV length? or does that weaken it?
sw := cipher.StreamWriter{S: stream, W: &buf}
plainSize, err := io.Copy(io.MultiWriter(sw, hash), source)
if err != nil {
return sb, err
}
if !plainBR.HashMatches(hash) {
return sb, blobserver.ErrCorruptBlob
}
encBR := blobref.SHA1FromBytes(buf.Bytes())
_, err = s.blobs.ReceiveBlob(encBR, bytes.NewReader(buf.Bytes()))
if err != nil {
log.Printf("encrypt: error writing encrypted blob %v (plaintext %v): %v", encBR, plainBR, err)
return sb, errors.New("encrypt: error writing encrypted blob")
}
meta := encodeMetaValue(plainSize, iv, encBR, buf.Len())
metaBlob := s.makeSingleMetaBlob(plainBR, meta)
_, err = s.meta.ReceiveBlob(blobref.SHA1FromBytes(metaBlob), bytes.NewReader(metaBlob))
if err != nil {
log.Printf("encrypt: error writing encrypted meta for plaintext %v (encrypted blob %v): %v", plainBR, encBR, err)
return sb, errors.New("encrypt: error writing encrypted meta")
}
err = s.index.Set(plainBR.String(), meta)
if err != nil {
return sb, fmt.Errorf("encrypt: error updating index for encrypted %v (plaintext %v): %v", err)
}
return blobref.SizedBlobRef{plainBR, plainSize}, nil
}
func (gs *Storage) ReceiveBlob(blob *blobref.BlobRef, source io.Reader) (blobref.SizedBlobRef, error) {
buf := &bytes.Buffer{}
hash := blob.Hash()
size, err := io.Copy(io.MultiWriter(hash, buf), source)
if err != nil {
return blobref.SizedBlobRef{}, err
}
if !blob.HashMatches(hash) {
return blobref.SizedBlobRef{}, blobserver.ErrCorruptBlob
}
for tries, shouldRetry := 0, true; tries < 2 && shouldRetry; tries++ {
shouldRetry, err = gs.client.PutObject(
&googlestorage.Object{Bucket: gs.bucket, Key: blob.String()},
ioutil.NopCloser(bytes.NewReader(buf.Bytes())))
}
if err != nil {
return blobref.SizedBlobRef{}, err
}
return blobref.SizedBlobRef{BlobRef: blob, Size: size}, nil
}
func (ix *Index) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (retsb blobref.SizedBlobRef, err error) {
sniffer := new(BlobSniffer)
hash := blobRef.Hash()
var written int64
written, err = io.Copy(io.MultiWriter(hash, sniffer), source)
if err != nil {
return
}
if !blobRef.HashMatches(hash) {
err = blobserver.ErrCorruptBlob
return
}
sniffer.Parse()
bm := ix.s.BeginBatch()
err = ix.populateMutation(blobRef, sniffer, bm)
if err != nil {
return
}
err = ix.s.CommitBatch(bm)
if err != nil {
return
}
// TODO(bradfitz): log levels? These are generally noisy
// (especially in tests, like search/handler_test), but I
// could see it being useful in production. For now, disabled:
//
// mimeType := sniffer.MimeType()
// log.Printf("indexer: received %s; type=%v; truncated=%v", blobRef, mimeType, sniffer.IsTruncated())
return blobref.SizedBlobRef{blobRef, written}, nil
}
func (ds *DiskStorage) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (blobGot blobref.SizedBlobRef, err error) {
pname := ds.partition
if pname != "" {
err = fmt.Errorf("refusing upload directly to queue partition %q", pname)
return
}
hashedDirectory := ds.blobDirectory(pname, blobRef)
err = os.MkdirAll(hashedDirectory, 0700)
if err != nil {
return
}
tempFile, err := ioutil.TempFile(hashedDirectory, BlobFileBaseName(blobRef)+".tmp")
if err != nil {
return
}
success := false // set true later
defer func() {
if !success {
log.Println("Removing temp file: ", tempFile.Name())
os.Remove(tempFile.Name())
}
}()
hash := blobRef.Hash()
written, err := io.Copy(io.MultiWriter(hash, tempFile), source)
if err != nil {
return
}
if err = tempFile.Sync(); err != nil {
return
}
if err = tempFile.Close(); err != nil {
return
}
if !blobRef.HashMatches(hash) {
err = blobserver.ErrCorruptBlob
return
}
fileName := ds.blobPath("", blobRef)
if err = os.Rename(tempFile.Name(), fileName); err != nil {
return
}
stat, err := os.Lstat(fileName)
if err != nil {
return
}
if !!stat.IsDir() || stat.Size() != written {
err = errors.New("Written size didn't match.")
return
}
for _, mirror := range ds.mirrorPartitions {
pname := mirror.partition
if pname == "" {
panic("expected partition name")
}
partitionDir := ds.blobDirectory(pname, blobRef)
// Prevent the directory from being unlinked by
// enumerate code, which cleans up.
defer keepDirectoryLock(partitionDir).Unlock()
defer keepDirectoryLock(filepath.Dir(partitionDir)).Unlock()
defer keepDirectoryLock(filepath.Dir(filepath.Dir(partitionDir))).Unlock()
if err = os.MkdirAll(partitionDir, 0700); err != nil {
return blobref.SizedBlobRef{}, fmt.Errorf("localdisk.receive: MkdirAll(%q) after lock on it: %v", partitionDir, err)
}
partitionFileName := ds.blobPath(pname, blobRef)
pfi, err := os.Stat(partitionFileName)
if err == nil && !pfi.IsDir() {
log.Printf("Skipped dup on partition %q", pname)
} else {
if err = linkOrCopy(fileName, partitionFileName); err != nil && !linkAlreadyExists(err) {
log.Fatalf("got link or copy error %T %#v", err, err)
return blobref.SizedBlobRef{}, err
}
log.Printf("Mirrored blob %s to partition %q", blobRef, pname)
}
}
blobGot = blobref.SizedBlobRef{BlobRef: blobRef, Size: stat.Size()}
success = true
hub := ds.GetBlobHub()
hub.NotifyBlobReceived(blobRef)
for _, mirror := range ds.mirrorPartitions {
mirror.GetBlobHub().NotifyBlobReceived(blobRef)
}
return
}
func (sto *appengineStorage) ReceiveBlob(br *blobref.BlobRef, in io.Reader) (sb blobref.SizedBlobRef, err os.Error) {
if sto.ctx == nil {
err = errNoContext
return
}
var b bytes.Buffer
hash := br.Hash()
written, err := io.Copy(io.MultiWriter(hash, &b), in)
if err != nil {
return
}
if !br.HashMatches(hash) {
err = blobserver.ErrCorruptBlob
return
}
// bkey is non-empty once we've uploaded the blob.
var bkey appengine.BlobKey
// uploadBlob uploads the blob, unless it's already been done.
uploadBlob := func(ctx appengine.Context) os.Error {
if len(bkey) > 0 {
return nil // already done in previous transaction attempt
}
bw, err := blobstore.Create(ctx, "application/octet-stream")
if err != nil {
return err
}
_, err = io.Copy(bw, &b)
if err != nil {
// TODO(bradfitz): try to clean up; close it, see if we can find the key, delete it.
ctx.Errorf("blobstore Copy error: %v", err)
return err
}
err = bw.Close()
if err != nil {
// TODO(bradfitz): try to clean up; see if we can find the key, delete it.
ctx.Errorf("blobstore Close error: %v", err)
return err
}
k, err := bw.Key()
if err == nil {
bkey = k
}
return err
}
tryFunc := func(tc appengine.Context) os.Error {
row, err := fetchEnt(sto.ctx, br)
switch err {
case datastore.ErrNoSuchEntity:
if err := uploadBlob(sto.ctx); err != nil {
tc.Errorf("uploadBlob failed: %v", err)
return err
}
row = &blobEnt{
Size: []byte(fmt.Sprintf("%d", written)),
BlobKey: []byte(string(bkey)),
Namespaces: []byte(sto.namespace),
}
_, err = datastore.Put(tc, entKey(tc, br), row)
if err != nil {
return err
}
case nil:
if row.inNamespace(sto.namespace) {
// Nothing to do
return nil
}
row.Namespaces = []byte(string(row.Namespaces) + "|" + sto.namespace)
_, err = datastore.Put(tc, entKey(tc, br), row)
if err != nil {
return err
}
default:
return err
}
// Add membership row
_, err = datastore.Put(tc, sto.memKey(tc, br), &memEnt{
Size: []byte(fmt.Sprintf("%d", written)),
})
return err
}
err = datastore.RunInTransaction(sto.ctx, tryFunc, crossGroupTransaction)
if err != nil {
if len(bkey) > 0 {
// If we just created this blob but we
// ultimately failed, try our best to delete
// it so it's not orphaned.
blobstore.Delete(sto.ctx, bkey)
}
return
}
return blobref.SizedBlobRef{br, written}, nil
}