func (s *storage) Fetch(plainBR blob.Ref) (file io.ReadCloser, size uint32, err error) { meta, err := s.fetchMeta(plainBR) if err != nil { return nil, 0, err } encData, _, err := s.blobs.Fetch(meta.EncBlobRef) if err != nil { log.Printf("encrypt: plaintext %s's encrypted %v blob not found", plainBR, meta.EncBlobRef) return } defer encData.Close() // Quick sanity check that the blob begins with the same IV we // have in our metadata. blobIV := make([]byte, len(meta.IV)) _, err = io.ReadFull(encData, blobIV) if err != nil { return nil, 0, fmt.Errorf("Error reading off IV header from blob: %v", err) } if !bytes.Equal(blobIV, meta.IV) { return nil, 0, fmt.Errorf("Blob and meta IV don't match") } // Slurp the whole blob into memory to validate its plaintext // checksum (no tampered bits) before returning it. Clients // should be the party doing this in the general case, but // we'll be extra paranoid and always do it here, at the cost // of sometimes having it be done twice. var plain bytes.Buffer plainHash := plainBR.Hash() plainSize, err := io.Copy(io.MultiWriter(&plain, plainHash), cipher.StreamReader{ S: cipher.NewCTR(s.block, meta.IV), R: encData, }) if err != nil { return nil, 0, err } size = types.U32(plainSize) if !plainBR.HashMatches(plainHash) { return nil, 0, blobserver.ErrCorruptBlob } return struct { *bytes.Reader io.Closer }{ bytes.NewReader(plain.Bytes()), types.NopCloser, }, uint32(plainSize), nil }
// NewSigner returns an Signer given an armored public key's blobref, // its armored content, and its associated private key entity. // The privateKeySource must be either an *openpgp.Entity or a string filename to a secret key. func NewSigner(pubKeyRef blob.Ref, armoredPubKey io.Reader, privateKeySource interface{}) (*Signer, error) { hash := pubKeyRef.Hash() keyId, armoredPubKeyString, err := jsonsign.ParseArmoredPublicKey(io.TeeReader(armoredPubKey, hash)) if err != nil { return nil, err } if !pubKeyRef.HashMatches(hash) { return nil, fmt.Errorf("pubkey ref of %v doesn't match provided armored public key", pubKeyRef) } var privateKey *openpgp.Entity switch v := privateKeySource.(type) { case *openpgp.Entity: privateKey = v case string: privateKey, err = jsonsign.EntityFromSecring(keyId, v) if err != nil { return nil, err } default: return nil, fmt.Errorf("invalid privateKeySource type %T", v) } if privateKey == nil { return nil, errors.New("nil privateKey") } return &Signer{ keyId: keyId, pubref: pubKeyRef, privEntity: privateKey, baseSigReq: jsonsign.SignRequest{ ServerMode: true, // shouldn't matter, since we're supplying the rest of the fields Fetcher: memoryBlobFetcher{ pubKeyRef: func() (uint32, io.ReadCloser) { return uint32(len(armoredPubKeyString)), ioutil.NopCloser(strings.NewReader(armoredPubKeyString)) }, }, EntityFetcher: entityFetcherFunc(func(wantKeyId string) (*openpgp.Entity, error) { if privateKey.PrivateKey.KeyIdString() != wantKeyId && privateKey.PrivateKey.KeyIdShortString() != wantKeyId { return nil, fmt.Errorf("jsonsign code unexpectedly requested keyId %q; only have %q", wantKeyId, keyId) } return privateKey, nil }), }, }, nil }
func (tf *Fetcher) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) { sb := blob.SizedRef{} h := br.Hash() if h == nil { return sb, fmt.Errorf("Unsupported blobref hash for %s", br) } all, err := ioutil.ReadAll(io.TeeReader(source, h)) if err != nil { return sb, err } if !br.HashMatches(h) { return sb, fmt.Errorf("Hash mismatch receiving blob %s", br) } b := &Blob{Contents: string(all)} tf.AddBlob(b) return blob.SizedRef{br, int64(len(all))}, nil }
func (sto *s3Storage) ReceiveBlob(b blob.Ref, source io.Reader) (outsb blob.SizedRef, outerr error) { zero := outsb slurper := newAmazonSlurper(b) defer slurper.Cleanup() hash := b.Hash() size, err := io.Copy(io.MultiWriter(hash, slurper), source) if err != nil { return zero, err } if !b.HashMatches(hash) { return zero, blobserver.ErrCorruptBlob } err = sto.s3Client.PutObject(b.String(), sto.bucket, slurper.md5, size, slurper) if err != nil { return zero, err } return blob.SizedRef{Ref: b, Size: size}, nil }
func (tf *Fetcher) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) { sb := blob.SizedRef{} h := br.Hash() if h == nil { return sb, fmt.Errorf("Unsupported blobref hash for %s", br) } all, err := ioutil.ReadAll(io.TeeReader(source, h)) if err != nil { return sb, err } if !br.HashMatches(h) { // This is a somewhat redundant check, since // blobserver.Receive now does it. But for testing code, // it's worth the cost. return sb, fmt.Errorf("Hash mismatch receiving blob %s", br) } b := &Blob{Contents: string(all)} tf.AddBlob(b) return blob.SizedRef{br, int64(len(all))}, nil }
func (s *storage) ReceiveBlob(plainBR blob.Ref, source io.Reader) (sb blob.SizedRef, err error) { iv := s.randIV() stream := cipher.NewCTR(s.block, iv) hash := plainBR.Hash() var buf bytes.Buffer // TODO: compress before encrypting? buf.Write(iv) // TODO: write more structured header w/ version & IV length? or does that weaken it? sw := cipher.StreamWriter{S: stream, W: &buf} plainSize, err := io.Copy(io.MultiWriter(sw, hash), source) if err != nil { return sb, err } if !plainBR.HashMatches(hash) { return sb, blobserver.ErrCorruptBlob } encBR := blob.SHA1FromBytes(buf.Bytes()) _, err = blobserver.Receive(s.blobs, encBR, bytes.NewReader(buf.Bytes())) if err != nil { log.Printf("encrypt: error writing encrypted blob %v (plaintext %v): %v", encBR, plainBR, err) return sb, errors.New("encrypt: error writing encrypted blob") } meta := encodeMetaValue(uint32(plainSize), iv, encBR, buf.Len()) metaBlob := s.makeSingleMetaBlob(plainBR, meta) _, err = blobserver.ReceiveNoHash(s.meta, blob.SHA1FromBytes(metaBlob), bytes.NewReader(metaBlob)) if err != nil { log.Printf("encrypt: error writing encrypted meta for plaintext %v (encrypted blob %v): %v", plainBR, encBR, err) return sb, errors.New("encrypt: error writing encrypted meta") } err = s.index.Set(plainBR.String(), meta) if err != nil { return sb, fmt.Errorf("encrypt: error updating index for encrypted %v (plaintext %v): %v", encBR, plainBR, err) } return blob.SizedRef{plainBR, uint32(plainSize)}, nil }
func (gs *Storage) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) { buf := &bytes.Buffer{} hash := br.Hash() size, err := io.Copy(io.MultiWriter(hash, buf), source) if err != nil { return blob.SizedRef{}, err } if !br.HashMatches(hash) { return blob.SizedRef{}, blobserver.ErrCorruptBlob } for tries, shouldRetry := 0, true; tries < 2 && shouldRetry; tries++ { shouldRetry, err = gs.client.PutObject( &googlestorage.Object{Bucket: gs.bucket, Key: br.String()}, ioutil.NopCloser(bytes.NewReader(buf.Bytes()))) } if err != nil { return blob.SizedRef{}, err } return blob.SizedRef{Ref: br, Size: size}, nil }
func (s *Storage) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) { sb := blob.SizedRef{} h := br.Hash() if h == nil { return sb, fmt.Errorf("Unsupported blobref hash for %s", br) } all, err := ioutil.ReadAll(io.TeeReader(source, h)) if err != nil { return sb, err } if !br.HashMatches(h) { // This is a somewhat redundant check, since // blobserver.Receive now does it. But for testing code, // it's worth the cost. return sb, fmt.Errorf("Hash mismatch receiving blob %s", br) } s.mu.Lock() defer s.mu.Unlock() if s.m == nil { s.m = make(map[blob.Ref][]byte) } _, had := s.m[br] if !had { s.m[br] = all if s.lru != nil { s.lru.Add(br.String(), nil) } s.size += int64(len(all)) for s.maxSize != 0 && s.size > s.maxSize { if key, _ := s.lru.RemoveOldest(); key != "" { s.removeBlobLocked(blob.MustParse(key)) } else { break // shouldn't happen } } } return blob.SizedRef{br, uint32(len(all))}, nil }
func (ix *Index) ReceiveBlob(blobRef blob.Ref, source io.Reader) (retsb blob.SizedRef, err error) { sniffer := NewBlobSniffer(blobRef) hash := blobRef.Hash() var written int64 written, err = io.Copy(io.MultiWriter(hash, sniffer), source) if err != nil { return } if !blobRef.HashMatches(hash) { err = blobserver.ErrCorruptBlob return } sniffer.Parse() bm := ix.s.BeginBatch() err = ix.populateMutation(blobRef, sniffer, bm) if err != nil { return } err = ix.s.CommitBatch(bm) if err != nil { return } // TODO(bradfitz): log levels? These are generally noisy // (especially in tests, like search/handler_test), but I // could see it being useful in production. For now, disabled: // // mimeType := sniffer.MIMEType() // log.Printf("indexer: received %s; type=%v; truncated=%v", blobRef, mimeType, sniffer.IsTruncated()) return blob.SizedRef{blobRef, written}, nil }
func (ds *DiskStorage) ReceiveBlob(blobRef blob.Ref, source io.Reader) (blobGot blob.SizedRef, err error) { pname := ds.partition if pname != "" { err = fmt.Errorf("refusing upload directly to queue partition %q", pname) return } hashedDirectory := ds.blobDirectory(pname, blobRef) err = os.MkdirAll(hashedDirectory, 0700) if err != nil { return } tempFile, err := ioutil.TempFile(hashedDirectory, blobFileBaseName(blobRef)+".tmp") if err != nil { return } success := false // set true later defer func() { if !success { log.Println("Removing temp file: ", tempFile.Name()) os.Remove(tempFile.Name()) } }() hash := blobRef.Hash() written, err := io.Copy(io.MultiWriter(hash, tempFile), source) if err != nil { return } if err = tempFile.Sync(); err != nil { return } if err = tempFile.Close(); err != nil { return } if !blobRef.HashMatches(hash) { err = blobserver.ErrCorruptBlob return } stat, err := os.Lstat(tempFile.Name()) if err != nil { return } if stat.Size() != written { err = fmt.Errorf("temp file %q size %d didn't match written size %d", tempFile.Name(), stat.Size(), written) return } fileName := ds.blobPath("", blobRef) if err = os.Rename(tempFile.Name(), fileName); err != nil { return } stat, err = os.Lstat(fileName) if err != nil { return } if stat.Size() != written { err = errors.New("Written size didn't match.") return } for _, mirror := range ds.mirrorPartitions { pname := mirror.partition if pname == "" { panic("expected partition name") } partitionDir := ds.blobDirectory(pname, blobRef) // Prevent the directory from being unlinked by // enumerate code, which cleans up. defer keepDirectoryLock(partitionDir).Unlock() defer keepDirectoryLock(filepath.Dir(partitionDir)).Unlock() defer keepDirectoryLock(filepath.Dir(filepath.Dir(partitionDir))).Unlock() if err = os.MkdirAll(partitionDir, 0700); err != nil { return blob.SizedRef{}, fmt.Errorf("localdisk.receive: MkdirAll(%q) after lock on it: %v", partitionDir, err) } partitionFileName := ds.blobPath(pname, blobRef) pfi, err := os.Stat(partitionFileName) if err == nil && !pfi.IsDir() { log.Printf("Skipped dup on partition %q", pname) } else { if err = linkOrCopy(fileName, partitionFileName); err != nil && !linkAlreadyExists(err) { log.Fatalf("got link or copy error %T %#v", err, err) return blob.SizedRef{}, err } log.Printf("Mirrored blob %s to partition %q", blobRef, pname) } } blobGot = blob.SizedRef{Ref: blobRef, Size: stat.Size()} success = true hub := ds.GetBlobHub() hub.NotifyBlobReceived(blobRef) for _, mirror := range ds.mirrorPartitions { mirror.GetBlobHub().NotifyBlobReceived(blobRef) } return }
func (sto *appengineStorage) ReceiveBlob(br blob.Ref, in io.Reader) (sb blob.SizedRef, err error) { ctx := sto.ctx if ctx == nil { loan := ctxPool.Get() defer loan.Return() ctx = loan } var b bytes.Buffer hash := br.Hash() written, err := io.Copy(io.MultiWriter(hash, &b), in) if err != nil { return } if !br.HashMatches(hash) { err = blobserver.ErrCorruptBlob return } // bkey is non-empty once we've uploaded the blob. var bkey appengine.BlobKey // uploadBlob uploads the blob, unless it's already been done. uploadBlob := func(ctx appengine.Context) error { if len(bkey) > 0 { return nil // already done in previous transaction attempt } bw, err := blobstore.Create(ctx, "application/octet-stream") if err != nil { return err } _, err = io.Copy(bw, &b) if err != nil { // TODO(bradfitz): try to clean up; close it, see if we can find the key, delete it. ctx.Errorf("blobstore Copy error: %v", err) return err } err = bw.Close() if err != nil { // TODO(bradfitz): try to clean up; see if we can find the key, delete it. ctx.Errorf("blobstore Close error: %v", err) return err } k, err := bw.Key() if err == nil { bkey = k } return err } tryFunc := func(tc appengine.Context) error { row, err := fetchEnt(tc, br) switch err { case datastore.ErrNoSuchEntity: if err := uploadBlob(tc); err != nil { tc.Errorf("uploadBlob failed: %v", err) return err } row = &blobEnt{ Size: written, BlobKey: bkey, Namespaces: sto.namespace, } _, err = datastore.Put(tc, entKey(tc, br), row) if err != nil { return err } case nil: if row.inNamespace(sto.namespace) { // Nothing to do return nil } row.Namespaces = row.Namespaces + "|" + sto.namespace _, err = datastore.Put(tc, entKey(tc, br), row) if err != nil { return err } default: return err } // Add membership row _, err = datastore.Put(tc, sto.memKey(tc, br), &memEnt{ Size: written, }) return err } err = datastore.RunInTransaction(ctx, tryFunc, crossGroupTransaction) if err != nil { if len(bkey) > 0 { // If we just created this blob but we // ultimately failed, try our best to delete // it so it's not orphaned. blobstore.Delete(ctx, bkey) } return } return blob.SizedRef{br, written}, nil }