Example #1
0
func receive(dst BlobReceiver, br blob.Ref, src io.Reader, checkHash bool) (sb blob.SizedRef, err error) {
	if checkHash {
		src = &checkHashReader{br.Hash(), br, src}
	}
	sb, err = dst.ReceiveBlob(br, src)
	if err != nil {
		return
	}
	GetHub(dst).NotifyBlobReceived(sb.Ref)
	return
}
func testSizedBlob(t *testing.T, r io.Reader, b1 blob.Ref, size int64) {
	h := b1.Hash()
	n, err := io.Copy(h, r)
	if err != nil {
		t.Fatalf("error reading from %s: %v", r, err)
	}
	if n != size {
		t.Fatalf("read %d bytes from %s, metadata said %d!", n, r, size)
	}
	b2 := blob.RefFromHash(h)
	if b2 != b1 {
		t.Fatalf("content mismatch (awaited %s, got %s)", b1, b2)
	}
}
Example #3
0
func (s *storage) Fetch(plainBR blob.Ref) (file io.ReadCloser, size uint32, err error) {
	meta, err := s.fetchMeta(plainBR)
	if err != nil {
		return nil, 0, err
	}
	encData, _, err := s.blobs.Fetch(meta.EncBlobRef)
	if err != nil {
		log.Printf("encrypt: plaintext %s's encrypted %v blob not found", plainBR, meta.EncBlobRef)
		return
	}
	defer encData.Close()

	// Quick sanity check that the blob begins with the same IV we
	// have in our metadata.
	blobIV := make([]byte, len(meta.IV))
	_, err = io.ReadFull(encData, blobIV)
	if err != nil {
		return nil, 0, fmt.Errorf("Error reading off IV header from blob: %v", err)
	}
	if !bytes.Equal(blobIV, meta.IV) {
		return nil, 0, fmt.Errorf("Blob and meta IV don't match")
	}

	// Slurp the whole blob into memory to validate its plaintext
	// checksum (no tampered bits) before returning it. Clients
	// should be the party doing this in the general case, but
	// we'll be extra paranoid and always do it here, at the cost
	// of sometimes having it be done twice.
	var plain bytes.Buffer
	plainHash := plainBR.Hash()
	plainSize, err := io.Copy(io.MultiWriter(&plain, plainHash), cipher.StreamReader{
		S: cipher.NewCTR(s.block, meta.IV),
		R: encData,
	})
	if err != nil {
		return nil, 0, err
	}
	size = types.U32(plainSize)
	if !plainBR.HashMatches(plainHash) {
		return nil, 0, blobserver.ErrCorruptBlob
	}
	return struct {
		*bytes.Reader
		io.Closer
	}{
		bytes.NewReader(plain.Bytes()),
		types.NopCloser,
	}, uint32(plainSize), nil
}
Example #4
0
func receive(dst BlobReceiver, br blob.Ref, src io.Reader, checkHash bool) (sb blob.SizedRef, err error) {
	src = io.LimitReader(src, MaxBlobSize)
	if checkHash {
		src = &checkHashReader{br.Hash(), br, src, false}
	}
	sb, err = dst.ReceiveBlob(br, src)
	if err != nil {
		if checkHash && src.(*checkHashReader).corrupt {
			err = ErrCorruptBlob
		}
		return
	}
	err = GetHub(dst).NotifyBlobReceived(sb)
	return
}
Example #5
0
// NewSigner returns an Signer given an armored public key's blobref,
// its armored content, and its associated private key entity.
// The privateKeySource must be either an *openpgp.Entity or a string filename to a secret key.
func NewSigner(pubKeyRef blob.Ref, armoredPubKey io.Reader, privateKeySource interface{}) (*Signer, error) {
	hash := pubKeyRef.Hash()
	keyId, armoredPubKeyString, err := jsonsign.ParseArmoredPublicKey(io.TeeReader(armoredPubKey, hash))
	if err != nil {
		return nil, err
	}
	if !pubKeyRef.HashMatches(hash) {
		return nil, fmt.Errorf("pubkey ref of %v doesn't match provided armored public key", pubKeyRef)
	}

	var privateKey *openpgp.Entity
	switch v := privateKeySource.(type) {
	case *openpgp.Entity:
		privateKey = v
	case string:
		privateKey, err = jsonsign.EntityFromSecring(keyId, v)
		if err != nil {
			return nil, err
		}
	default:
		return nil, fmt.Errorf("invalid privateKeySource type %T", v)
	}
	if privateKey == nil {
		return nil, errors.New("nil privateKey")
	}

	return &Signer{
		keyId:      keyId,
		pubref:     pubKeyRef,
		privEntity: privateKey,
		baseSigReq: jsonsign.SignRequest{
			ServerMode: true, // shouldn't matter, since we're supplying the rest of the fields
			Fetcher: memoryBlobFetcher{
				pubKeyRef: func() (uint32, io.ReadCloser) {
					return uint32(len(armoredPubKeyString)), ioutil.NopCloser(strings.NewReader(armoredPubKeyString))
				},
			},
			EntityFetcher: entityFetcherFunc(func(wantKeyId string) (*openpgp.Entity, error) {
				if privateKey.PrivateKey.KeyIdString() != wantKeyId &&
					privateKey.PrivateKey.KeyIdShortString() != wantKeyId {
					return nil, fmt.Errorf("jsonsign code unexpectedly requested keyId %q; only have %q",
						wantKeyId, keyId)
				}
				return privateKey, nil
			}),
		},
	}, nil
}
Example #6
0
func (tf *Fetcher) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) {
	sb := blob.SizedRef{}
	h := br.Hash()
	if h == nil {
		return sb, fmt.Errorf("Unsupported blobref hash for %s", br)
	}
	all, err := ioutil.ReadAll(io.TeeReader(source, h))
	if err != nil {
		return sb, err
	}
	if !br.HashMatches(h) {
		return sb, fmt.Errorf("Hash mismatch receiving blob %s", br)
	}
	b := &Blob{Contents: string(all)}
	tf.AddBlob(b)
	return blob.SizedRef{br, int64(len(all))}, nil
}
Example #7
0
func (sto *s3Storage) ReceiveBlob(b blob.Ref, source io.Reader) (outsb blob.SizedRef, outerr error) {
	zero := outsb
	slurper := newAmazonSlurper(b)
	defer slurper.Cleanup()

	hash := b.Hash()
	size, err := io.Copy(io.MultiWriter(hash, slurper), source)
	if err != nil {
		return zero, err
	}
	if !b.HashMatches(hash) {
		return zero, blobserver.ErrCorruptBlob
	}
	err = sto.s3Client.PutObject(b.String(), sto.bucket, slurper.md5, size, slurper)
	if err != nil {
		return zero, err
	}
	return blob.SizedRef{Ref: b, Size: size}, nil
}
Example #8
0
func (tf *Fetcher) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) {
	sb := blob.SizedRef{}
	h := br.Hash()
	if h == nil {
		return sb, fmt.Errorf("Unsupported blobref hash for %s", br)
	}
	all, err := ioutil.ReadAll(io.TeeReader(source, h))
	if err != nil {
		return sb, err
	}
	if !br.HashMatches(h) {
		// This is a somewhat redundant check, since
		// blobserver.Receive now does it. But for testing code,
		// it's worth the cost.
		return sb, fmt.Errorf("Hash mismatch receiving blob %s", br)
	}
	b := &Blob{Contents: string(all)}
	tf.AddBlob(b)
	return blob.SizedRef{br, int64(len(all))}, nil
}
Example #9
0
func (s *storage) ReceiveBlob(plainBR blob.Ref, source io.Reader) (sb blob.SizedRef, err error) {
	iv := s.randIV()
	stream := cipher.NewCTR(s.block, iv)

	hash := plainBR.Hash()
	var buf bytes.Buffer
	// TODO: compress before encrypting?
	buf.Write(iv) // TODO: write more structured header w/ version & IV length? or does that weaken it?
	sw := cipher.StreamWriter{S: stream, W: &buf}
	plainSize, err := io.Copy(io.MultiWriter(sw, hash), source)
	if err != nil {
		return sb, err
	}
	if !plainBR.HashMatches(hash) {
		return sb, blobserver.ErrCorruptBlob
	}

	encBR := blob.SHA1FromBytes(buf.Bytes())
	_, err = blobserver.Receive(s.blobs, encBR, bytes.NewReader(buf.Bytes()))
	if err != nil {
		log.Printf("encrypt: error writing encrypted blob %v (plaintext %v): %v", encBR, plainBR, err)
		return sb, errors.New("encrypt: error writing encrypted blob")
	}

	meta := encodeMetaValue(uint32(plainSize), iv, encBR, buf.Len())
	metaBlob := s.makeSingleMetaBlob(plainBR, meta)
	_, err = blobserver.ReceiveNoHash(s.meta, blob.SHA1FromBytes(metaBlob), bytes.NewReader(metaBlob))
	if err != nil {
		log.Printf("encrypt: error writing encrypted meta for plaintext %v (encrypted blob %v): %v", plainBR, encBR, err)
		return sb, errors.New("encrypt: error writing encrypted meta")
	}

	err = s.index.Set(plainBR.String(), meta)
	if err != nil {
		return sb, fmt.Errorf("encrypt: error updating index for encrypted %v (plaintext %v): %v", encBR, plainBR, err)
	}

	return blob.SizedRef{plainBR, uint32(plainSize)}, nil
}
Example #10
0
func (gs *Storage) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) {
	buf := &bytes.Buffer{}
	hash := br.Hash()
	size, err := io.Copy(io.MultiWriter(hash, buf), source)
	if err != nil {
		return blob.SizedRef{}, err
	}
	if !br.HashMatches(hash) {
		return blob.SizedRef{}, blobserver.ErrCorruptBlob
	}

	for tries, shouldRetry := 0, true; tries < 2 && shouldRetry; tries++ {
		shouldRetry, err = gs.client.PutObject(
			&googlestorage.Object{Bucket: gs.bucket, Key: br.String()},
			ioutil.NopCloser(bytes.NewReader(buf.Bytes())))
	}
	if err != nil {
		return blob.SizedRef{}, err
	}

	return blob.SizedRef{Ref: br, Size: size}, nil
}
Example #11
0
func (s *Storage) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) {
	sb := blob.SizedRef{}
	h := br.Hash()
	if h == nil {
		return sb, fmt.Errorf("Unsupported blobref hash for %s", br)
	}
	all, err := ioutil.ReadAll(io.TeeReader(source, h))
	if err != nil {
		return sb, err
	}
	if !br.HashMatches(h) {
		// This is a somewhat redundant check, since
		// blobserver.Receive now does it. But for testing code,
		// it's worth the cost.
		return sb, fmt.Errorf("Hash mismatch receiving blob %s", br)
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.m == nil {
		s.m = make(map[blob.Ref][]byte)
	}
	_, had := s.m[br]
	if !had {
		s.m[br] = all
		if s.lru != nil {
			s.lru.Add(br.String(), nil)
		}
		s.size += int64(len(all))
		for s.maxSize != 0 && s.size > s.maxSize {
			if key, _ := s.lru.RemoveOldest(); key != "" {
				s.removeBlobLocked(blob.MustParse(key))
			} else {
				break // shouldn't happen
			}
		}
	}
	return blob.SizedRef{br, uint32(len(all))}, nil
}
Example #12
0
func (ix *Index) ReceiveBlob(blobRef blob.Ref, source io.Reader) (retsb blob.SizedRef, err error) {
	sniffer := NewBlobSniffer(blobRef)
	hash := blobRef.Hash()
	var written int64
	written, err = io.Copy(io.MultiWriter(hash, sniffer), source)
	if err != nil {
		return
	}

	if !blobRef.HashMatches(hash) {
		err = blobserver.ErrCorruptBlob
		return
	}
	sniffer.Parse()

	bm := ix.s.BeginBatch()

	err = ix.populateMutation(blobRef, sniffer, bm)
	if err != nil {
		return
	}

	err = ix.s.CommitBatch(bm)
	if err != nil {
		return
	}

	// TODO(bradfitz): log levels? These are generally noisy
	// (especially in tests, like search/handler_test), but I
	// could see it being useful in production. For now, disabled:
	//
	// mimeType := sniffer.MIMEType()
	// log.Printf("indexer: received %s; type=%v; truncated=%v", blobRef, mimeType, sniffer.IsTruncated())

	return blob.SizedRef{blobRef, written}, nil
}
Example #13
0
func (ds *DiskStorage) ReceiveBlob(blobRef blob.Ref, source io.Reader) (blobGot blob.SizedRef, err error) {
	pname := ds.partition
	if pname != "" {
		err = fmt.Errorf("refusing upload directly to queue partition %q", pname)
		return
	}
	hashedDirectory := ds.blobDirectory(pname, blobRef)
	err = os.MkdirAll(hashedDirectory, 0700)
	if err != nil {
		return
	}

	tempFile, err := ioutil.TempFile(hashedDirectory, blobFileBaseName(blobRef)+".tmp")
	if err != nil {
		return
	}

	success := false // set true later
	defer func() {
		if !success {
			log.Println("Removing temp file: ", tempFile.Name())
			os.Remove(tempFile.Name())
		}
	}()

	hash := blobRef.Hash()
	written, err := io.Copy(io.MultiWriter(hash, tempFile), source)
	if err != nil {
		return
	}
	if err = tempFile.Sync(); err != nil {
		return
	}
	if err = tempFile.Close(); err != nil {
		return
	}
	if !blobRef.HashMatches(hash) {
		err = blobserver.ErrCorruptBlob
		return
	}
	stat, err := os.Lstat(tempFile.Name())
	if err != nil {
		return
	}
	if stat.Size() != written {
		err = fmt.Errorf("temp file %q size %d didn't match written size %d", tempFile.Name(), stat.Size(), written)
		return
	}

	fileName := ds.blobPath("", blobRef)
	if err = os.Rename(tempFile.Name(), fileName); err != nil {
		return
	}

	stat, err = os.Lstat(fileName)
	if err != nil {
		return
	}
	if stat.Size() != written {
		err = errors.New("Written size didn't match.")
		return
	}

	for _, mirror := range ds.mirrorPartitions {
		pname := mirror.partition
		if pname == "" {
			panic("expected partition name")
		}
		partitionDir := ds.blobDirectory(pname, blobRef)

		// Prevent the directory from being unlinked by
		// enumerate code, which cleans up.
		defer keepDirectoryLock(partitionDir).Unlock()
		defer keepDirectoryLock(filepath.Dir(partitionDir)).Unlock()
		defer keepDirectoryLock(filepath.Dir(filepath.Dir(partitionDir))).Unlock()

		if err = os.MkdirAll(partitionDir, 0700); err != nil {
			return blob.SizedRef{}, fmt.Errorf("localdisk.receive: MkdirAll(%q) after lock on it: %v", partitionDir, err)
		}
		partitionFileName := ds.blobPath(pname, blobRef)
		pfi, err := os.Stat(partitionFileName)
		if err == nil && !pfi.IsDir() {
			log.Printf("Skipped dup on partition %q", pname)
		} else {
			if err = linkOrCopy(fileName, partitionFileName); err != nil && !linkAlreadyExists(err) {
				log.Fatalf("got link or copy error %T %#v", err, err)
				return blob.SizedRef{}, err
			}
			log.Printf("Mirrored blob %s to partition %q", blobRef, pname)
		}
	}

	blobGot = blob.SizedRef{Ref: blobRef, Size: stat.Size()}
	success = true

	hub := ds.GetBlobHub()
	hub.NotifyBlobReceived(blobRef)
	for _, mirror := range ds.mirrorPartitions {
		mirror.GetBlobHub().NotifyBlobReceived(blobRef)
	}
	return
}
Example #14
0
func (sto *appengineStorage) ReceiveBlob(br blob.Ref, in io.Reader) (sb blob.SizedRef, err error) {
	ctx := sto.ctx
	if ctx == nil {
		loan := ctxPool.Get()
		defer loan.Return()
		ctx = loan
	}

	var b bytes.Buffer
	hash := br.Hash()
	written, err := io.Copy(io.MultiWriter(hash, &b), in)
	if err != nil {
		return
	}
	if !br.HashMatches(hash) {
		err = blobserver.ErrCorruptBlob
		return
	}

	// bkey is non-empty once we've uploaded the blob.
	var bkey appengine.BlobKey

	// uploadBlob uploads the blob, unless it's already been done.
	uploadBlob := func(ctx appengine.Context) error {
		if len(bkey) > 0 {
			return nil // already done in previous transaction attempt
		}
		bw, err := blobstore.Create(ctx, "application/octet-stream")
		if err != nil {
			return err
		}
		_, err = io.Copy(bw, &b)
		if err != nil {
			// TODO(bradfitz): try to clean up; close it, see if we can find the key, delete it.
			ctx.Errorf("blobstore Copy error: %v", err)
			return err
		}
		err = bw.Close()
		if err != nil {
			// TODO(bradfitz): try to clean up; see if we can find the key, delete it.
			ctx.Errorf("blobstore Close error: %v", err)
			return err
		}
		k, err := bw.Key()
		if err == nil {
			bkey = k
		}
		return err
	}

	tryFunc := func(tc appengine.Context) error {
		row, err := fetchEnt(tc, br)
		switch err {
		case datastore.ErrNoSuchEntity:
			if err := uploadBlob(tc); err != nil {
				tc.Errorf("uploadBlob failed: %v", err)
				return err
			}
			row = &blobEnt{
				Size:       written,
				BlobKey:    bkey,
				Namespaces: sto.namespace,
			}
			_, err = datastore.Put(tc, entKey(tc, br), row)
			if err != nil {
				return err
			}
		case nil:
			if row.inNamespace(sto.namespace) {
				// Nothing to do
				return nil
			}
			row.Namespaces = row.Namespaces + "|" + sto.namespace
			_, err = datastore.Put(tc, entKey(tc, br), row)
			if err != nil {
				return err
			}
		default:
			return err
		}

		// Add membership row
		_, err = datastore.Put(tc, sto.memKey(tc, br), &memEnt{
			Size: written,
		})
		return err
	}
	err = datastore.RunInTransaction(ctx, tryFunc, crossGroupTransaction)
	if err != nil {
		if len(bkey) > 0 {
			// If we just created this blob but we
			// ultimately failed, try our best to delete
			// it so it's not orphaned.
			blobstore.Delete(ctx, bkey)
		}
		return
	}
	return blob.SizedRef{br, written}, nil
}