Ejemplo n.º 1
0
func TestReceiveIsSchema(t *testing.T) {
	ld := test.NewLoader()
	sto := newCond(t, ld, map[string]interface{}{
		"write": map[string]interface{}{
			"if":   "isSchema",
			"then": "/good-schema/",
			"else": "/good-other/",
		},
		"read": "/good-other/",
	})
	otherBlob := &test.Blob{Contents: "stuff"}
	schemaBlob := &test.Blob{Contents: `{"camliVersion": 1, "camliType": "foo"}`}

	ssb := mustReceive(t, sto, schemaBlob)
	osb := mustReceive(t, sto, otherBlob)

	ssto, _ := ld.GetStorage("/good-schema/")
	osto, _ := ld.GetStorage("/good-other/")

	if _, err := blobserver.StatBlob(ssto, ssb.Ref); err != nil {
		t.Errorf("schema blob didn't end up on schema storage")
	}
	if _, err := blobserver.StatBlob(osto, osb.Ref); err != nil {
		t.Errorf("other blob didn't end up on other storage")
	}
}
Ejemplo n.º 2
0
func (h *Handler) uploadPublicKey() error {
	h.pubKeyUploadMu.RLock()
	if h.pubKeyUploaded {
		h.pubKeyUploadMu.RUnlock()
		return nil
	}
	h.pubKeyUploadMu.RUnlock()

	sto := h.pubKeyDest

	h.pubKeyUploadMu.Lock()
	defer h.pubKeyUploadMu.Unlock()
	if h.pubKeyUploaded {
		return nil
	}
	_, err := blobserver.StatBlob(sto, h.pubKeyBlobRef)
	if err == nil {
		h.pubKeyUploaded = true
		return nil
	}
	_, err = blobserver.Receive(sto, h.pubKeyBlobRef, strings.NewReader(h.pubKey))
	log.Printf("uploadPublicKey(%T, %v) = %v", sto, h.pubKeyBlobRef, err)
	if err == nil {
		h.pubKeyUploaded = true
	}
	return err
}
Ejemplo n.º 3
0
func (h *Handler) uploadPublicKey(sto blobserver.Storage, key string) error {
	_, err := blobserver.StatBlob(sto, h.pubKeyBlobRef)
	if err == nil {
		return nil
	}
	_, err = sto.ReceiveBlob(h.pubKeyBlobRef, strings.NewReader(key))
	return err
}
Ejemplo n.º 4
0
func serverHasBlob(bs blobserver.BlobStatter, br blob.Ref) (have bool, err error) {
	_, err = blobserver.StatBlob(bs, br)
	if err == nil {
		have = true
	} else if err == os.ErrNotExist {
		err = nil
	}
	return
}
Ejemplo n.º 5
0
// see if storage proxies through to small for Fetch, Stat, and Enumerate.
func TestSmallFallback(t *testing.T) {
	small := new(test.Fetcher)
	s := &storage{
		small: small,
		large: new(test.Fetcher),
		meta:  sorted.NewMemoryKeyValue(),
		log:   test.NewLogger(t, "blobpacked: "),
	}
	s.init()
	b1 := &test.Blob{"foo"}
	b1.MustUpload(t, small)
	wantSB := b1.SizedRef()

	// Fetch
	rc, _, err := s.Fetch(b1.BlobRef())
	if err != nil {
		t.Errorf("failed to Get blob: %v", err)
	} else {
		rc.Close()
	}

	// Stat.
	sb, err := blobserver.StatBlob(s, b1.BlobRef())
	if err != nil {
		t.Errorf("failed to Stat blob: %v", err)
	} else if sb != wantSB {
		t.Errorf("Stat = %v; want %v", sb, wantSB)
	}

	// Enumerate
	saw := false
	ctx, cancel := context.WithCancel(context.TODO())
	defer cancel()
	if err := blobserver.EnumerateAll(ctx, s, func(sb blob.SizedRef) error {
		if sb != wantSB {
			return fmt.Errorf("saw blob %v; want %v", sb, wantSB)
		}
		saw = true
		return nil
	}); err != nil {
		t.Errorf("EnuerateAll: %v", err)
	}
	if !saw {
		t.Error("didn't see blob in Enumerate")
	}
}
Ejemplo n.º 6
0
func TestReceiveGood(t *testing.T) {
	sto := newReplica(t, map[string]interface{}{
		"backends": []interface{}{"/good-1/", "/good-2/"},
	})
	tb := &test.Blob{Contents: "stuff"}
	sb := mustReceive(t, sto, tb)

	if len(sto.replicas) != 2 {
		t.Fatalf("replicas = %d; want 2", len(sto.replicas))
	}
	for i, rep := range sto.replicas {
		got, err := blobserver.StatBlob(rep, sb.Ref)
		if err != nil {
			t.Errorf("Replica %s got stat error %v", sto.replicaPrefixes[i], err)
		} else if got != sb {
			t.Errorf("Replica %s got %+v; want %+v", sto.replicaPrefixes[i], got, sb)
		}
	}
}
Ejemplo n.º 7
0
func TestReceiveOneGoodOneFail(t *testing.T) {
	sto := newReplica(t, map[string]interface{}{
		"backends":            []interface{}{"/good-1/", "/fail-1/"},
		"minWritesForSuccess": float64(1),
	})
	tb := &test.Blob{Contents: "stuff"}
	sb := mustReceive(t, sto, tb)

	if len(sto.replicas) != 2 {
		t.Fatalf("replicas = %d; want 2", len(sto.replicas))
	}
	for i, rep := range sto.replicas {
		got, err := blobserver.StatBlob(rep, sb.Ref)
		pfx := sto.replicaPrefixes[i]
		if (i == 0) != (err == nil) {
			t.Errorf("For replica %s, unexpected error: %v", pfx, err)
		}
		if err == nil && got != sb {
			t.Errorf("Replica %s got %+v; want %+v", sto.replicaPrefixes[i], got, sb)
		}
	}
}
Ejemplo n.º 8
0
// foreachZipBlob calls fn for each blob in the zip pack blob
// identified by zipRef.  If fn returns a non-nil error,
// foreachZipBlob stops enumerating with that error.
func (s *storage) foreachZipBlob(zipRef blob.Ref, fn func(BlobAndPos) error) error {
	sb, err := blobserver.StatBlob(s.large, zipRef)
	if err != nil {
		return err
	}
	zr, err := zip.NewReader(blob.ReaderAt(s.large, zipRef), int64(sb.Size))
	if err != nil {
		return zipOpenError{zipRef, err}
	}
	var maniFile *zip.File // or nil if not found
	var firstOff int64     // offset of first file (the packed data chunks)
	for i, f := range zr.File {
		if i == 0 {
			firstOff, err = f.DataOffset()
			if err != nil {
				return err
			}
		}
		if f.Name == zipManifestPath {
			maniFile = f
			break
		}
	}
	if maniFile == nil {
		return errors.New("no camlistore manifest file found in zip")
	}
	// apply fn to all the schema blobs
	for _, f := range zr.File {
		if !strings.HasPrefix(f.Name, "camlistore/") || f.Name == zipManifestPath ||
			!strings.HasSuffix(f.Name, ".json") {
			continue
		}
		brStr := strings.TrimSuffix(strings.TrimPrefix(f.Name, "camlistore/"), ".json")
		br, ok := blob.Parse(brStr)
		if ok {
			off, err := f.DataOffset()
			if err != nil {
				return err
			}
			if err := fn(BlobAndPos{
				SizedRef: blob.SizedRef{Ref: br, Size: uint32(f.UncompressedSize64)},
				Offset:   off,
			}); err != nil {
				return err
			}
		}
	}
	maniRC, err := maniFile.Open()
	if err != nil {
		return err
	}
	defer maniRC.Close()
	var mf Manifest
	if err := json.NewDecoder(maniRC).Decode(&mf); err != nil {
		return err
	}
	if !mf.WholeRef.Valid() || mf.WholeSize == 0 || !mf.DataBlobsOrigin.Valid() {
		return errors.New("incomplete blobpack manifest JSON")
	}
	// apply fn to all the data blobs
	for _, bap := range mf.DataBlobs {
		bap.Offset += firstOff
		if err := fn(bap); err != nil {
			return err
		}
	}
	return nil
}