func TestIsolation(t *testing.T) {
ld := test.NewLoader()
master, _ := ld.GetStorage("/good-storage/")
ns1 := newNamespace(t, ld)
ns2 := newNamespace(t, ld)
stoMap := map[string]blobserver.Storage{
"ns1": ns1,
"ns2": ns2,
"master": master,
}
want := func(src string, want ...blob.Ref) {
if _, ok := stoMap[src]; !ok {
t.Fatalf("undefined storage %q", src)
}
sort.Sort(blob.ByRef(want))
var got []blob.Ref
if err := blobserver.EnumerateAll(context.TODO(), stoMap[src], func(sb blob.SizedRef) error {
got = append(got, sb.Ref)
return nil
}); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got, want) {
t.Errorf("server %q = %q; want %q", src, got, want)
}
}
b1 := &test.Blob{Contents: "Blob 1"}
b1r := b1.BlobRef()
b2 := &test.Blob{Contents: "Blob 2"}
b2r := b2.BlobRef()
b3 := &test.Blob{Contents: "Shared Blob"}
b3r := b3.BlobRef()
b1.MustUpload(t, ns1)
want("ns1", b1r)
want("ns2")
want("master", b1r)
b2.MustUpload(t, ns2)
want("ns1", b1r)
want("ns2", b2r)
want("master", b1r, b2r)
b3.MustUpload(t, ns2)
want("ns1", b1r)
want("ns2", b2r, b3r)
want("master", b1r, b2r, b3r)
b3.MustUpload(t, ns1)
want("ns1", b1r, b3r)
want("ns2", b2r, b3r)
want("master", b1r, b2r, b3r)
if _, _, err := ns2.FetchStreaming(b1r); err == nil {
t.Errorf("b1 shouldn't be accessible via ns2")
}
}
func (s *Storage) StreamBlobs(ctx *context.Context, dest chan<- blobserver.BlobAndToken, contToken string) error {
// for this impl, contToken is >= blobref.String()
defer close(dest)
s.mu.RLock()
defer s.mu.RUnlock()
sorted := make([]blob.Ref, 0, len(s.m))
for br := range s.m {
sorted = append(sorted, br)
}
sort.Sort(blob.ByRef(sorted))
for _, br := range sorted {
if br.String() < contToken {
continue
}
select {
case <-ctx.Done():
return context.ErrCanceled
case dest <- blobserver.BlobAndToken{
Blob: blob.NewBlob(br, uint32(len(s.m[br])), func() types.ReadSeekCloser {
return blob.NewLazyReadSeekCloser(s, br)
}),
Token: br.String(),
}:
}
}
return nil
}
func (s *Storage) EnumerateBlobs(ctx *context.Context, dest chan<- blob.SizedRef, after string, limit int) error {
defer close(dest)
s.mu.RLock()
defer s.mu.RUnlock()
// TODO(bradfitz): care about keeping this sorted like we used
// to? I think it was more expensive than it was worth before,
// since maintaining it was more costly than how often it was
// used. But perhaps it'd make sense to maintain it lazily:
// construct it on EnumerateBlobs but invalidate it everywhere
// else. Probably doesn't matter much.
sorted := make([]blob.Ref, 0, len(s.m))
for br := range s.m {
sorted = append(sorted, br)
}
sort.Sort(blob.ByRef(sorted))
n := 0
for _, br := range sorted {
if after != "" && br.String() <= after {
continue
}
select {
case dest <- blob.SizedRef{br, uint32(len(s.m[br]))}:
case <-ctx.Done():
return context.ErrCanceled
}
n++
if limit > 0 && n == limit {
break
}
}
return nil
}
// Tests the continue token on recent permanodes, notably when the
// page limit truncates in the middle of a bunch of permanodes with the
// same modtime.
func TestQueryRecentPermanodes_Continue(t *testing.T) {
testQueryTypes(t, memIndexTypes, func(qt *queryTest) {
id := qt.id
var blobs []blob.Ref
for i := 1; i <= 4; i++ {
pn := id.NewPlannedPermanode(fmt.Sprint(i))
blobs = append(blobs, pn)
t.Logf("permanode %d is %v", i, pn)
id.SetAttribute_NoTimeMove(pn, "foo", "bar")
}
sort.Sort(blob.ByRef(blobs))
for i, br := range blobs {
t.Logf("Sorted %d = %v", i, br)
}
handler := qt.Handler()
contToken := ""
tests := [][]blob.Ref{
[]blob.Ref{blobs[3], blobs[2]},
[]blob.Ref{blobs[1], blobs[0]},
[]blob.Ref{},
}
for i, wantBlobs := range tests {
req := &SearchQuery{
Constraint: &Constraint{
Permanode: &PermanodeConstraint{},
},
Limit: 2,
Sort: UnspecifiedSort,
Continue: contToken,
}
res, err := handler.Query(req)
if err != nil {
qt.t.Fatalf("Error on query %d: %v", i+1, err)
}
t.Logf("Query %d/%d: continue = %q", i+1, len(tests), res.Continue)
for i, sb := range res.Blobs {
t.Logf(" res[%d]: %v", i, sb.Blob)
}
var want []*SearchResultBlob
for _, br := range wantBlobs {
want = append(want, &SearchResultBlob{Blob: br})
}
if !reflect.DeepEqual(res.Blobs, want) {
gotj, wantj := prettyJSON(res.Blobs), prettyJSON(want)
t.Fatalf("Query %d: Got blobs:\n%s\nWant:\n%s\n", i+1, gotj, wantj)
}
contToken = res.Continue
haveToken := contToken != ""
wantHaveToken := (i + 1) < len(tests)
if haveToken != wantHaveToken {
t.Fatalf("Query %d: token = %q; want token = %v", i+1, contToken, wantHaveToken)
}
}
})
}
func (sh *SyncHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if req.Method == "POST" {
if req.FormValue("mode") == "validate" {
token := req.FormValue("token")
if xsrftoken.Valid(token, auth.ProcessRandom(), "user", "runFullValidate") {
sh.startFullValidation()
http.Redirect(rw, req, "./", http.StatusFound)
}
}
http.Error(rw, "Bad POST request", http.StatusBadRequest)
return
}
// TODO: remove this lock and instead just call currentStatus,
// and transition to using that here.
sh.mu.Lock()
defer sh.mu.Unlock()
f := func(p string, a ...interface{}) {
fmt.Fprintf(rw, p, a...)
}
now := time.Now()
f("<h1>Sync Status (for %s to %s)</h1>", sh.fromName, sh.toName)
f("<p><b>Current status: </b>%s</p>", html.EscapeString(sh.status))
if sh.idle {
return
}
f("<h2>Stats:</h2><ul>")
f("<li>Source: %s</li>", html.EscapeString(storageDesc(sh.from)))
f("<li>Target: %s</li>", html.EscapeString(storageDesc(sh.to)))
f("<li>Blobs synced: %d</li>", sh.totalCopies)
f("<li>Bytes synced: %d</li>", sh.totalCopyBytes)
f("<li>Blobs yet to copy: %d</li>", len(sh.needCopy))
f("<li>Bytes yet to copy: %d</li>", sh.bytesRemain)
if !sh.recentCopyTime.IsZero() {
f("<li>Most recent copy: %s (%v ago)</li>", sh.recentCopyTime.Format(time.RFC3339), now.Sub(sh.recentCopyTime))
}
clarification := ""
if len(sh.needCopy) == 0 && sh.totalErrors > 0 {
clarification = "(all since resolved)"
}
f("<li>Previous copy errors: %d %s</li>", sh.totalErrors, clarification)
f("</ul>")
f("<h2>Validation</h2>")
if len(sh.vshards) == 0 {
f("Validation disabled")
token := xsrftoken.Generate(auth.ProcessRandom(), "user", "runFullValidate")
f("<form method='POST'><input type='hidden' name='mode' value='validate'><input type='hidden' name='token' value='%s'><input type='submit' value='Start validation'></form>", token)
} else {
f("<p>Background scan of source and destination to ensure that the destination has everything the source does, or is at least enqueued to sync.</p>")
f("<ul>")
f("<li>Shards complete: %d/%d (%.1f%%)</li>",
sh.vshardDone,
len(sh.vshards),
100*float64(sh.vshardDone)/float64(len(sh.vshards)))
f("<li>Source blobs seen: %d</li>", sh.vsrcCount)
f("<li>Source bytes seen: %d</li>", sh.vsrcBytes)
f("<li>Dest blobs seen: %d</li>", sh.vdestCount)
f("<li>Dest bytes seen: %d</li>", sh.vdestBytes)
f("<li>Blobs found missing + fixed: %d</li>", sh.vmissing)
if len(sh.vshardErrs) > 0 {
f("<li>Validation errors: %s</li>", sh.vshardErrs)
}
f("</ul>")
}
if len(sh.copying) > 0 {
f("<h2>Currently Copying</h2><ul>")
copying := make([]blob.Ref, 0, len(sh.copying))
for br := range sh.copying {
copying = append(copying, br)
}
sort.Sort(blob.ByRef(copying))
for _, br := range copying {
f("<li>%s</li>\n", sh.copying[br])
}
f("</ul>")
}
recentErrors := make([]blob.Ref, 0, len(sh.recentErrors))
for _, br := range sh.recentErrors {
if _, ok := sh.needCopy[br]; ok {
// Only show it in the web UI if it's still a problem. Blobs that
// have since succeeded just confused people.
recentErrors = append(recentErrors, br)
}
}
if len(recentErrors) > 0 {
f("<h2>Recent Errors</h2><p>Blobs that haven't successfully copied over yet, and their last errors:</p><ul>")
for _, br := range recentErrors {
fail := sh.lastFail[br]
f("<li>%s: %s: %s</li>\n",
br,
fail.when.Format(time.RFC3339),
html.EscapeString(fail.err.Error()))
}
f("</ul>")
}
}