func querySetup(t test.TB) (*indextest.IndexDeps, *Handler) {
idx := index.NewMemoryIndex() // string key-value pairs in memory, as if they were on disk
id := indextest.NewIndexDeps(idx)
id.Fataler = t
h := NewHandler(idx, id.SignerBlobRef)
return id, h
}
func testQueryType(t test.TB, fn func(*queryTest), itype indexType) {
defer index.SetVerboseCorpusLogging(true)
index.SetVerboseCorpusLogging(false)
idx := index.NewMemoryIndex() // string key-value pairs in memory, as if they were on disk
var err error
var corpus *index.Corpus
if itype == indexCorpusBuild {
if corpus, err = idx.KeepInMemory(); err != nil {
t.Fatal(err)
}
}
qt := &queryTest{
t: t,
id: indextest.NewIndexDeps(idx),
}
qt.id.Fataler = t
qt.Handler = func() *Handler {
h := NewHandler(idx, qt.id.SignerBlobRef)
if itype == indexCorpusScan {
if corpus, err = idx.KeepInMemory(); err != nil {
t.Fatal(err)
}
idx.PreventStorageAccessForTesting()
}
if corpus != nil {
h.SetCorpus(corpus)
}
return h
}
fn(qt)
}
// To make sure we don't regress into issue 881: i.e. a permanode with no attr
// should not lead us to call index.claimsIntfAttrValue with a nil claims argument.
func TestDescribePermNoAttr(t *testing.T) {
ix := index.NewMemoryIndex()
ctx := context.Background()
h := search.NewHandler(ix, owner)
corpus, err := ix.KeepInMemory()
if err != nil {
t.Fatal(err)
}
h.SetCorpus(corpus)
id := indextest.NewIndexDeps(ix)
br := id.NewPlannedPermanode("noattr-0")
ix.RLock()
defer ix.RUnlock()
res, err := h.Describe(ctx, &search.DescribeRequest{
BlobRef: br,
Depth: 1,
})
if err != nil {
t.Fatalf("Describe for %v failed: %v", br, err)
}
db := res.Meta[br.String()]
if db == nil {
t.Fatalf("Describe result for %v is missing", br)
}
}
func querySetup(t *testing.T) (*indextest.IndexDeps, *Handler) {
idx := index.NewMemoryIndex()
id := indextest.NewIndexDeps(idx)
id.Fataler = t
h := NewHandler(idx, id.SignerBlobRef)
return id, h
}
func BenchmarkCorpusFromStorage(b *testing.B) {
defer test.TLog(b)()
buildKvOnce.Do(func() {
kvForBenchmark = sorted.NewMemoryKeyValue()
idx := index.New(kvForBenchmark)
id := indextest.NewIndexDeps(idx)
id.Fataler = b
for i := 0; i < 10; i++ {
fileRef, _ := id.UploadFile("file.txt", fmt.Sprintf("some file %d", i), time.Unix(1382073153, 0))
pn := id.NewPlannedPermanode(fmt.Sprint(i))
id.SetAttribute(pn, "camliContent", fileRef.String())
}
})
defer index.SetVerboseCorpusLogging(true)
index.SetVerboseCorpusLogging(false)
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := index.NewCorpusFromStorage(kvForBenchmark)
if err != nil {
b.Fatal(err)
}
}
}
// should be run with -race
func TestDescribeRace(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
idx := index.NewMemoryIndex()
idxd := indextest.NewIndexDeps(idx)
idxd.Fataler = t
corpus, err := idxd.Index.KeepInMemory()
if err != nil {
t.Fatalf("error slurping index to memory: %v", err)
}
h := search.NewHandler(idx, idxd.SignerBlobRef)
h.SetCorpus(corpus)
donec := make(chan struct{})
headstart := 500
blobrefs := make([]blob.Ref, headstart)
headstartc := make(chan struct{})
go func() {
for i := 0; i < headstart*2; i++ {
nth := fmt.Sprintf("%d", i)
// No need to lock the index here. It is already done within NewPlannedPermanode,
// because it calls idxd.Index.ReceiveBlob.
pn := idxd.NewPlannedPermanode(nth)
idxd.SetAttribute(pn, "tag", nth)
if i > headstart {
continue
}
if i == headstart {
headstartc <- struct{}{}
continue
}
blobrefs[i] = pn
}
}()
<-headstartc
ctx := context.Background()
go func() {
for i := 0; i < headstart; i++ {
br := blobrefs[i]
res, err := h.Describe(ctx, &search.DescribeRequest{
BlobRef: br,
Depth: 1,
})
if err != nil {
t.Fatal(err)
}
_, ok := res.Meta[br.String()]
if !ok {
t.Errorf("permanode %v wasn't in Describe response", br)
}
}
donec <- struct{}{}
}()
<-donec
}
func testEnumerateOrder(t *testing.T,
enumFunc func(*index.Corpus, context.Context, chan<- camtypes.BlobMeta) error,
order int) {
idx := index.NewMemoryIndex()
idxd := indextest.NewIndexDeps(idx)
// permanode with no contents
foopn := idxd.NewPlannedPermanode("foo")
idxd.SetAttribute(foopn, "tag", "foo")
// permanode with file contents
// we set the time of the contents 1 second older than the modtime of foopn
fooModTime := idxd.LastTime()
fileTime := fooModTime.Add(-1 * time.Second)
fileRef, _ := idxd.UploadFile("foo.html", "<html>I am an html file.</html>", fileTime)
barpn := idxd.NewPlannedPermanode("bar")
idxd.SetAttribute(barpn, "camliContent", fileRef.String())
c, err := idxd.Index.KeepInMemory()
if err != nil {
t.Fatalf("error slurping index to memory: %v", err)
}
// check that we get a different order whether with enumerate according to
// contents time, or to permanode modtime.
var want []blob.Ref
if order == modtimeOrder {
// modtime.
want = []blob.Ref{barpn, foopn}
} else {
// creation time.
want = []blob.Ref{foopn, barpn}
}
ch := make(chan camtypes.BlobMeta, 10)
var got []camtypes.BlobMeta
errc := make(chan error, 1)
c.RLock()
go func() { errc <- enumFunc(c, context.TODO(), ch) }()
for blobMeta := range ch {
got = append(got, blobMeta)
}
err = <-errc
c.RUnlock()
if err != nil {
t.Fatalf("Could not enumerate permanodes: %v", err)
}
if len(got) != len(want) {
t.Fatalf("Saw %d permanodes in corpus; want %d", len(got), len(want))
}
for k, v := range got {
if v.Ref != want[k] {
t.Fatalf("Wrong result from enumeration. Got %v, wanted %v.", v.Ref, want[k])
}
}
}
func TestIndexingClaimMissingPubkey(t *testing.T) {
s := sorted.NewMemoryKeyValue()
idx, err := index.New(s)
if err != nil {
t.Fatal(err)
}
id := indextest.NewIndexDeps(idx)
id.Fataler = t
goodKeyFetcher := id.Index.KeyFetcher
emptyFetcher := new(test.Fetcher)
pn := id.NewPermanode()
// Prevent the index from being able to find the public key:
idx.KeyFetcher = emptyFetcher
// This previous failed to upload, since the signer's public key was
// unavailable.
claimRef := id.SetAttribute(pn, "tag", "foo")
t.Logf(" Claim is %v", claimRef)
t.Logf("Signer is %v", id.SignerBlobRef)
// Verify that populateClaim noted the missing public key blob:
{
key := fmt.Sprintf("missing|%s|%s", claimRef, id.SignerBlobRef)
if got, err := s.Get(key); got == "" || err != nil {
t.Errorf("key %q missing (err: %v); want 1", key, err)
}
}
// Now make it available again:
idx.KeyFetcher = idx.Exp_BlobSource()
if err := copyBlob(id.SignerBlobRef, idx.Exp_BlobSource().(*test.Fetcher), goodKeyFetcher); err != nil {
t.Errorf("Error copying public key to BlobSource: %v", err)
}
if err := copyBlob(id.SignerBlobRef, idx, goodKeyFetcher); err != nil {
t.Errorf("Error uploading public key to indexer: %v", err)
}
idx.Exp_AwaitReindexing(t)
// Verify that populateClaim noted the missing public key blob:
{
key := fmt.Sprintf("missing|%s|%s", claimRef, id.SignerBlobRef)
if got, err := s.Get(key); got != "" || err == nil {
t.Errorf("row %q still exists", key)
}
}
}
func testCacheSortedPermanodesRace(t *testing.T,
enumFunc func(*index.Corpus, context.Context, chan<- camtypes.BlobMeta) error) {
idx := index.NewMemoryIndex()
idxd := indextest.NewIndexDeps(idx)
idxd.Fataler = t
c, err := idxd.Index.KeepInMemory()
if err != nil {
t.Fatalf("error slurping index to memory: %v", err)
}
donec := make(chan struct{})
go func() {
for i := 0; i < 100; i++ {
nth := fmt.Sprintf("%d", i)
// No need to lock the index here. It is already done within NewPlannedPermanode,
// because it calls idxd.Index.ReceiveBlob.
pn := idxd.NewPlannedPermanode(nth)
idxd.SetAttribute(pn, "tag", nth)
}
donec <- struct{}{}
}()
go func() {
for i := 0; i < 10; i++ {
ch := make(chan camtypes.BlobMeta, 10)
errc := make(chan error, 1)
go func() {
idx.RLock()
defer idx.RUnlock()
errc <- enumFunc(c, context.TODO(), ch)
}()
for range ch {
}
err := <-errc
if err != nil {
t.Fatalf("Could not enumerate permanodes: %v", err)
}
}
donec <- struct{}{}
}()
<-donec
<-donec
}
},
"modtime": "` + addToClockOrigin(1*time.Second) + `"
}
}
}
}`),
},
// Test recent permanodes
{
name: "recent-1",
setup: func(*test.FakeIndex) index.Interface {
// Ignore the fakeindex and use the real (but in-memory) implementation,
// using IndexDeps to populate it.
idx := index.NewMemoryIndex()
id := indextest.NewIndexDeps(idx)
pn := id.NewPlannedPermanode("pn1")
id.SetAttribute(pn, "title", "Some title")
return indexAndOwner{idx, id.SignerBlobRef}
},
query: "recent",
want: parseJSON(`{
"recent": [
{"blobref": "sha1-7ca7743e38854598680d94ef85348f2c48a44513",
"modtime": "2011-11-28T01:32:37.000123456Z",
"owner": "sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"}
],
"meta": {
"sha1-7ca7743e38854598680d94ef85348f2c48a44513": {
"blobRef": "sha1-7ca7743e38854598680d94ef85348f2c48a44513",
func setupContent(rootName string) *indextest.IndexDeps {
idx := index.NewMemoryIndex()
idxd := indextest.NewIndexDeps(idx)
picNode := idxd.NewPlannedPermanode("picpn-1234") // sha1-f5e90fcc50a79caa8b22a4aa63ba92e436cab9ec
galRef := idxd.NewPlannedPermanode("gal-1234") // sha1-2bdf2053922c3dfa70b01a4827168fce1c1df691
rootRef := idxd.NewPlannedPermanode("root-abcd") // sha1-dbb3e5f28c7e01536d43ce194f3dd7b921b8460d
camp0 := idxd.NewPlannedPermanode("picpn-9876543210") // sha1-2d473e07ca760231dd82edeef4019d5b7d0ccb42
camp1 := idxd.NewPlannedPermanode("picpn-9876543211") // sha1-961b700536d5151fc1f3920955cc92767572a064
camp0f, _ := idxd.UploadFile("picfile-f00ff00f00a5.jpg", "picfile-f00ff00f00a5", time.Time{}) // sha1-01dbcb193fc789033fb2d08ed22abe7105b48640
camp1f, _ := idxd.UploadFile("picfile-f00ff00f00b6.jpg", "picfile-f00ff00f00b6", time.Time{}) // sha1-1213ec17a42cc51bdeb95ff91ac1b5fc5157740f
idxd.SetAttribute(rootRef, "camliRoot", rootName)
idxd.SetAttribute(rootRef, "camliPath:singlepic", picNode.String())
idxd.SetAttribute(picNode, "title", "picnode without a pic?")
idxd.SetAttribute(rootRef, "camliPath:camping", galRef.String())
idxd.AddAttribute(galRef, "camliMember", camp0.String())
idxd.AddAttribute(galRef, "camliMember", camp1.String())
idxd.SetAttribute(camp0, "camliContent", camp0f.String())
idxd.SetAttribute(camp1, "camliContent", camp1f.String())
publishURLTests = []publishURLTest{
// URL to a single picture permanode (returning its HTML wrapper page)
{
path: "/pics/singlepic",
subject: picNode.String(),
},
// URL to a gallery permanode (returning its HTML wrapper page)
{
path: "/pics/camping",
subject: galRef.String(),
},
// URL to a picture permanode within a gallery (following one hop, returning HTML)
{
path: "/pics/camping/-/h2d473e07ca",
subject: camp0.String(),
},
// URL to a gallery -> picture permanode -> its file
// (following two hops, returning HTML)
{
path: "/pics/camping/-/h2d473e07ca/h01dbcb193f",
subject: camp0f.String(),
},
// URL to a gallery -> picture permanode -> its file
// (following two hops, returning the file download)
{
path: "/pics/camping/-/h2d473e07ca/h01dbcb193f/=f/marshmallow.jpg",
subject: camp0f.String(),
subres: "/=f/marshmallow.jpg",
},
// URL to a gallery -> picture permanode -> its file
// (following two hops, returning the file, scaled as an image)
{
path: "/pics/camping/-/h961b700536/h1213ec17a4/=i/marshmallow.jpg?mw=200&mh=200",
subject: camp1f.String(),
subres: "/=i/marshmallow.jpg",
},
// Path to a static file in the root.
// TODO: ditch these and use content-addressable javascript + css, having
// the server digest them on start, or rather part of fileembed. This is
// a short-term hack to unblock Lindsey.
{
path: "/pics/=s/pics.js",
subject: "",
subres: "/=s/pics.js",
},
}
return idxd
}
func TestLazySortedPermanodes(t *testing.T) {
idx := index.NewMemoryIndex()
idxd := indextest.NewIndexDeps(idx)
idxd.Fataler = t
c, err := idxd.Index.KeepInMemory()
if err != nil {
t.Fatalf("error slurping index to memory: %v", err)
}
lsp := c.Exp_LSPByTime(false)
if len(lsp) != 0 {
t.Fatal("LazySortedPermanodes cache should be empty on startup")
}
pn := idxd.NewPlannedPermanode("one")
idxd.SetAttribute(pn, "tag", "one")
ctx := context.Background()
enum := func(reverse bool) {
ch := make(chan camtypes.BlobMeta, 10)
errc := make(chan error, 1)
go func() { errc <- c.EnumeratePermanodesCreated(ctx, ch, reverse) }()
for range ch {
}
err := <-errc
if err != nil {
t.Fatalf("Could not enumerate permanodes: %v", err)
}
}
enum(false)
lsp = c.Exp_LSPByTime(false)
if len(lsp) != 1 {
t.Fatalf("LazySortedPermanodes after 1st enum: got %v items, wanted 1", len(lsp))
}
lsp = c.Exp_LSPByTime(true)
if len(lsp) != 0 {
t.Fatalf("LazySortedPermanodes reversed after 1st enum: got %v items, wanted 0", len(lsp))
}
enum(true)
lsp = c.Exp_LSPByTime(false)
if len(lsp) != 1 {
t.Fatalf("LazySortedPermanodes after 2nd enum: got %v items, wanted 1", len(lsp))
}
lsp = c.Exp_LSPByTime(true)
if len(lsp) != 1 {
t.Fatalf("LazySortedPermanodes reversed after 2nd enum: got %v items, wanted 1", len(lsp))
}
pn = idxd.NewPlannedPermanode("two")
idxd.SetAttribute(pn, "tag", "two")
enum(true)
lsp = c.Exp_LSPByTime(false)
if len(lsp) != 0 {
t.Fatalf("LazySortedPermanodes after 2nd permanode: got %v items, wanted 0 because of cache invalidation", len(lsp))
}
lsp = c.Exp_LSPByTime(true)
if len(lsp) != 2 {
t.Fatalf("LazySortedPermanodes reversed after 2nd permanode: got %v items, wanted 2", len(lsp))
}
pn = idxd.NewPlannedPermanode("three")
idxd.SetAttribute(pn, "tag", "three")
enum(false)
lsp = c.Exp_LSPByTime(true)
if len(lsp) != 0 {
t.Fatalf("LazySortedPermanodes reversed after 3rd permanode: got %v items, wanted 0 because of cache invalidation", len(lsp))
}
lsp = c.Exp_LSPByTime(false)
if len(lsp) != 3 {
t.Fatalf("LazySortedPermanodes after 3rd permanode: got %v items, wanted 3", len(lsp))
}
enum(true)
lsp = c.Exp_LSPByTime(false)
if len(lsp) != 3 {
t.Fatalf("LazySortedPermanodes after 5th enum: got %v items, wanted 3", len(lsp))
}
lsp = c.Exp_LSPByTime(true)
if len(lsp) != 3 {
t.Fatalf("LazySortedPermanodes reversed after 5th enum: got %v items, wanted 3", len(lsp))
}
}
func testDeletePermanodes(t *testing.T,
enumFunc func(*index.Corpus, context.Context, chan<- camtypes.BlobMeta) error) {
idx := index.NewMemoryIndex()
idxd := indextest.NewIndexDeps(idx)
foopn := idxd.NewPlannedPermanode("foo")
idxd.SetAttribute(foopn, "tag", "foo")
barpn := idxd.NewPlannedPermanode("bar")
idxd.SetAttribute(barpn, "tag", "bar")
bazpn := idxd.NewPlannedPermanode("baz")
idxd.SetAttribute(bazpn, "tag", "baz")
idxd.Delete(barpn)
c, err := idxd.Index.KeepInMemory()
if err != nil {
t.Fatalf("error slurping index to memory: %v", err)
}
// check that we initially only find permanodes foo and baz,
// because bar is already marked as deleted.
want := []blob.Ref{foopn, bazpn}
ch := make(chan camtypes.BlobMeta, 10)
var got []camtypes.BlobMeta
errc := make(chan error, 1)
ctx := context.Background()
go func() { errc <- enumFunc(c, ctx, ch) }()
for blobMeta := range ch {
got = append(got, blobMeta)
}
err = <-errc
if err != nil {
t.Fatalf("Could not enumerate permanodes: %v", err)
}
if len(got) != len(want) {
t.Fatalf("Saw %d permanodes in corpus; want %d", len(got), len(want))
}
for _, bm := range got {
found := false
for _, perm := range want {
if bm.Ref == perm {
found = true
break
}
}
if !found {
t.Fatalf("permanode %v was not found in corpus", bm.Ref)
}
}
// now add a delete claim for permanode baz, and check that we're only left with foo permanode
delbaz := idxd.Delete(bazpn)
want = []blob.Ref{foopn}
got = got[:0]
ch = make(chan camtypes.BlobMeta, 10)
go func() { errc <- enumFunc(c, context.Background(), ch) }()
for blobMeta := range ch {
got = append(got, blobMeta)
}
err = <-errc
if err != nil {
t.Fatalf("Could not enumerate permanodes: %v", err)
}
if len(got) != len(want) {
t.Fatalf("Saw %d permanodes in corpus; want %d", len(got), len(want))
}
if got[0].Ref != foopn {
t.Fatalf("Wrong permanode found in corpus. Wanted %v, got %v", foopn, got[0].Ref)
}
// baz undeletion. delete delbaz.
idxd.Delete(delbaz)
want = []blob.Ref{foopn, bazpn}
got = got[:0]
ch = make(chan camtypes.BlobMeta, 10)
go func() { errc <- enumFunc(c, context.Background(), ch) }()
for blobMeta := range ch {
got = append(got, blobMeta)
}
err = <-errc
if err != nil {
t.Fatalf("Could not enumerate permanodes: %v", err)
}
if len(got) != len(want) {
t.Fatalf("Saw %d permanodes in corpus; want %d", len(got), len(want))
}
for _, bm := range got {
found := false
for _, perm := range want {
if bm.Ref == perm {
found = true
break
}
}
if !found {
t.Fatalf("permanode %v was not found in corpus", bm.Ref)
}
}
}
// TestGetPermanodeLocationAllocs helps us making sure we keep
// Handler.getPermanodeLocation (or equivalent), allocation-free.
func TestGetPermanodeLocationAllocs(t *testing.T) {
defer index.SetVerboseCorpusLogging(true)
index.SetVerboseCorpusLogging(false)
idx := index.NewMemoryIndex() // string key-value pairs in memory, as if they were on disk
idd := indextest.NewIndexDeps(idx)
h := NewHandler(idx, idd.SignerBlobRef)
corpus, err := idx.KeepInMemory()
if err != nil {
t.Fatal(err)
}
h.SetCorpus(corpus)
pn1 := idd.NewPermanode()
lat := 45.18
long := 5.72
idd.SetAttribute(pn1, "latitude", fmt.Sprintf("%f", lat))
idd.SetAttribute(pn1, "longitude", fmt.Sprintf("%f", long))
pnVenue := idd.NewPermanode()
idd.SetAttribute(pnVenue, "camliNodeType", "foursquare.com:venue")
idd.SetAttribute(pnVenue, "latitude", fmt.Sprintf("%f", lat))
idd.SetAttribute(pnVenue, "longitude", fmt.Sprintf("%f", long))
pnCheckin := idd.NewPermanode()
idd.SetAttribute(pnCheckin, "camliNodeType", "foursquare.com:checkin")
idd.SetAttribute(pnCheckin, "foursquareVenuePermanode", pnVenue.String())
br, _ := idd.UploadFile("photo.jpg", exifFileContentLatLong(lat, long), time.Now())
pnPhoto := idd.NewPermanode()
idd.SetAttribute(pnPhoto, "camliContent", br.String())
const (
blobParseAlloc = 1 // blob.Parse uses one alloc
// allocs permitted in different tests
latLongAttr = 0 // latitude/longitude attr lookup musn't alloc
altLocRef = blobParseAlloc
camliContentFileLoc = blobParseAlloc
)
for _, tt := range []struct {
title string
pn blob.Ref
maxAlloc int
}{
{"explicit location from attrs", pn1, latLongAttr},
{"referenced permanode location", pnCheckin, latLongAttr + altLocRef},
{"location from exif photo", pnPhoto, latLongAttr + camliContentFileLoc},
} {
n := testing.AllocsPerRun(20, func() {
loc, err := h.ExportGetPermanodeLocation(context.TODO(), tt.pn, time.Now())
if err != nil {
t.Fatal(err)
}
if loc.Latitude != lat {
t.Fatalf("wrong latitude: got %v, wanted %v", loc.Latitude, lat)
}
if loc.Longitude != long {
t.Fatalf("wrong longitude: got %v, wanted %v", loc.Longitude, long)
}
})
t.Logf("%s: %v allocations (max %v)", tt.title, n, tt.maxAlloc)
if int(n) != tt.maxAlloc {
t.Errorf("LocationHandler.PermanodeLocation should not allocate more than %d", tt.maxAlloc)
}
}
}