// Query implements Datastore.Query
func (d *Datastore) Query(q query.Query) (query.Results, error) {
results := make(chan query.Result)
walkFn := func(path string, info os.FileInfo, err error) error {
// remove ds path prefix
if strings.HasPrefix(path, d.path) {
path = path[len(d.path):]
}
if !info.IsDir() {
if strings.HasSuffix(path, ObjectKeySuffix) {
path = path[:len(path)-len(ObjectKeySuffix)]
}
key := ds.NewKey(path)
entry := query.Entry{Key: key.String(), Value: query.NotFetched}
results <- query.Result{Entry: entry}
}
return nil
}
go func() {
filepath.Walk(d.path, walkFn)
close(results)
}()
r := query.ResultsWithChan(q, results)
r = query.NaiveQueryApply(q, r)
return r, nil
}
func (fs *Datastore) Query(q query.Query) (query.Results, error) {
if (q.Prefix != "" && q.Prefix != "/") ||
len(q.Filters) > 0 ||
len(q.Orders) > 0 ||
q.Limit > 0 ||
q.Offset > 0 ||
!q.KeysOnly {
// TODO this is overly simplistic, but the only caller is
// `ipfs refs local` for now, and this gets us moving.
return nil, errors.New("flatfs only supports listing all keys in random order")
}
reschan := make(chan query.Result)
go func() {
defer close(reschan)
err := filepath.Walk(fs.path, func(path string, info os.FileInfo, err error) error {
if !info.Mode().IsRegular() || info.Name()[0] == '.' {
return nil
}
key, ok := fs.decode(info.Name())
if !ok {
log.Warning("failed to decode entry in flatfs")
return nil
}
reschan <- query.Result{
Entry: query.Entry{
Key: key.String(),
},
}
return nil
})
if err != nil {
log.Warning("walk failed: ", err)
}
}()
return query.ResultsWithChan(q, reschan), nil
}
func TestAllKeysRespectsContext(t *testing.T) {
N := 100
d := &queryTestDS{ds: ds.NewMapDatastore()}
bs, _ := newBlockStoreWithKeys(t, d, N)
started := make(chan struct{}, 1)
done := make(chan struct{}, 1)
errors := make(chan error, 100)
getKeys := func(ctx context.Context) {
started <- struct{}{}
ch, err := bs.AllKeysChan(ctx) // once without cancelling
if err != nil {
errors <- err
}
_ = collect(ch)
done <- struct{}{}
errors <- nil // a nil one to signal break
}
// Once without context, to make sure it all works
{
var results dsq.Results
var resultsmu = make(chan struct{})
resultChan := make(chan dsq.Result)
d.SetFunc(func(q dsq.Query) (dsq.Results, error) {
results = dsq.ResultsWithChan(q, resultChan)
resultsmu <- struct{}{}
return results, nil
})
go getKeys(context.Background())
// make sure it's waiting.
<-started
<-resultsmu
select {
case <-done:
t.Fatal("sync is wrong")
case <-results.Process().Closing():
t.Fatal("should not be closing")
case <-results.Process().Closed():
t.Fatal("should not be closed")
default:
}
e := dsq.Entry{Key: BlockPrefix.ChildString("foo").String()}
resultChan <- dsq.Result{Entry: e} // let it go.
close(resultChan)
<-done // should be done now.
<-results.Process().Closed() // should be closed now
// print any errors
for err := range errors {
if err == nil {
break
}
t.Error(err)
}
}
// Once with
{
var results dsq.Results
var resultsmu = make(chan struct{})
resultChan := make(chan dsq.Result)
d.SetFunc(func(q dsq.Query) (dsq.Results, error) {
results = dsq.ResultsWithChan(q, resultChan)
resultsmu <- struct{}{}
return results, nil
})
ctx, cancel := context.WithCancel(context.Background())
go getKeys(ctx)
// make sure it's waiting.
<-started
<-resultsmu
select {
case <-done:
t.Fatal("sync is wrong")
case <-results.Process().Closing():
t.Fatal("should not be closing")
case <-results.Process().Closed():
t.Fatal("should not be closed")
default:
}
cancel() // let it go.
select {
case <-done:
t.Fatal("sync is wrong")
case <-results.Process().Closed():
t.Fatal("should not be closed") // should not be closed yet.
case <-results.Process().Closing():
// should be closing now!
t.Log("closing correctly at this point.")
}
close(resultChan)
<-done // should be done now.
<-results.Process().Closed() // should be closed now
// print any errors
for err := range errors {
if err == nil {
break
}
t.Error(err)
}
}
}