示例#1
0
func Test2iScanRange(t *testing.T) {
	c.LogIgnore()
	//c.SetLogLevel(c.LogLevelDebug)
	low, high := value.NewValue("aaaa"), value.NewValue("zzzz")
	span := &datastore.Span{
		Range: &datastore.Range{
			Low:       value.Values{low},
			High:      value.Values{high},
			Inclusion: datastore.BOTH,
		},
	}
	conn := datastore.NewIndexConnection(nil)
	entrych := conn.EntryChannel()
	quitch := conn.StopChannel()

	go index.Scan("", span, false, 10000, conn)

	count := 0
loop:
	for {
		select {
		case _, ok := <-entrych:
			if !ok {
				break loop
			}
			count++
		case <-quitch:
			break loop
		}
	}
	if count != 20000 {
		t.Fatal("failed ScanRange() - ", count)
	}
}
示例#2
0
func Test2iScanEntries(t *testing.T) {
	c.LogIgnore()
	//c.SetLogLevel(c.LogLevelDebug)
	conn := datastore.NewIndexConnection(nil)
	entrych := conn.EntryChannel()
	quitch := conn.StopChannel()

	go index.ScanEntries(10000, conn)

	count := 0
loop:
	for {
		select {
		case _, ok := <-entrych:
			if !ok {
				break loop
			}
			count++

		case <-quitch:
			break loop
		}
	}
	if count != 20000 {
		t.Fatal("failed ScanEntries() - ", count)
	}
}
示例#3
0
// Helper function to perform a primary index scan on the given keyspace. Returns a map of
// all primary key names.
func doPrimaryIndexScan(t *testing.T, b datastore.Keyspace) (m map[string]bool, excp errors.Error) {
	conn := datastore.NewIndexConnection(&testingContext{t})

	m = map[string]bool{}

	nitems, excp := b.Count()
	if excp != nil {
		t.Fatalf("failed to get keyspace count")
		return
	}

	indexers, excp := b.Indexers()
	if excp != nil {
		t.Fatalf("failed to retrieve indexers")
		return
	}

	pindexes, excp := indexers[0].PrimaryIndexes()
	if excp != nil || len(pindexes) < 1 {
		t.Fatalf("failed to retrieve primary indexes")
		return
	}

	idx := pindexes[0]
	go idx.ScanEntries("", nitems, datastore.UNBOUNDED, nil, conn)
	for {
		v, ok := <-conn.EntryChannel()
		if !ok {
			// Channel closed => Scan complete
			return
		}

		m[v.PrimaryKey] = true
	}
}
示例#4
0
func (this *spanScan) RunOnce(context *Context, parent value.Value) {
	this.once.Do(func() {
		defer context.Recover()       // Recover from any panic
		defer close(this.itemChannel) // Broadcast that I have stopped
		defer this.notify()           // Notify that I have stopped

		conn := datastore.NewIndexConnection(context)
		defer notifyConn(conn) // Notify index that I have stopped

		var duration time.Duration
		timer := time.Now()
		defer context.AddPhaseTime("scan", time.Since(timer)-duration)

		go this.scan(context, conn)

		var entry *datastore.IndexEntry
		ok := true
		for ok {
			select {
			case <-this.stopChannel:
				return
			default:
			}

			select {
			case entry, ok = <-conn.EntryChannel():
				t := time.Now()

				if ok {
					cv := value.NewScopeValue(make(map[string]interface{}), parent)
					av := value.NewAnnotatedValue(cv)
					meta := map[string]interface{}{"id": entry.PrimaryKey}
					av.SetAttachment("meta", meta)

					covers := this.plan.Covers()
					if len(covers) > 0 {
						for i, c := range covers {
							if i == 0 {
								av.SetCover(c.Text(), value.NewValue(entry.PrimaryKey))
							} else {
								av.SetCover(c.Text(), entry.EntryKey[i-1])
							}
						}

						av.SetField(this.plan.Term().Alias(), av)
					}

					ok = this.sendItem(av)
				}

				duration += time.Since(t)
			case <-this.stopChannel:
				return
			}
		}
	})
}
示例#5
0
func (this *IndexJoin) processItem(item value.AnnotatedValue, context *Context) bool {
	idv, e := this.plan.IdExpr().Evaluate(item, context)
	if e != nil {
		context.Error(errors.NewEvaluationError(e, fmt.Sprintf("JOIN FOR %s", this.plan.For())))
		return false
	}

	found, foundOne := false, false

	if idv.Type() == value.STRING {
		var wg sync.WaitGroup
		defer wg.Wait()

		id := idv.Actual().(string)
		conn := datastore.NewIndexConnection(context)
		defer notifyConn(conn.StopChannel()) // Notify index that I have stopped

		wg.Add(1)
		go this.scan(id, context, conn, &wg)

		var entry *datastore.IndexEntry
		ok := true
		for ok {
			select {
			case <-this.stopChannel:
				return false
			default:
			}

			select {
			case entry, ok = <-conn.EntryChannel():
				t := time.Now()

				if ok {
					foundOne, ok = this.joinEntry(item, entry, context)
					found = found || foundOne
				}

				this.duration += time.Since(t)
			case <-this.stopChannel:
				return false
			}
		}
	}

	return found || !this.plan.Outer() || this.sendItem(item)
}
示例#6
0
func (this *IndexNest) processItem(item value.AnnotatedValue, context *Context) bool {
	idv, e := this.plan.IdExpr().Evaluate(item, context)
	if e != nil {
		context.Error(errors.NewEvaluationError(e, fmt.Sprintf("NEST FOR %s", this.plan.For())))
		return false
	}

	var entry *datastore.IndexEntry
	entries := _INDEX_ENTRY_POOL.Get()
	defer _INDEX_ENTRY_POOL.Put(entries)

	if idv.Type() == value.STRING {
		var wg sync.WaitGroup
		defer wg.Wait()

		id := idv.Actual().(string)
		conn := datastore.NewIndexConnection(context)
		defer notifyConn(conn) // Notify index that I have stopped

		wg.Add(1)
		go this.scan(id, context, conn, &wg)

		ok := true
		for ok {
			select {
			case <-this.stopChannel:
				return false
			default:
			}

			select {
			case entry, ok = <-conn.EntryChannel():
				if ok {
					entries = append(entries, entry)
				}
			case <-this.stopChannel:
				return false
			}
		}
	}

	return this.nestEntries(item, entries, context)
}
示例#7
0
func (this *PrimaryScan) newIndexConnection(context *Context) *datastore.IndexConnection {
	var conn *datastore.IndexConnection

	// Use keyspace count to create a sized index connection
	keyspace := this.plan.Keyspace()
	size, err := keyspace.Count()
	if err == nil {
		if size <= 0 {
			size = 1
		}

		conn, err = datastore.NewSizedIndexConnection(size, context)
		conn.SetPrimary()
	}

	// Use non-sized API and log error
	if err != nil {
		conn = datastore.NewIndexConnection(context)
		conn.SetPrimary()
		logging.Errorp("PrimaryScan.newIndexConnection ", logging.Pair{"error", err})
	}

	return conn
}
示例#8
0
// Helper function to scan the primary index of given keyspace with given span
func doIndexScan(t *testing.T, b datastore.Keyspace, span *datastore.Span) (
	e []*datastore.IndexEntry, excp errors.Error) {
	conn := datastore.NewIndexConnection(&testingContext{t})
	e = []*datastore.IndexEntry{}

	nitems, excp := b.Count()
	if excp != nil {
		t.Fatalf("failed to get keyspace count")
		return
	}

	indexers, excp := b.Indexers()
	if excp != nil {
		t.Fatalf("failed to retrieve indexers")
		return
	}

	idx, excp := indexers[0].IndexByName("#primary")
	if excp != nil {
		t.Fatalf("failed to retrieve primary index")
		return
	}

	go idx.Scan("", span, false, nitems, datastore.UNBOUNDED, nil, conn)

	for {
		entry, ok := <-conn.EntryChannel()
		if !ok {
			return
		}

		e = append(e, entry)
	}

	return
}
示例#9
0
func TestServer(t *testing.T) {

	logger, _ := log_resolver.NewLogger("golog")
	if logger == nil {
		t.Fatalf("Invalid logger")
	}

	logging.SetLogger(logger)

	site, err := NewDatastore(TEST_URL)
	if err != nil {
		t.Skipf("SKIPPING TEST: %v", err)
	}

	namespaceNames, err := site.NamespaceNames()
	if err != nil {
		t.Fatalf("Failed to get Namespace names . error %v", err)
	}

	fmt.Printf("Namespaces in this instance %v", namespaceNames)

	namespace, err := site.NamespaceByName("default")
	if err != nil {
		t.Fatalf("Namespace default not found, error %v", err)
	}

	keyspaceNames, err := namespace.KeyspaceNames()
	if err != nil {
		t.Fatalf(" Cannot fetch keyspaces names. error %v", err)
	}

	fmt.Printf("Keyspaces in this namespace %v", keyspaceNames)

	//connect to beer-sample
	ks, err := namespace.KeyspaceByName("beer-sample")
	if err != nil {
		t.Fatalf(" Cannot connect to beer-sample. Error %v", err)
		return
	}

	indexer, err := ks.Indexer(datastore.VIEW)
	if err != nil {
		fmt.Printf("No indexers found")
		return
	}

	// try create a primary index
	index, err := indexer.CreatePrimaryIndex("", "#primary", nil)
	if err != nil {
		// keep going. maybe index already exists
		fmt.Printf(" Cannot create a primary index on bucket. Error %v", err)
	} else {

		fmt.Printf("primary index created %v", index)
	}

	pair, errs := ks.Fetch([]string{"357", "aass_brewery"})
	if errs != nil {
		t.Fatalf(" Cannot fetch keys errors %v", errs)

	}

	fmt.Printf("Keys fetched %v", pair)
	insertKey := datastore.Pair{Key: "testBeerKey", Value: value.NewValue(("This is a random test key-value"))}

	_, err = ks.Insert([]datastore.Pair{insertKey})
	if err != nil {
		t.Fatalf("Cannot insert key %v", insertKey)
	}

	deleted, err := ks.Delete([]string{insertKey.Key})
	if err != nil || (len(deleted) != 1 && deleted[0] != insertKey.Key) {
		t.Fatalf("Failed to delete %v", err)
	}

	pi, err := indexer.PrimaryIndexes()
	if err != nil || len(pi) < 1 {
		fmt.Printf("No primary index found")
		return
	}

	//fmt.Printf(" got primary index %s", pi.name)
	conn := datastore.NewIndexConnection(nil)
	go pi[0].ScanEntries("", math.MaxInt64, datastore.UNBOUNDED, nil, conn)

	var entry *datastore.IndexEntry

	ok := true
	for ok {

		select {
		case entry, ok = <-conn.EntryChannel():
			if ok {
				fmt.Printf("\n primary key %v", entry.PrimaryKey)
			}
		}
	}
}
示例#10
0
func TestFile(t *testing.T) {
	store, err := NewDatastore("../../test/filestore/json")
	if err != nil {
		t.Fatalf("failed to create store: %v", err)
	}

	namespaceIds, err := store.NamespaceIds()
	if err != nil {
		t.Errorf("failed to get namespace ids: %v", err)
	}

	if len(namespaceIds) != 1 || namespaceIds[0] != "default" {
		t.Errorf("expected 1 namespace id'd default")
	}

	namespace, err := store.NamespaceById("default")
	if err != nil {
		t.Errorf("failed to get namespace: %v", err)
	}

	namespaceNames, err := store.NamespaceNames()
	if err != nil {
		t.Errorf("failed to get namespace names: %v", err)
	}

	if len(namespaceNames) != 1 || namespaceNames[0] != "default" {
		t.Errorf("expected 1 namespace named json")
	}

	fmt.Printf("Found namespaces %v", namespaceNames)

	namespace, err = store.NamespaceByName("default")
	if err != nil {
		t.Fatalf("failed to get namespace: %v", err)
	}

	ks, err := namespace.KeyspaceIds()
	if err != nil {
		t.Errorf("failed to get keyspace ids: %v", err)
	}

	fmt.Printf("Keyspace ids %v", ks)

	keyspace, err := namespace.KeyspaceById("contacts")
	if err != nil {
		t.Errorf("failed to get keyspace by id: contacts")
	}

	_, err = namespace.KeyspaceNames()
	if err != nil {
		t.Errorf("failed to get keyspace names: %v", err)
	}

	keyspace, err = namespace.KeyspaceByName("contacts")
	if err != nil {
		t.Fatalf("failed to get keyspace by name: contacts")
	}

	indexers, err := keyspace.Indexers()
	if err != nil {
		t.Errorf("failed to get indexers")
	}

	indexes, err := indexers[0].Indexes()
	if err != nil {
		t.Errorf("failed to get indexes")
	}

	if len(indexes) < 1 {
		t.Errorf("Expected at least 1 index for keyspace")
	}

	pindexes, err := indexers[0].PrimaryIndexes()
	if err != nil {
		t.Errorf("failed to get primary indexes")
	}

	if len(pindexes) < 1 {
		t.Errorf("Expected at least 1 primary index for keyspace")
	}

	index := pindexes[0]

	context := &testingContext{t}
	conn := datastore.NewIndexConnection(context)

	go index.ScanEntries("", math.MaxInt64, datastore.UNBOUNDED, nil, conn)

	ok := true
	for ok {
		entry, ok := <-conn.EntryChannel()
		if ok {
			fmt.Printf("\nScanned %s", entry.PrimaryKey)
		} else {
			break
		}
	}

	freds, errs := keyspace.Fetch([]string{"fred"})
	if errs != nil || len(freds) == 0 {
		t.Errorf("failed to fetch fred: %v", errs)
	}

	// DML test cases

	fred := freds[0].Value
	var dmlKey datastore.Pair
	dmlKey.Key = "fred2"
	dmlKey.Value = fred

	_, err = keyspace.Insert([]datastore.Pair{dmlKey})
	if err != nil {
		t.Errorf("failed to insert fred2: %v", err)
	}

	_, err = keyspace.Update([]datastore.Pair{dmlKey})
	if err != nil {
		t.Errorf("failed to insert fred2: %v", err)
	}

	_, err = keyspace.Upsert([]datastore.Pair{dmlKey})
	if err != nil {
		t.Errorf("failed to insert fred2: %v", err)
	}

	dmlKey.Key = "fred3"
	_, err = keyspace.Upsert([]datastore.Pair{dmlKey})
	if err != nil {
		t.Errorf("failed to insert fred2: %v", err)
	}

	// negative cases
	_, err = keyspace.Insert([]datastore.Pair{dmlKey})
	if err == nil {
		t.Errorf("Insert should not have succeeded for fred2")
	}

	// delete all the freds
	deleted, err := keyspace.Delete([]string{"fred2", "fred3"})
	if err != nil && len(deleted) != 2 {
		fmt.Printf("Warning: Failed to delete. Error %v", err)
	}

	_, err = keyspace.Update([]datastore.Pair{dmlKey})
	if err == nil {
		t.Errorf("Update should have failed. Key fred3 doesn't exist")
	}

	// finally upsert the key. this should work
	_, err = keyspace.Upsert([]datastore.Pair{dmlKey})
	if err != nil {
		t.Errorf("failed to insert fred2: %v", err)
	}

	// some deletes should fail
	deleted, err = keyspace.Delete([]string{"fred2", "fred3"})
	if len(deleted) != 1 && deleted[0] != "fred2" {
		t.Errorf("failed to delete fred2: %v, #deleted=%d", deleted, len(deleted))
	}

}
示例#11
0
文件: fetch.go 项目: maniktaneja/perf
func FetchDocs(serverURL string, bucketName string) map[string]interface{} {

	logger, _ := log_resolver.NewLogger("golog")
	if logger == nil {
		log.Fatalf("Invalid logger")
	}

	logging.SetLogger(logger)
	site, err := couchbase.NewDatastore(serverURL)
	if err != nil {
		log.Fatalf("Cannot create datastore %v", err)
	}

	namespace, err := site.NamespaceByName("default")
	if err != nil {
		log.Fatalf("Namespace default not found, error %v", err)
	}

	ks, err := namespace.KeyspaceByName(bucketName)
	if err != nil {
		log.Fatalf(" Cannot connect to %s. Error %v", bucketName, err)
	}

	indexer, err := ks.Indexer(datastore.VIEW)
	if err != nil {
		log.Fatalf("No view indexer found %v", err)
	}

	// try create a primary index
	index, err := indexer.CreatePrimaryIndex("", "#primary", nil)
	if err != nil {
		// keep going. maybe index already exists
		log.Printf(" Cannot create a primary index on bucket. Error %v", err)
		pi, err := indexer.PrimaryIndexes()
		if err != nil || len(pi) < 1 {
			log.Fatalf("No primary index found")
		}
		index = pi[0]
	} else {
		log.Printf("primary index created %v", index)
	}

	conn := datastore.NewIndexConnection(nil)
	go index.ScanEntries("", math.MaxInt64, datastore.UNBOUNDED, nil, conn)

	var entry *datastore.IndexEntry
	var fetchKeys = make([]string, 0, 1000)

	ok := true
	for ok {

		select {
		case entry, ok = <-conn.EntryChannel():
			if ok {
				fetchKeys = append(fetchKeys, entry.PrimaryKey)
			}
		}
	}

	//fetch all the keys
	pairs, errs := ks.Fetch(fetchKeys)
	if errs != nil {
		log.Fatalf(" Failed to fetch keys %v", errs)
	}

	var keyMap = make(map[string]interface{})
	for _, value := range pairs {
		keyMap[value.Key] = value.Value.Actual()
	}

	log.Printf("Got %v docs", len(keyMap))

	return keyMap
}