Esempio n. 1
0
// KeysInRange returns a range of type-specific key components spanning (TkBeg, TkEnd).
func (db *BigTable) KeysInRange(ctx storage.Context, TkBeg, TkEnd storage.TKey) ([]storage.TKey, error) {
	if db == nil {
		return nil, fmt.Errorf("Can't call KeysInRange() on nil BigTable")
	}
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in KeysInRange()")
	}

	tKeys := make([]storage.TKey, 0)

	unvKeyBeg, _, err := ctx.SplitKey(TkBeg)
	if err != nil {
		dvid.Errorf("Error in KeysInRange(): %v\n", err)
	}

	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		dvid.Errorf("Error in KeysInRange(): %v\n", err)
	}

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))

	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		if len(r[familyName]) == 0 {
			dvid.Errorf("Error in KeysInRange(): row has no columns")
			return false
		}

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in KeysInRange(): %v\n", err)
			return false
		}

		verKeyRow, err := decodeKey(r[familyName][0].Column)
		if err != nil {
			dvid.Errorf("Error in KeysInRange(): %v\n", err)
			return false
		}

		fullKey := storage.MergeKey(unvKeyRow, verKeyRow)
		tkey, err := storage.TKeyFromKey(fullKey)
		if err != nil {
			dvid.Errorf("Error in KeysInRange(): %v\n", err)
			return false
		}

		tKeys = append(tKeys, tkey)

		return true // keep going
	}, api.RowFilter(api.StripValueFilter()))

	return tKeys, err
}
Esempio n. 2
0
func doRead(ctx context.Context, args ...string) {
	if len(args) < 1 {
		log.Fatalf("usage: cbt read <table> [args ...]")
	}
	tbl := getClient().Open(args[0])

	parsed := make(map[string]string)
	for _, arg := range args[1:] {
		i := strings.Index(arg, "=")
		if i < 0 {
			log.Fatalf("Bad arg %q", arg)
		}
		key, val := arg[:i], arg[i+1:]
		switch key {
		default:
			log.Fatalf("Unknown arg key %q", key)
		case "limit":
			// Be nicer; we used to support this, but renamed it to "end".
			log.Fatalf("Unknown arg key %q; did you mean %q?", key, "end")
		case "start", "end", "prefix", "count":
			parsed[key] = val
		}
	}
	if (parsed["start"] != "" || parsed["end"] != "") && parsed["prefix"] != "" {
		log.Fatal(`"start"/"end" may not be mixed with "prefix"`)
	}

	var rr bigtable.RowRange
	if start, end := parsed["start"], parsed["end"]; end != "" {
		rr = bigtable.NewRange(start, end)
	} else if start != "" {
		rr = bigtable.InfiniteRange(start)
	}
	if prefix := parsed["prefix"]; prefix != "" {
		rr = bigtable.PrefixRange(prefix)
	}

	var opts []bigtable.ReadOption
	if count := parsed["count"]; count != "" {
		n, err := strconv.ParseInt(count, 0, 64)
		if err != nil {
			log.Fatalf("Bad count %q: %v", count, err)
		}
		opts = append(opts, bigtable.LimitRows(n))
	}

	// TODO(dsymonds): Support filters.
	err := tbl.ReadRows(ctx, rr, func(r bigtable.Row) bool {
		printRow(r)
		return true
	}, opts...)
	if err != nil {
		log.Fatalf("Reading rows: %v", err)
	}
}
Esempio n. 3
0
// GetRange returns a range of values spanning (TkBeg, kEnd) keys.
func (db *BigTable) GetRange(ctx storage.Context, TkBeg, TkEnd storage.TKey) ([]*storage.TKeyValue, error) {
	if db == nil {
		return nil, fmt.Errorf("Can't call GetRange() on nil BigTable")
	}
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in GetRange()")
	}

	unvKeyBeg, _, err := ctx.SplitKey(TkBeg)
	if err != nil {
		dvid.Errorf("Error in GetRange(): %v\n", err)
	}
	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		dvid.Errorf("Error in GetRange(): %v\n", err)
	}

	tKeyValues := make([]*storage.TKeyValue, 0)

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))
	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in GetRange() decodeKey(r.Key()): %v\n", err)
			return false
		}

		// dvid.Infof("GetRange() with row key= %v", r.Key())
		for _, readItem := range r[familyName] {

			verKey, err := decodeKey(readItem.Column)
			if err != nil {
				dvid.Errorf("Error in GetRange() decodeKey(readItem.Column): %v\n", err)
				return false
			}

			fullKey := storage.MergeKey(unvKeyRow, verKey)
			// dvid.Infof("colum key= %v , timestamp = %v", verKey, readItem.Timestamp)
			tkey, err := storage.TKeyFromKey(fullKey)
			if err != nil {
				dvid.Errorf("Error in GetRange() storage.TKeyFromKey(fullKey): %v\n", err)
				return false
			}

			kv := storage.TKeyValue{tkey, readItem.Value}
			tKeyValues = append(tKeyValues, &kv)
		}

		return true // keep going
	})

	return tKeyValues, err
}
Esempio n. 4
0
// SendKeysInRange sends a range of full keys down a key channel.
func (db *BigTable) SendKeysInRange(ctx storage.Context, TkBeg, TkEnd storage.TKey, ch storage.KeyChan) error {
	if db == nil {
		return fmt.Errorf("Can't call SendKeysInRange() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in SendKeysInRange()")
	}

	unvKeyBeg, _, err := ctx.SplitKey(TkBeg)
	if err != nil {
		dvid.Errorf("Error in SendKeysInRange(): %v\n", err)
	}

	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		dvid.Errorf("Error in SendKeysInRange(): %v\n", err)
	}

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))

	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in SendKeysInRange(): %v\n", err)
			return false
		}

		//I need the versioned key to merged it with the unversioned
		// and send it throu the channel
		for _, readItem := range r[familyName] {

			verKey, err := decodeKey(readItem.Column)
			if err != nil {
				dvid.Errorf("Error in SendKeysInRange(): %v\n", err)
				return false
			}

			fullKey := storage.MergeKey(unvKeyRow, verKey)

			ch <- fullKey
		}

		return true // keep going
	}, api.RowFilter(api.StripValueFilter()))

	return err
}
Esempio n. 5
0
func (db *BigTable) metadataExists() (bool, error) {
	if db == nil {
		return false, fmt.Errorf("Can't call metadataExists() on nil BigTable")
	}

	var ctx storage.MetadataContext
	unvKeyBeg, unvKeyEnd := ctx.KeyRange()

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))

	var found bool
	err := tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {
		found = true
		return false
	})
	return found, err
}
Esempio n. 6
0
// DeleteRange removes all key-value pairs with keys in the given range.
// For all versions
func (db *BigTable) DeleteRange(ctx storage.Context, TkBeg, TkEnd storage.TKey) error {
	if db == nil {
		return fmt.Errorf("Can't call DeleteRange() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in DeleteRange()")
	}

	unvKeyBeg, _, err := ctx.SplitKey(TkBeg)
	if err != nil {
		return fmt.Errorf("Error in DeleteRange(): %v\n", err)
	}
	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		return fmt.Errorf("Error in DeleteRange(): %v\n", err)
	}

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))

	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in DeleteRange(): %v\n", err)
			return false
		}

		mut := api.NewMutation()
		mut.DeleteRow()
		err = tbl.Apply(db.ctx, encodeKey(unvKeyRow), mut)
		if err != nil {
			dvid.Errorf("Failed to delete row in DeleteRange()")
		}

		return true // keep going
	}, api.RowFilter(api.StripValueFilter()))

	return err
}
func main() {
	flag.Usage = func() {
		fmt.Printf("Usage: scantest [options] <table_name>\n\n")
		flag.PrintDefaults()
	}

	var err error
	config, err = cbtconfig.Load()
	if err != nil {
		log.Fatal(err)
	}
	config.RegisterFlags()

	flag.Parse()
	if err := config.CheckFlags(cbtconfig.ProjectAndInstanceRequired); err != nil {
		log.Fatal(err)
	}
	if config.Creds != "" {
		os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds)
	}
	if flag.NArg() != 1 {
		flag.Usage()
		os.Exit(1)
	}

	table := flag.Arg(0)

	log.Printf("Dialing connections...")
	client, err = bigtable.NewClient(context.Background(), config.Project, config.Instance)
	if err != nil {
		log.Fatalf("Making bigtable.Client: %v", err)
	}
	defer client.Close()

	log.Printf("Starting scan test... (run for %v)", *runFor)
	tbl := client.Open(table)
	sem := make(chan int, *numScans) // limit the number of requests happening at once
	var scans stats

	stopTime := time.Now().Add(*runFor)
	var wg sync.WaitGroup
	for time.Now().Before(stopTime) {
		sem <- 1
		wg.Add(1)
		go func() {
			defer wg.Done()
			defer func() { <-sem }()

			ok := true
			opStart := time.Now()
			defer func() {
				scans.Record(ok, time.Since(opStart))
			}()

			// Start at a random row key
			key := fmt.Sprintf("user%d", rand.Int63())
			limit := bigtable.LimitRows(int64(*rowLimit))
			noop := func(bigtable.Row) bool { return true }
			if err := tbl.ReadRows(context.Background(), bigtable.NewRange(key, ""), noop, limit); err != nil {
				log.Printf("Error during scan: %v", err)
				ok = false
			}
		}()
	}
	wg.Wait()

	agg := stat.NewAggregate("scans", scans.ds, scans.tries-scans.ok)
	log.Printf("Scans (%d ok / %d tries):\nscan times:\n%v\nthroughput (rows/second):\n%v",
		scans.ok, scans.tries, agg, throughputString(agg))
}
Esempio n. 8
0
// DeleteAll removes all key-value pairs for the context.  If allVersions is true,
// then all versions of the data instance are deleted.
func (db *BigTable) DeleteAll(ctx storage.Context, allVersions bool) error {
	if db == nil {
		return fmt.Errorf("Can't call DeleteAll() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in DeleteAll()")
	}

	//Row range corresponde to all keys corresponding to this data instace.
	min, max := ctx.KeyRange()
	rr := api.NewRange(encodeKey(min), encodeKey(max))

	err := tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in DeleteAll(): %v\n", err)
			return false
		}

		if allVersions {

			mut := api.NewMutation()
			mut.DeleteRow()
			err := tbl.Apply(db.ctx, encodeKey(unvKeyRow), mut)
			if err != nil {
				dvid.Errorf("Failed to delete row")
			}
		} else {

			emptyTkey := make([]byte, 0)
			_, versionToDelete, err := ctx.SplitKey(emptyTkey)
			if err != nil {
				dvid.Errorf("Error in DeleteAll(): %v\n", err)
				return false
			}

			for _, readItem := range r[familyName] {

				verKey, err := decodeKey(readItem.Column)
				if err != nil {
					dvid.Errorf("Error in DeleteAll(): %v\n", err)
					return false
				}

				if bytes.Equal(verKey, versionToDelete) {
					mut := api.NewMutation()
					mut.DeleteCellsInColumn(familyName, encodeKey(verKey))
					err := tbl.Apply(db.ctx, encodeKey(unvKeyRow), mut)
					if err != nil {
						dvid.Errorf("Failed to DeleteCellsInColumn in DeleteAll()")
					}
					return true // One I found the version I don't have to keep serching for it.
				}
			}
		}

		return true // keep going
	}, api.RowFilter(api.StripValueFilter()))

	return err
}
Esempio n. 9
0
// RawRangeQuery sends a range of full keys.  This is to be used for low-level data
// retrieval like DVID-to-DVID communication and should not be used by data type
// implementations if possible because each version's key-value pairs are sent
// without filtering by the current version and its ancestor graph.  A nil is sent
// down the channel when the range is complete.
func (db *BigTable) RawRangeQuery(kStart, kEnd storage.Key, keysOnly bool, out chan *storage.KeyValue, cancel <-chan struct{}) error {
	if db == nil {
		return fmt.Errorf("Can't call RawRangeQuery() on nil BigTable")
	}

	unvKeyBeg, verKeyBeg, err := storage.SplitKey(kStart)
	if err != nil {
		dvid.Errorf("Error in RawRangeQuery(): %v\n", err)
		return err
	}

	unvKeyEnd, verKeyEnd, err := storage.SplitKey(kEnd)
	if err != nil {
		dvid.Errorf("Error in RawRangeQuery(): %v\n", err)
		return err
	}

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))

	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {
		if cancel != nil {
			select {
			case <-cancel:
				out <- nil
				return nil
			default:
			}
		}

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in RawRangeQuery(): %v\n", err)
			return false
		}

		//I need the versioned key to merged it with the unversioned
		// and send it throu the channel
		for _, readItem := range r[familyName] {

			verKey, err := decodeKey(readItem.Column)
			if err != nil {
				dvid.Errorf("Error in RawRangeQuery(): %v\n", err)
				return false
			}

			lowerLimit := bytes.Equal(unvKeyBeg, unvKeyRow) && bytes.Compare(verKey, verKeyBeg) == -1
			upperLimit := bytes.Equal(unvKeyEnd, unvKeyRow) && bytes.Compare(verKey, verKeyEnd) >= 0

			if lowerLimit || upperLimit {
				continue
			}

			fullKey := storage.MergeKey(unvKeyRow, verKey)

			kv := storage.KeyValue{fullKey, readItem.Value}

			out <- &kv
		}

		return true // keep going
	})
	return nil
}
Esempio n. 10
0
// ProcessRange sends a range of type key-value pairs to type-specific chunk handlers,
// allowing chunk processing to be concurrent with key-value sequential reads.
// Since the chunks are typically sent during sequential read iteration, the
// receiving function can be organized as a pool of chunk handling goroutines.
// See datatype/imageblk.ProcessChunk() for an example.
func (db *BigTable) ProcessRange(ctx storage.Context, TkBeg, TkEnd storage.TKey, op *storage.ChunkOp, f storage.ChunkFunc) error {
	if db == nil {
		return fmt.Errorf("Can't call ProcessRange() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in ProcessRange()")
	}

	unvKeyBeg, verKey, err := ctx.SplitKey(TkBeg)
	if err != nil {
		dvid.Errorf("Error in ProcessRange(): %v\n", err)
	}

	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		dvid.Errorf("Error in ProcessRange(): %v\n", err)
	}

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))
	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		if len(r[familyName]) == 0 {
			dvid.Errorf("Error in KeysInRange(): row has no columns")
			return false
		}

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in ProcessRange(): %v\n", err)
			return false
		}

		verKeyRow, err := decodeKey(r[familyName][0].Column)
		if err != nil {
			dvid.Errorf("Error in ProcessRange(): %v\n", err)
			return false
		}

		fullKey := storage.MergeKey(unvKeyRow, verKeyRow)
		tkey, err := storage.TKeyFromKey(fullKey)
		if err != nil {
			dvid.Errorf("Error in ProcessRange(): %v\n", err)
			return false
		}

		if op.Wg != nil {
			op.Wg.Add(1)
		}
		value, err := getValue(r, verKey)
		if err != nil {
			dvid.Errorf("Error in ProcessRange(): %v\n", err)
			return false
		}

		tkv := storage.TKeyValue{tkey, value}
		chunk := &storage.Chunk{op, &tkv}
		if err := f(chunk); err != nil {
			dvid.Errorf("Error in ProcessRange(): %v\n", err)
			return false
		}

		return true // keep going
	})

	return err
}