Beispiel #1
0
func deleteConflict(data DataService, extnode *extensionNode, k storage.Key) error {
	store, err := getOrderedKeyValueDB(data)
	if err != nil {
		return err
	}

	// Create new node if necessary
	if extnode.newUUID == dvid.NilUUID {
		childUUID, err := manager.newVersion(extnode.oldUUID, "Version for deleting conflicts before merge", nil)
		if err != nil {
			return err
		}
		extnode.newUUID = childUUID
		childV, err := manager.versionFromUUID(childUUID)
		if err != nil {
			return err
		}
		extnode.newV = childV
	}

	// Perform the deletion.
	tk, err := storage.TKeyFromKey(k)
	if err != nil {
		return err
	}
	ctx := NewVersionedCtx(data, extnode.newV)
	return store.Delete(ctx, tk)
}
Beispiel #2
0
// GetRange returns a range of values spanning (kStart, kEnd) keys.  These key-value
// pairs will be sorted in ascending key order.  If the keys are versioned, all key-value
// pairs for the particular version will be returned.
func (db *KVAutobus) GetRange(ctx storage.Context, kStart, kEnd storage.TKey) ([]*storage.TKeyValue, error) {
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in GetRange()")
	}
	ch := make(chan errorableKV)

	// Run the range query on a potentially versioned key in a goroutine.
	go func() {
		if ctx == nil || !ctx.Versioned() {
			db.unversionedRange(ctx, kStart, kEnd, ch, false)
		} else {
			db.versionedRange(ctx.(storage.VersionedCtx), kStart, kEnd, ch, false)
		}
	}()

	// Consume the key-value pairs.
	values := []*storage.TKeyValue{}
	for {
		result := <-ch
		if result.KeyValue == nil {
			return values, nil
		}
		if result.error != nil {
			return nil, result.error
		}
		tk, err := storage.TKeyFromKey(result.KeyValue.K)
		if err != nil {
			return nil, err
		}
		tkv := storage.TKeyValue{tk, result.KeyValue.V}
		values = append(values, &tkv)
	}
}
Beispiel #3
0
// KeysInRange returns a range of type-specific key components spanning (TkBeg, TkEnd).
func (db *BigTable) KeysInRange(ctx storage.Context, TkBeg, TkEnd storage.TKey) ([]storage.TKey, error) {
	if db == nil {
		return nil, fmt.Errorf("Can't call KeysInRange() on nil BigTable")
	}
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in KeysInRange()")
	}

	tKeys := make([]storage.TKey, 0)

	unvKeyBeg, _, err := ctx.SplitKey(TkBeg)
	if err != nil {
		dvid.Errorf("Error in KeysInRange(): %v\n", err)
	}

	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		dvid.Errorf("Error in KeysInRange(): %v\n", err)
	}

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))

	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		if len(r[familyName]) == 0 {
			dvid.Errorf("Error in KeysInRange(): row has no columns")
			return false
		}

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in KeysInRange(): %v\n", err)
			return false
		}

		verKeyRow, err := decodeKey(r[familyName][0].Column)
		if err != nil {
			dvid.Errorf("Error in KeysInRange(): %v\n", err)
			return false
		}

		fullKey := storage.MergeKey(unvKeyRow, verKeyRow)
		tkey, err := storage.TKeyFromKey(fullKey)
		if err != nil {
			dvid.Errorf("Error in KeysInRange(): %v\n", err)
			return false
		}

		tKeys = append(tKeys, tkey)

		return true // keep going
	}, api.RowFilter(api.StripValueFilter()))

	return tKeys, err
}
Beispiel #4
0
// versionedRange sends a range of key-value pairs for a particular version down a channel.
func (db *KVAutobus) versionedRange(vctx storage.VersionedCtx, kStart, kEnd storage.TKey, ch chan errorableKV, keysOnly bool) {
	minKey, err := vctx.MinVersionKey(kStart)
	if err != nil {
		ch <- errorableKV{nil, err}
		return
	}
	maxKey, err := vctx.MaxVersionKey(kEnd)
	if err != nil {
		ch <- errorableKV{nil, err}
		return
	}
	maxVersionKey, err := vctx.MaxVersionKey(kStart)
	if err != nil {
		ch <- errorableKV{nil, err}
		return
	}

	kvs, err := db.getRange(vctx, minKey, maxKey)
	if err != nil {
		ch <- errorableKV{nil, err}
		return
	}
	versions := []*storage.KeyValue{}
	for _, kv := range kvs {
		// Did we pass all versions for last key read?
		if bytes.Compare(kv.K, maxVersionKey) > 0 {
			indexBytes, err := storage.TKeyFromKey(kv.K)
			if err != nil {
				ch <- errorableKV{nil, err}
				return
			}
			maxVersionKey, err = vctx.MaxVersionKey(indexBytes)
			if err != nil {
				ch <- errorableKV{nil, err}
				return
			}
			// log.Printf("->maxVersionKey %v (transmitting %d values)\n", maxVersionKey, len(values))
			sendKV(vctx, versions, ch)
			versions = []*storage.KeyValue{}
		}
		// Did we pass the final key?
		if bytes.Compare(kv.K, maxKey) > 0 {
			if len(versions) > 0 {
				sendKV(vctx, versions, ch)
			}
			break
		}
		// log.Printf("Appending value with key %v\n", itKey)
		versions = append(versions, kv)
	}
	if len(versions) >= 0 {
		sendKV(vctx, versions, ch)
	}
	ch <- errorableKV{nil, nil}
}
Beispiel #5
0
// GetRange returns a range of values spanning (TkBeg, kEnd) keys.
func (db *BigTable) GetRange(ctx storage.Context, TkBeg, TkEnd storage.TKey) ([]*storage.TKeyValue, error) {
	if db == nil {
		return nil, fmt.Errorf("Can't call GetRange() on nil BigTable")
	}
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in GetRange()")
	}

	unvKeyBeg, _, err := ctx.SplitKey(TkBeg)
	if err != nil {
		dvid.Errorf("Error in GetRange(): %v\n", err)
	}
	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		dvid.Errorf("Error in GetRange(): %v\n", err)
	}

	tKeyValues := make([]*storage.TKeyValue, 0)

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))
	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in GetRange() decodeKey(r.Key()): %v\n", err)
			return false
		}

		// dvid.Infof("GetRange() with row key= %v", r.Key())
		for _, readItem := range r[familyName] {

			verKey, err := decodeKey(readItem.Column)
			if err != nil {
				dvid.Errorf("Error in GetRange() decodeKey(readItem.Column): %v\n", err)
				return false
			}

			fullKey := storage.MergeKey(unvKeyRow, verKey)
			// dvid.Infof("colum key= %v , timestamp = %v", verKey, readItem.Timestamp)
			tkey, err := storage.TKeyFromKey(fullKey)
			if err != nil {
				dvid.Errorf("Error in GetRange() storage.TKeyFromKey(fullKey): %v\n", err)
				return false
			}

			kv := storage.TKeyValue{tkey, readItem.Value}
			tKeyValues = append(tKeyValues, &kv)
		}

		return true // keep going
	})

	return tKeyValues, err
}
Beispiel #6
0
// ProcessRange sends a range of key-value pairs to chunk handlers.  If the keys are versioned,
// only key-value pairs for kStart's version will be transmitted.  If f returns an error, the
// function is immediately terminated and returns an error.
func (db *LevelDB) ProcessRange(ctx storage.Context, kStart, kEnd storage.TKey, op *storage.ChunkOp, f storage.ChunkFunc) error {
	if db == nil {
		return fmt.Errorf("Can't call ProcessRange on nil LevelDB")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in ProcessRange()")
	}
	ch := make(chan errorableKV)
	done := make(chan struct{})
	defer close(done)

	// Run the range query on a potentially versioned key in a goroutine.
	go func() {
		if ctx == nil || !ctx.Versioned() {
			db.unversionedRange(ctx, kStart, kEnd, ch, done, false)
		} else {
			db.versionedRange(ctx.(storage.VersionedCtx), kStart, kEnd, ch, done, false)
		}
	}()

	// Consume the key-value pairs.
	for {
		result := <-ch
		if result.KeyValue == nil {
			return nil
		}
		if result.error != nil {
			return result.error
		}
		if op != nil && op.Wg != nil {
			op.Wg.Add(1)
		}
		tk, err := storage.TKeyFromKey(result.KeyValue.K)
		if err != nil {
			return err
		}
		tkv := storage.TKeyValue{tk, result.KeyValue.V}
		chunk := &storage.Chunk{op, &tkv}
		if err := f(chunk); err != nil {
			return err
		}
	}
}
Beispiel #7
0
func TestTileKey(t *testing.T) {
	datastore.OpenTest()
	defer datastore.CloseTest()

	uuid, _ := initTestRepo()
	server.CreateTestInstance(t, uuid, "imagetile", "tiles", dvid.Config{})

	keyURL := fmt.Sprintf("%snode/%s/tiles/tilekey/xy/0/1_2_3", server.WebAPIPath, uuid)
	respStr := server.TestHTTP(t, "GET", keyURL, nil)
	keyResp := struct {
		Key string `json:"key"`
	}{}
	if err := json.Unmarshal(respStr, &keyResp); err != nil {
		t.Fatalf("Couldn't parse JSON response to tilekey request (%v):\n%s\n", err, keyResp)
	}
	kb := make([]byte, hex.DecodedLen(len(keyResp.Key)))
	_, err := hex.Decode(kb, []byte(keyResp.Key))
	if err != nil {
		t.Fatalf("Couldn't parse return hex key: %s", keyResp.Key)
	}

	// Decipher TKey portion to make sure it's correct.
	key := storage.Key(kb)
	tk, err := storage.TKeyFromKey(key)
	if err != nil {
		t.Fatalf("Couldn't get TKey from returned key (%v): %x", err, kb)
	}
	tile, plane, scale, err := DecodeTKey(tk)
	if err != nil {
		t.Fatalf("Bad decode of TKey (%v): %x", err, tk)
	}
	expectTile := dvid.ChunkPoint3d{1, 2, 3}
	if tile != expectTile {
		t.Errorf("Expected tile %v, got %v\n", expectTile, tile)
	}
	if !plane.Equals(dvid.XY) {
		t.Errorf("Expected plane to be XY, got %v\n", plane)
	}
	if scale != 0 {
		t.Errorf("Expected scale to be 0, got %d\n", scale)
	}
}
Beispiel #8
0
// KeysInRange returns a range of present keys spanning (kStart, kEnd).  Values
// associated with the keys are not read.   If the keys are versioned, only keys
// in the ancestor path of the current context's version will be returned.
func (db *LevelDB) KeysInRange(ctx storage.Context, kStart, kEnd storage.TKey) ([]storage.TKey, error) {
	if db == nil {
		return nil, fmt.Errorf("Can't call KeysInRange on nil LevelDB")
	}
	if ctx == nil {
		return nil, fmt.Errorf("Received nil context in KeysInRange()")
	}
	ch := make(chan errorableKV)
	done := make(chan struct{})
	defer close(done)

	// Run the range query on a potentially versioned key in a goroutine.
	go func() {
		if !ctx.Versioned() {
			db.unversionedRange(ctx, kStart, kEnd, ch, done, true)
		} else {
			db.versionedRange(ctx.(storage.VersionedCtx), kStart, kEnd, ch, done, true)
		}
	}()

	// Consume the keys.
	values := []storage.TKey{}
	for {
		result := <-ch
		if result.KeyValue == nil {
			return values, nil
		}
		if result.error != nil {
			return nil, result.error
		}
		tk, err := storage.TKeyFromKey(result.KeyValue.K)
		if err != nil {
			return nil, err
		}
		values = append(values, tk)
	}
}
Beispiel #9
0
// copyData copies all key-value pairs pertinent to the given data instance d2.  If d2 is nil,
// the destination data instance is d1, useful for migration of data to a new store.
// Each datatype can implement filters that can restrict the transmitted key-value pairs
// based on the given FilterSpec.
func copyData(oldKV, newKV storage.OrderedKeyValueDB, d1, d2 dvid.Data, uuid dvid.UUID, f storage.Filter, flatten bool) error {
	// Get data context for this UUID.
	v, err := VersionFromUUID(uuid)
	if err != nil {
		return err
	}
	srcCtx := NewVersionedCtx(d1, v)
	var dstCtx *VersionedCtx
	if d2 == nil {
		d2 = d1
		dstCtx = srcCtx
	} else {
		dstCtx = NewVersionedCtx(d2, v)
	}

	// Send this instance's key-value pairs
	var wg sync.WaitGroup
	wg.Add(1)

	stats := new(txStats)
	stats.lastTime = time.Now()

	var kvTotal, kvSent int
	var bytesTotal, bytesSent uint64
	keysOnly := false
	if flatten {
		// Start goroutine to receive flattened key-value pairs and store them.
		ch := make(chan *storage.TKeyValue, 1000)
		go func() {
			for {
				tkv := <-ch
				if tkv == nil {
					wg.Done()
					dvid.Infof("Copied %d %q key-value pairs (%s, out of %d kv pairs, %s) [flattened]\n",
						kvSent, d1.DataName(), humanize.Bytes(bytesSent), kvTotal, humanize.Bytes(bytesTotal))
					stats.printStats()
					return
				}
				kvTotal++
				curBytes := uint64(len(tkv.V) + len(tkv.K))
				bytesTotal += curBytes
				if f != nil {
					skip, err := f.Check(tkv)
					if err != nil {
						dvid.Errorf("problem applying filter on data %q: %v\n", d1.DataName(), err)
						continue
					}
					if skip {
						continue
					}
				}
				kvSent++
				bytesSent += curBytes
				if err := newKV.Put(dstCtx, tkv.K, tkv.V); err != nil {
					dvid.Errorf("can't put k/v pair to destination instance %q: %v\n", d2.DataName(), err)
				}
				stats.addKV(tkv.K, tkv.V)
			}
		}()

		begKey, endKey := srcCtx.TKeyRange()
		err := oldKV.ProcessRange(srcCtx, begKey, endKey, &storage.ChunkOp{}, func(c *storage.Chunk) error {
			if c == nil {
				return fmt.Errorf("received nil chunk in flatten push for data %s", d1.DataName())
			}
			ch <- c.TKeyValue
			return nil
		})
		ch <- nil
		if err != nil {
			return fmt.Errorf("error in flatten push for data %q: %v", d1.DataName(), err)
		}
	} else {
		// Start goroutine to receive all key-value pairs and store them.
		ch := make(chan *storage.KeyValue, 1000)
		go func() {
			for {
				kv := <-ch
				if kv == nil {
					wg.Done()
					dvid.Infof("Sent %d %q key-value pairs (%s, out of %d kv pairs, %s)\n",
						kvSent, d1.DataName(), humanize.Bytes(bytesSent), kvTotal, humanize.Bytes(bytesTotal))
					stats.printStats()
					return
				}
				tkey, err := storage.TKeyFromKey(kv.K)
				if err != nil {
					dvid.Errorf("couldn't get %q TKey from Key %v: %v\n", d1.DataName(), kv.K, err)
					continue
				}

				kvTotal++
				curBytes := uint64(len(kv.V) + len(kv.K))
				bytesTotal += curBytes
				if f != nil {
					skip, err := f.Check(&storage.TKeyValue{K: tkey, V: kv.V})
					if err != nil {
						dvid.Errorf("problem applying filter on data %q: %v\n", d1.DataName(), err)
						continue
					}
					if skip {
						continue
					}
				}
				kvSent++
				bytesSent += curBytes
				if dstCtx != nil {
					err := dstCtx.UpdateInstance(kv.K)
					if err != nil {
						dvid.Errorf("can't update raw key to new data instance %q: %v\n", d2.DataName(), err)
					}
				}
				if err := newKV.RawPut(kv.K, kv.V); err != nil {
					dvid.Errorf("can't put k/v pair to destination instance %q: %v\n", d2.DataName(), err)
				}
				stats.addKV(kv.K, kv.V)
			}
		}()

		begKey, endKey := srcCtx.KeyRange()
		if err = oldKV.RawRangeQuery(begKey, endKey, keysOnly, ch, nil); err != nil {
			return fmt.Errorf("push voxels %q range query: %v", d1.DataName(), err)
		}
	}
	wg.Wait()
	return nil
}
Beispiel #10
0
// versionedRange sends a range of key-value pairs for a particular version down a channel.
func (db *LevelDB) versionedRange(vctx storage.VersionedCtx, begTKey, endTKey storage.TKey, ch chan errorableKV, done <-chan struct{}, keysOnly bool) {
	dvid.StartCgo()
	ro := levigo.NewReadOptions()
	it := db.ldb.NewIterator(ro)
	defer func() {
		it.Close()
		dvid.StopCgo()
	}()

	minKey, err := vctx.MinVersionKey(begTKey)
	if err != nil {
		ch <- errorableKV{nil, err}
		return
	}
	maxKey, err := vctx.MaxVersionKey(endTKey)
	if err != nil {
		ch <- errorableKV{nil, err}
		return
	}

	values := []*storage.KeyValue{}
	maxVersionKey, err := vctx.MaxVersionKey(begTKey)
	if err != nil {
		ch <- errorableKV{nil, err}
		return
	}
	// log.Printf("         minKey %v\n", minKey)
	// log.Printf("         maxKey %v\n", maxKey)
	// log.Printf("  maxVersionKey %v\n", maxVersionKey)

	it.Seek(minKey)
	var itValue []byte
	for {
		select {
		case <-done: // only happens if we don't care about rest of data.
			ch <- errorableKV{nil, nil}
			return
		default:
		}
		if it.Valid() {
			if !keysOnly {
				itValue = it.Value()
				storage.StoreValueBytesRead <- len(itValue)
			}
			itKey := it.Key()
			// log.Printf("   +++valid key %v\n", itKey)
			storage.StoreKeyBytesRead <- len(itKey)

			// Did we pass all versions for last key read?
			if bytes.Compare(itKey, maxVersionKey) > 0 {
				indexBytes, err := storage.TKeyFromKey(itKey)
				if err != nil {
					ch <- errorableKV{nil, err}
					return
				}
				maxVersionKey, err = vctx.MaxVersionKey(indexBytes)
				if err != nil {
					ch <- errorableKV{nil, err}
					return
				}
				// log.Printf("->maxVersionKey %v (transmitting %d values)\n", maxVersionKey, len(values))
				sendKV(vctx, values, ch)
				values = []*storage.KeyValue{}
			}
			// Did we pass the final key?
			if bytes.Compare(itKey, maxKey) > 0 {
				if len(values) > 0 {
					sendKV(vctx, values, ch)
				}
				ch <- errorableKV{nil, nil}
				return
			}
			// log.Printf("Appending value with key %v\n", itKey)
			values = append(values, &storage.KeyValue{K: itKey, V: itValue})
			it.Next()
		} else {
			if err = it.GetError(); err != nil {
				ch <- errorableKV{nil, err}
			} else {
				sendKV(vctx, values, ch)
				ch <- errorableKV{nil, nil}
			}
			return
		}
	}
}
Beispiel #11
0
// DeleteRange removes all key-value pairs with keys in the given range.
func (db *LevelDB) DeleteRange(ctx storage.Context, kStart, kEnd storage.TKey) error {
	if db == nil {
		return fmt.Errorf("Can't call DeleteRange on nil LevelDB")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in DeleteRange()")
	}

	// For leveldb, we just iterate over keys in range and delete each one using batch.
	const BATCH_SIZE = 10000
	batch := db.NewBatch(ctx).(*goBatch)

	ch := make(chan errorableKV)
	done := make(chan struct{})
	defer close(done)

	// Run the keys-only range query in a goroutine.
	go func() {
		if ctx == nil || !ctx.Versioned() {
			db.unversionedRange(ctx, kStart, kEnd, ch, done, true)
		} else {
			db.versionedRange(ctx.(storage.VersionedCtx), kStart, kEnd, ch, done, true)
		}
	}()

	// Consume the key-value pairs.
	numKV := 0
	for {
		dvid.Infof("Waiting for result...\n")
		result := <-ch
		dvid.Infof("Got result: %v\n", result)
		if result.KeyValue == nil {
			break
		}
		if result.error != nil {
			return result.error
		}

		// The key coming down channel is not index but full key, so no need to construct key using context.
		// If versioned, write a tombstone using current version id since we don't want to delete locked ancestors.
		// If unversioned, just delete.
		tk, err := storage.TKeyFromKey(result.KeyValue.K)
		if err != nil {
			return err
		}
		batch.Delete(tk)

		if (numKV+1)%BATCH_SIZE == 0 {
			if err := batch.Commit(); err != nil {
				dvid.Criticalf("Error on batch commit of DeleteRange at key-value pair %d: %v\n", numKV, err)
				return fmt.Errorf("Error on batch commit of DeleteRange at key-value pair %d: %v\n", numKV, err)
			}
			batch = db.NewBatch(ctx).(*goBatch)
		}
		numKV++
	}
	if numKV%BATCH_SIZE != 0 {
		if err := batch.Commit(); err != nil {
			dvid.Criticalf("Error on last batch commit of DeleteRange: %v\n", err)
			return fmt.Errorf("Error on last batch commit of DeleteRange: %v\n", err)
		}
	}
	dvid.Debugf("Deleted %d key-value pairs via delete range for %s.\n", numKV, ctx)
	return nil
}
Beispiel #12
0
// DeleteConflicts removes all conflicted kv pairs for the given data instance using the priority
// established by parents.  As a side effect, newParents are modified by new children of parents.
func DeleteConflicts(uuid dvid.UUID, data DataService, oldParents, newParents []dvid.UUID) error {
	if manager == nil {
		return ErrManagerNotInitialized
	}

	// Convert UUIDs to versions + bool for whether it's a child suitable for add deletions.
	parents := make(map[dvid.VersionID]*extensionNode, len(oldParents))
	parentsV := make([]dvid.VersionID, len(oldParents))
	for i, oldUUID := range oldParents {
		oldV, err := manager.versionFromUUID(oldUUID)
		if err != nil {
			return err
		}
		parentsV[i] = oldV
		if newParents[i] != dvid.NilUUID {
			newV, err := manager.versionFromUUID(newParents[i])
			if err != nil {
				return err
			}
			parents[oldV] = &extensionNode{oldUUID, newParents[i], newV}
		} else {
			parents[oldV] = &extensionNode{oldUUID, dvid.NilUUID, 0}
		}
	}

	// Process stream of incoming kv pair for this data instance.
	baseCtx := NewVersionedCtx(data, 0)
	ch := make(chan *storage.KeyValue, 1000)
	wg := new(sync.WaitGroup)
	wg.Add(1)
	go func() {
		defer wg.Done()

		var err error
		var curV dvid.VersionID
		var curTK, batchTK storage.TKey
		kvv := kvVersions{}
		for {
			kv := <-ch
			if kv == nil {
				curTK = nil
			} else {
				curV, err = baseCtx.VersionFromKey(kv.K)
				if err != nil {
					dvid.Errorf("Can't decode key when deleting conflicts for %s", data.DataName())
					continue
				}

				// If we have a different TKey, then process the batch of versions.
				curTK, err = storage.TKeyFromKey(kv.K)
				if err != nil {
					dvid.Errorf("Error in processing kv pairs in DeleteConflicts: %v\n", err)
					continue
				}
				if batchTK == nil {
					batchTK = curTK
				}
			}
			if !bytes.Equal(curTK, batchTK) {
				// Get conflicts.
				toDelete, err := kvv.FindConflicts(parentsV)
				if err != nil {
					dvid.Errorf("Error finding conflicts: %v\n", err)
					continue
				}

				// Create new node if necessary to apply deletions, and if so, store new node.
				for v, k := range toDelete {
					if err := deleteConflict(data, parents[v], k); err != nil {
						dvid.Errorf("Unable to delete conflict: %v\n", err)
						continue
					}
				}

				// Delete the stash of kv pairs
				kvv = kvVersions{}

				batchTK = curTK
			}
			if kv == nil {
				return
			}
			kvv[curV] = kvvNode{kv: kv}
		}
	}()

	// Iterate through all k/v for this data instance.
	store, err := getOrderedKeyValueDB(data)
	if err != nil {
		return err
	}

	minKey, maxKey := baseCtx.KeyRange()
	keysOnly := true
	if err := store.RawRangeQuery(minKey, maxKey, keysOnly, ch, nil); err != nil {
		return err
	}
	wg.Wait()

	// Return the new parents which were needed for deletions.
	//newParents = make([]dvid.UUID, len(oldParents))
	for i, oldV := range parentsV {
		if parents[oldV].newUUID == dvid.NilUUID {
			newParents[i] = parents[oldV].oldUUID
		} else {
			newParents[i] = parents[oldV].newUUID
		}
	}

	return nil
}
Beispiel #13
0
// ProcessRange sends a range of type key-value pairs to type-specific chunk handlers,
// allowing chunk processing to be concurrent with key-value sequential reads.
// Since the chunks are typically sent during sequential read iteration, the
// receiving function can be organized as a pool of chunk handling goroutines.
// See datatype/imageblk.ProcessChunk() for an example.
func (db *BigTable) ProcessRange(ctx storage.Context, TkBeg, TkEnd storage.TKey, op *storage.ChunkOp, f storage.ChunkFunc) error {
	if db == nil {
		return fmt.Errorf("Can't call ProcessRange() on nil BigTable")
	}
	if ctx == nil {
		return fmt.Errorf("Received nil context in ProcessRange()")
	}

	unvKeyBeg, verKey, err := ctx.SplitKey(TkBeg)
	if err != nil {
		dvid.Errorf("Error in ProcessRange(): %v\n", err)
	}

	unvKeyEnd, _, err := ctx.SplitKey(TkEnd)
	if err != nil {
		dvid.Errorf("Error in ProcessRange(): %v\n", err)
	}

	rr := api.NewRange(encodeKey(unvKeyBeg), encodeKey(unvKeyEnd))
	err = tbl.ReadRows(db.ctx, rr, func(r api.Row) bool {

		if len(r[familyName]) == 0 {
			dvid.Errorf("Error in KeysInRange(): row has no columns")
			return false
		}

		unvKeyRow, err := decodeKey(r.Key())
		if err != nil {
			dvid.Errorf("Error in ProcessRange(): %v\n", err)
			return false
		}

		verKeyRow, err := decodeKey(r[familyName][0].Column)
		if err != nil {
			dvid.Errorf("Error in ProcessRange(): %v\n", err)
			return false
		}

		fullKey := storage.MergeKey(unvKeyRow, verKeyRow)
		tkey, err := storage.TKeyFromKey(fullKey)
		if err != nil {
			dvid.Errorf("Error in ProcessRange(): %v\n", err)
			return false
		}

		if op.Wg != nil {
			op.Wg.Add(1)
		}
		value, err := getValue(r, verKey)
		if err != nil {
			dvid.Errorf("Error in ProcessRange(): %v\n", err)
			return false
		}

		tkv := storage.TKeyValue{tkey, value}
		chunk := &storage.Chunk{op, &tkv}
		if err := f(chunk); err != nil {
			dvid.Errorf("Error in ProcessRange(): %v\n", err)
			return false
		}

		return true // keep going
	})

	return err
}
Beispiel #14
0
// PushData transfers all key-value pairs pertinent to the given data instance.
// Each datatype can implement filters that can restrict the transmitted key-value pairs
// based on the given FilterSpec.  Note that because of the generality of this function,
// a particular datatype implementation could be much more efficient when implementing
// filtering.  For example, the imageblk datatype could scan its key-values using the ROI
// to generate keys (since imageblk keys will likely be a vast superset of ROI spans),
// while this generic routine will scan every key-value pair for a data instance and
// query the ROI to see if this key is ok to send.
func PushData(d dvid.Data, p *PushSession) error {
	// We should be able to get the backing store (only ordered kv for now)
	storer, ok := d.(storage.Accessor)
	if !ok {
		return fmt.Errorf("unable to push data %q: unable to access backing store", d.DataName())
	}
	store, err := storer.GetOrderedKeyValueDB()
	if err != nil {
		return fmt.Errorf("unable to get backing store for data %q: %v\n", d.DataName(), err)
	}

	// See if this data instance implements a Send filter.
	var filter storage.Filter
	filterer, ok := d.(storage.Filterer)
	if ok {
		var err error
		filter, err = filterer.NewFilter(p.Filter)
		if err != nil {
			return err
		}
	}

	// pick any version because flatten transmit will only have one version, and all or branch transmit will
	// be looking at all versions anyway.
	if len(p.Versions) == 0 {
		return fmt.Errorf("need at least one version to send")
	}
	var v dvid.VersionID
	for v = range p.Versions {
		break
	}
	ctx := NewVersionedCtx(d, v)

	// Send the initial data instance start message
	if err := p.StartInstancePush(d); err != nil {
		return err
	}

	// Send this instance's key-value pairs
	var wg sync.WaitGroup
	wg.Add(1)

	var kvTotal, kvSent int
	var bytesTotal, bytesSent uint64
	keysOnly := false
	if p.t == rpc.TransmitFlatten {
		// Start goroutine to receive flattened key-value pairs and transmit to remote.
		ch := make(chan *storage.TKeyValue, 1000)
		go func() {
			for {
				tkv := <-ch
				if tkv == nil {
					if err := p.EndInstancePush(); err != nil {
						dvid.Errorf("Bad data %q termination: %v\n", d.DataName(), err)
					}
					wg.Done()
					dvid.Infof("Sent %d %q key-value pairs (%s, out of %d kv pairs, %s) [flattened]\n",
						kvSent, d.DataName(), humanize.Bytes(bytesSent), kvTotal, humanize.Bytes(bytesTotal))
					return
				}
				kvTotal++
				curBytes := uint64(len(tkv.V) + len(tkv.K))
				bytesTotal += curBytes
				if filter != nil {
					skip, err := filter.Check(tkv)
					if err != nil {
						dvid.Errorf("problem applying filter on data %q: %v\n", d.DataName(), err)
						continue
					}
					if skip {
						continue
					}
				}
				kvSent++
				bytesSent += curBytes
				kv := storage.KeyValue{
					K: ctx.ConstructKey(tkv.K),
					V: tkv.V,
				}
				if err := p.SendKV(&kv); err != nil {
					dvid.Errorf("Bad data %q send KV: %v", d.DataName(), err)
				}
			}
		}()

		begKey, endKey := ctx.TKeyRange()
		err := store.ProcessRange(ctx, begKey, endKey, &storage.ChunkOp{}, func(c *storage.Chunk) error {
			if c == nil {
				return fmt.Errorf("received nil chunk in flatten push for data %s", d.DataName())
			}
			ch <- c.TKeyValue
			return nil
		})
		ch <- nil
		if err != nil {
			return fmt.Errorf("error in flatten push for data %q: %v", d.DataName(), err)
		}
	} else {
		// Start goroutine to receive all key-value pairs and transmit to remote.
		ch := make(chan *storage.KeyValue, 1000)
		go func() {
			for {
				kv := <-ch
				if kv == nil {
					if err := p.EndInstancePush(); err != nil {
						dvid.Errorf("Bad data %q termination: %v\n", d.DataName(), err)
					}
					wg.Done()
					dvid.Infof("Sent %d %q key-value pairs (%s, out of %d kv pairs, %s)\n",
						kvSent, d.DataName(), humanize.Bytes(bytesSent), kvTotal, humanize.Bytes(bytesTotal))
					return
				}
				if !ctx.ValidKV(kv, p.Versions) {
					continue
				}
				kvTotal++
				curBytes := uint64(len(kv.V) + len(kv.K))
				bytesTotal += curBytes
				if filter != nil {
					tkey, err := storage.TKeyFromKey(kv.K)
					if err != nil {
						dvid.Errorf("couldn't get %q TKey from Key %v: %v\n", d.DataName(), kv.K, err)
						continue
					}
					skip, err := filter.Check(&storage.TKeyValue{K: tkey, V: kv.V})
					if err != nil {
						dvid.Errorf("problem applying filter on data %q: %v\n", d.DataName(), err)
						continue
					}
					if skip {
						continue
					}
				}
				kvSent++
				bytesSent += curBytes
				if err := p.SendKV(kv); err != nil {
					dvid.Errorf("Bad data %q send KV: %v", d.DataName(), err)
				}
			}
		}()

		begKey, endKey := ctx.KeyRange()
		if err = store.RawRangeQuery(begKey, endKey, keysOnly, ch, nil); err != nil {
			return fmt.Errorf("push voxels %q range query: %v", d.DataName(), err)
		}
	}
	wg.Wait()
	return nil
}