func TestRequests(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := initTestRepo()
config := dvid.NewConfig()
dataservice, err := datastore.NewData(uuid, syntype, "mysynapses", config)
if err != nil {
t.Fatalf("Error creating new data instance: %v\n", err)
}
data, ok := dataservice.(*Data)
if !ok {
t.Fatalf("Returned new data instance is not synapse.Data\n")
}
// PUT first batch of synapses
testJSON, err := json.Marshal(testData)
if err != nil {
t.Fatal(err)
}
url1 := fmt.Sprintf("%snode/%s/%s/elements", server.WebAPIPath, uuid, data.DataName())
server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
// GET synapses back within superset bounding box and make sure all data is there.
testResponse(t, testData, "%snode/%s/%s/elements/1000_1000_1000/0_0_0", server.WebAPIPath, uuid, data.DataName())
// Test subset GET
testResponse(t, expected3, "%snode/%s/%s/elements/5_5_5/126_60_97", server.WebAPIPath, uuid, data.DataName())
// Test Tag 1
tag := Tag("Synapse2")
synapse2 := getTag(tag, testData)
testResponse(t, synapse2, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag)
// Test Tag 2
tag2 := Tag("Zlt90")
zlt90 := getTag(tag2, testData)
testResponse(t, zlt90, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag2)
// Test move
url5 := fmt.Sprintf("%snode/%s/%s/move/127_63_99/127_64_100", server.WebAPIPath, uuid, data.DataName())
server.TestHTTP(t, "POST", url5, nil)
testResponse(t, afterMove, "%snode/%s/%s/elements/1000_1000_1000/0_0_0", server.WebAPIPath, uuid, data.DataName())
// --- check tag
synapse2 = getTag(tag, afterMove)
testResponse(t, synapse2, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag)
// Test delete
url6 := fmt.Sprintf("%snode/%s/%s/element/127_64_100", server.WebAPIPath, uuid, data.DataName())
server.TestHTTP(t, "DELETE", url6, nil)
testResponse(t, afterDelete, "%snode/%s/%s/elements/1000_1000_1000/0_0_0", server.WebAPIPath, uuid, data.DataName())
// --- check tag
synapse2 = getTag(tag, afterDelete)
testResponse(t, synapse2, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag)
}
func TestFloatInstanceCreation(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := datastore.NewTestRepo()
// Create new voxels instance with optional parameters
name := "weights"
metadata := fmt.Sprintf(`{
"typename": "float32blk",
"dataname": %q,
"blocksize": "64,43,28",
"VoxelSize": "13.1, 14.2, 15.3",
"VoxelUnits": "picometers,nanometers,microns"
}`, name)
apiStr := fmt.Sprintf("%srepo/%s/instance", server.WebAPIPath, uuid)
server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(metadata))
// Get metadata and make sure optional settings have been set.
apiStr = fmt.Sprintf("%snode/%s/%s/info", server.WebAPIPath, uuid, name)
result := server.TestHTTP(t, "GET", apiStr, nil)
var parsed = struct {
Base struct {
TypeName, Name string
}
Extended struct {
BlockSize dvid.Point3d
VoxelSize dvid.NdFloat32
VoxelUnits dvid.NdString
}
}{}
if err := json.Unmarshal(result, &parsed); err != nil {
t.Fatalf("Error parsing JSON response of new instance metadata: %v\n", err)
}
if parsed.Base.Name != name {
t.Errorf("Parsed new instance has unexpected name: %s != %s (expected)\n",
parsed.Base.Name, name)
}
if parsed.Base.TypeName != "float32blk" {
t.Errorf("Parsed new instance has unexpected type name: %s != uint8blk (expected)\n",
parsed.Base.TypeName)
}
if !parsed.Extended.BlockSize.Equals(dvid.Point3d{64, 43, 28}) {
t.Errorf("Bad block size in new uint8blk instance: %s\n", parsed.Extended.BlockSize)
}
if !parsed.Extended.VoxelSize.Equals(dvid.NdFloat32{13.1, 14.2, 15.3}) {
t.Errorf("Bad voxel size in new uint8blk instance: %s\n", parsed.Extended.VoxelSize)
}
if parsed.Extended.VoxelUnits[0] != "picometers" {
t.Errorf("Got %q for X voxel units, not picometers\n", parsed.Extended.VoxelUnits[0])
}
if parsed.Extended.VoxelUnits[1] != "nanometers" {
t.Errorf("Got %q for X voxel units, not picometers\n", parsed.Extended.VoxelUnits[0])
}
if parsed.Extended.VoxelUnits[2] != "microns" {
t.Errorf("Got %q for X voxel units, not picometers\n", parsed.Extended.VoxelUnits[0])
}
}
// Test added after error in getting two paths to the same ancestor k/v after merge.
func TestDiamondGetOnMerge(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := initTestRepo()
config := dvid.NewConfig()
dataservice, err := datastore.NewData(uuid, kvtype, "mergetest", config)
if err != nil {
t.Fatalf("Error creating new keyvalue instance: %v\n", err)
}
data, ok := dataservice.(*Data)
if !ok {
t.Fatalf("Returned new data instance is not roi.Data\n")
}
// PUT a value
key1 := "mykey"
value1 := "some stuff"
key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1)
server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil {
t.Errorf("Unable to lock root node %s: %v\n", uuid, err)
}
uuid2, err := datastore.NewVersion(uuid, "first child", nil)
if err != nil {
t.Fatalf("Unable to create 1st child off root %s: %v\n", uuid, err)
}
if err = datastore.Commit(uuid2, "first child", nil); err != nil {
t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
}
uuid3, err := datastore.NewVersion(uuid, "second child", nil)
if err != nil {
t.Fatalf("Unable to create 2nd child off root %s: %v\n", uuid, err)
}
if err = datastore.Commit(uuid3, "second child", nil); err != nil {
t.Errorf("Unable to commit node %s: %v\n", uuid3, err)
}
child, err := datastore.Merge([]dvid.UUID{uuid2, uuid3}, "merging stuff", datastore.MergeConflictFree)
if err != nil {
t.Errorf("Error doing merge: %v\n", err)
}
// We should be able to see just the original uuid value of the k/v
childreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, child, data.DataName(), key1)
returnValue := server.TestHTTP(t, "GET", childreq, nil)
if string(returnValue) != value1 {
t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key1, value1, string(returnValue))
}
}
func TestCommitAndBranch(t *testing.T) {
tests.UseStore()
defer tests.CloseStore()
apiStr := fmt.Sprintf("%srepos", server.WebAPIPath)
r := server.TestHTTP(t, "POST", apiStr, nil)
var jsonResp map[string]interface{}
if err := json.Unmarshal(r, &jsonResp); err != nil {
t.Fatalf("Unable to unmarshal repo creation response: %s\n", string(r))
}
v, ok := jsonResp["root"]
if !ok {
t.Fatalf("No 'root' metadata returned: %s\n", string(r))
}
uuidStr, ok := v.(string)
if !ok {
t.Fatalf("Couldn't cast returned 'root' data (%v) into string.\n", v)
}
uuid := dvid.UUID(uuidStr)
// Shouldn't be able to create branch on open node.
branchReq := fmt.Sprintf("%snode/%s/branch", server.WebAPIPath, uuid)
server.TestBadHTTP(t, "POST", branchReq, nil)
// Add a keyvalue instance.
server.CreateTestInstance(t, uuid, "keyvalue", "mykv", dvid.Config{})
// Commit it.
payload := bytes.NewBufferString(`{"note": "This is my test commit", "log": ["line1", "line2", "some more stuff in a line"]}`)
apiStr = fmt.Sprintf("%snode/%s/commit", server.WebAPIPath, uuid)
server.TestHTTP(t, "POST", apiStr, payload)
// Make sure committed nodes can only be read.
// We shouldn't be able to write to keyvalue..
keyReq := fmt.Sprintf("%snode/%s/mykv/key/foo", server.WebAPIPath, uuid)
server.TestBadHTTP(t, "POST", keyReq, bytes.NewBufferString("some data"))
// Should be able to create branch now that we've committed parent.
respData := server.TestHTTP(t, "POST", branchReq, nil)
resp := struct {
Child dvid.UUID `json:"child"`
}{}
if err := json.Unmarshal(respData, &resp); err != nil {
t.Errorf("Expected 'child' JSON response. Got %s\n", string(respData))
}
// We should be able to write to that keyvalue now in the child.
keyReq = fmt.Sprintf("%snode/%s/mykv/key/foo", server.WebAPIPath, resp.Child)
server.TestHTTP(t, "POST", keyReq, bytes.NewBufferString("some data"))
}
func TestBlockAPI(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := initTestRepo()
grayscale := makeGrayscale(uuid, t, "grayscale")
// construct random blocks of data.
numBlockBytes := int32(grayscale.BlockSize().Prod())
testBlocks := 1001
var blockData []byte
for i := 0; i < testBlocks; i++ {
blockData = append(blockData, dvid.RandomBytes(numBlockBytes)...)
}
// set start of span
x := rand.Int31()
y := rand.Int31()
z := rand.Int31()
// Test uncompressed roundtrip
// send the blocks
blockReq := fmt.Sprintf("%snode/%s/grayscale/blocks/%d_%d_%d/%d", server.WebAPIPath, uuid, x, y, z, testBlocks)
server.TestHTTP(t, "POST", blockReq, bytes.NewBuffer(blockData))
// read same span of blocks
returnedData := server.TestHTTP(t, "GET", blockReq, nil)
// make sure blocks are same
totBytes := testBlocks * int(numBlockBytes)
if len(returnedData) != totBytes {
t.Errorf("Returned %d bytes, expected %d bytes", len(returnedData), totBytes)
}
if !reflect.DeepEqual(returnedData, blockData) {
t.Errorf("Returned block data != original block data\n")
}
// We should get blank blocks at different z
z += 1
blockReq = fmt.Sprintf("%snode/%s/grayscale/blocks/%d_%d_%d/%d", server.WebAPIPath, uuid, x, y, z, testBlocks)
returnedData = server.TestHTTP(t, "GET", blockReq, nil)
if len(returnedData) != totBytes {
t.Errorf("Returned %d bytes, expected %d bytes", len(returnedData), totBytes)
}
for i, b := range returnedData {
if b != 0 {
t.Fatalf("Expected 0 at returned byte %d, got %d instead.\n", i, b)
}
}
}
// Each voxel in volume has sequential labels in X, Y, then Z order.
// volSize = size of volume in blocks
// blockSize = size of a block in voxels
func (vol labelVol) postLabelVolume(t *testing.T, labelsName string, uuid dvid.UUID) {
server.CreateTestInstance(t, uuid, "labelblk", labelsName, dvid.Config{})
offset := vol.offset
nx := vol.size[0] * vol.blockSize[0]
ny := vol.size[1] * vol.blockSize[1]
nz := vol.size[2] * vol.blockSize[2]
buf := make([]byte, nx*ny*nz*8)
var label uint64
var x, y, z, v int32
for z = 0; z < nz; z++ {
for y = 0; y < ny; y++ {
for x = 0; x < nx; x++ {
label++
binary.LittleEndian.PutUint64(buf[v:v+8], label)
v += 8
}
}
}
apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/%d_%d_%d", server.WebAPIPath,
uuid, labelsName, nx, ny, nz, offset[0], offset[1], offset[2])
server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(buf))
}
func TestDeleteInstance(t *testing.T) {
tests.UseStore()
defer tests.CloseStore()
apiStr := fmt.Sprintf("%srepos", server.WebAPIPath)
r := server.TestHTTP(t, "POST", apiStr, nil)
var jsonResp map[string]interface{}
if err := json.Unmarshal(r, &jsonResp); err != nil {
t.Fatalf("Unable to unmarshal repo creation response: %s\n", string(r))
}
v, ok := jsonResp["root"]
if !ok {
t.Fatalf("No 'root' metadata returned: %s\n", string(r))
}
uuidStr, ok := v.(string)
if !ok {
t.Fatalf("Couldn't cast returned 'root' data (%v) into string.\n", v)
}
uuid := dvid.UUID(uuidStr)
// Add a data instance.
var config dvid.Config
server.CreateTestInstance(t, uuid, "keyvalue", "foo", config)
// Make sure it exists.
_, err := datastore.GetDataByUUID(uuid, "foo")
if err != nil {
t.Errorf("Couldn't create data instance 'foo'\n")
}
// Shouldn't be able to delete instance without "imsure"
delReq := fmt.Sprintf("%srepo/%s/%s", server.WebAPIPath, uuid, "foo")
server.TestBadHTTP(t, "DELETE", delReq, nil)
delReq = fmt.Sprintf("%srepo/%s/%s?imsure=true", server.WebAPIPath, uuid, "foo")
server.TestHTTP(t, "DELETE", delReq, nil)
// Make sure it no longer exists.
_, err = datastore.GetDataByUUID(uuid, "foo")
if err == nil {
t.Errorf("Shouldn't be able to access a deleted data instance 'foo'\n")
}
}
// check subgraph endpoint
func TestLabelgraphPostAndDelete(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
// Create the ROI dataservice.
uuid, _ := initTestRepo()
config := dvid.NewConfig()
dataservice, err := datastore.NewData(uuid, dtype, "lg", config)
if err != nil {
t.Errorf("Error creating new labelgraph instance: %v\n", err)
}
data, ok := dataservice.(*Data)
if !ok {
t.Errorf("Returned new data instance is not labelgraph.Data\n")
}
// PUT a labelraph
subgraphRequest := fmt.Sprintf("%snode/%s/%s/subgraph", server.WebAPIPath, uuid, data.DataName())
server.TestHTTP(t, "POST", subgraphRequest, getGraphJSON())
// Get back the labelgraph
returnedData := server.TestHTTP(t, "GET", subgraphRequest, nil)
retgraph, err := loadGraphJSON(returnedData)
if err != nil {
t.Errorf("Error on getting back JSON from roi GET: %v\n", err)
}
// Make sure the two are the same.
if !reflect.DeepEqual(retgraph, getTestGraph()) {
t.Errorf("Bad PUT/GET ROI roundtrip\nOriginal:\n%s\nReturned:\n%s\n", getTestGraph(), retgraph)
}
// Delete the labelgraph
_ = server.TestHTTP(t, "DELETE", subgraphRequest, nil)
// Subgraph should now be empty
returnedData = server.TestHTTP(t, "GET", subgraphRequest, nil)
expectedResp := "{\"Transactions\":[],\"Vertices\":[],\"Edges\":[]}"
if string(returnedData) != expectedResp {
t.Errorf("Bad ROI after ROI delete. Should be %s got: %s\n", expectedResp, string(returnedData))
}
}
func TestROIPostAndDelete(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
// Create the ROI dataservice.
uuid, _ := initTestRepo()
config := dvid.NewConfig()
dataservice, err := datastore.NewData(uuid, roitype, "roi", config)
if err != nil {
t.Errorf("Error creating new roi instance: %v\n", err)
}
data, ok := dataservice.(*Data)
if !ok {
t.Errorf("Returned new data instance is not roi.Data\n")
}
// PUT an ROI
roiRequest := fmt.Sprintf("%snode/%s/%s/roi", server.WebAPIPath, uuid, data.DataName())
server.TestHTTP(t, "POST", roiRequest, getSpansJSON(testSpans))
// Get back the ROI
returnedData := server.TestHTTP(t, "GET", roiRequest, nil)
spans, err := putSpansJSON(returnedData)
if err != nil {
t.Errorf("Error on getting back JSON from roi GET: %v\n", err)
}
// Make sure the two are the same.
if !reflect.DeepEqual(spans, testSpans) {
t.Errorf("Bad PUT/GET ROI roundtrip\nOriginal:\n%s\nReturned:\n%s\n", testSpans, spans)
}
// Delete the ROI
_ = server.TestHTTP(t, "DELETE", roiRequest, nil)
// ROI should now be empty
returnedData = server.TestHTTP(t, "GET", roiRequest, nil)
if string(returnedData) != "[]" {
t.Errorf("Bad ROI after ROI delete. Should be [ ] got: %s\n", string(returnedData))
}
}
func TestSetMetadata(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := initTestRepo()
server.CreateTestInstance(t, uuid, "imagetile", "tiles", dvid.Config{})
// Store Metadata
url := fmt.Sprintf("%snode/%s/tiles/metadata", server.WebAPIPath, uuid)
server.TestHTTP(t, "POST", url, bytes.NewBufferString(testMetadata))
// Check instance really has it set.
var metadata metadataJSON
respStr := server.TestHTTP(t, "GET", url, nil)
if err := json.Unmarshal(respStr, &metadata); err != nil {
t.Fatalf("Couldn't parse JSON response to metadata request (%v):\n%s\n", err, respStr)
}
expectMin := dvid.Point3d{0, 0, 0}
expectMax := dvid.Point3d{5, 5, 4}
if !expectMin.Equals(metadata.MinTileCoord) {
t.Errorf("Expected min tile coord %s, got %s\n", expectMin, metadata.MinTileCoord)
}
if !expectMax.Equals(metadata.MaxTileCoord) {
t.Errorf("Expected max tile coord %s, got %s\n", expectMax, metadata.MaxTileCoord)
}
tileSpec, err := parseTileSpec(metadata.Levels)
if err != nil {
t.Errorf("Error parsing returned tile level spec:\n%v\n", metadata.Levels)
}
if len(tileSpec) != 4 {
t.Errorf("Bad tile spec load: only %d elements != 4\n", len(tileSpec))
}
if tileSpec[2].Resolution.GetMax() != 40.0 {
t.Errorf("Bad tile spec at level 2: %v\n", tileSpec[2])
}
if tileSpec[3].TileSize.Value(2) != 512 {
t.Errorf("Bad tile spec at level 3: %v\n", tileSpec[3])
}
}
func TestReloadMetadata(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := datastore.NewTestRepo()
// Add data instances
var config dvid.Config
server.CreateTestInstance(t, uuid, "keyvalue", "foo", config)
server.CreateTestInstance(t, uuid, "labelblk", "labels", config)
server.CreateTestInstance(t, uuid, "roi", "someroi", config)
// Reload the metadata
apiStr := fmt.Sprintf("%sserver/reload-metadata", server.WebAPIPath)
server.TestHTTP(t, "POST", apiStr, nil)
// Make sure repo UUID still there
jsonStr, err := datastore.MarshalJSON()
if err != nil {
t.Fatalf("can't get repos JSON: %v\n", err)
}
var jsonResp map[string](map[string]interface{})
if err := json.Unmarshal(jsonStr, &jsonResp); err != nil {
t.Fatalf("Unable to unmarshal repos info response: %s\n", jsonStr)
}
if len(jsonResp) != 1 {
t.Errorf("reloaded repos had more than one repo: %v\n", jsonResp)
}
for k := range jsonResp {
if dvid.UUID(k) != uuid {
t.Fatalf("Expected uuid %s, got %s. Full JSON:\n%v\n", uuid, k, jsonResp)
}
}
// Make sure the data instances are still there.
_, err = datastore.GetDataByUUIDName(uuid, "foo")
if err != nil {
t.Errorf("Couldn't get keyvalue data instance after reload\n")
}
_, err = datastore.GetDataByUUIDName(uuid, "labels")
if err != nil {
t.Errorf("Couldn't get labelblk data instance after reload\n")
}
_, err = datastore.GetDataByUUIDName(uuid, "someroi")
if err != nil {
t.Errorf("Couldn't get roi data instance after reload\n")
}
}
func (vol *labelVol) testBlocks(t *testing.T, uuid dvid.UUID, compression, roi string) {
span := 5
apiStr := fmt.Sprintf("%snode/%s/%s/blocks/%d_%d_%d/%d_%d_%d", server.WebAPIPath,
uuid, vol.name, 160, 32, 32, vol.offset[0], vol.offset[1], vol.offset[2])
if compression == "uncompressed" {
apiStr += "?compression=uncompressed"
}
data := server.TestHTTP(t, "GET", apiStr, nil)
fmt.Printf("Got %d bytes of data\n", len(data))
blockBytes := 32 * 32 * 32 * 8
// Check if values are what we expect
bx := vol.offset[0] / 32
by := vol.offset[1] / 32
bz := vol.offset[2] / 32
b := 0
for i := 0; i < span; i++ {
// Get block coord + block size
if b+16 > len(data) {
t.Fatalf("Only got %d bytes back from block API call, yet next coord in span would put index @ %d\n", len(data), b+16)
}
x := int32(binary.LittleEndian.Uint32(data[b : b+4]))
b += 4
y := int32(binary.LittleEndian.Uint32(data[b : b+4]))
b += 4
z := int32(binary.LittleEndian.Uint32(data[b : b+4]))
b += 4
n := int(binary.LittleEndian.Uint32(data[b : b+4]))
b += 4
if x != bx || y != by || z != bz {
t.Errorf("Bad block coordinate: expected (%d,%d,%d), got (%d,%d,%d)\n", bx, by, bz, x, y, z)
}
// Read in the block data as assumed LZ4 and check it.
var uncompressed []byte
if compression != "uncompressed" {
uncompressed = make([]byte, blockBytes)
if err := lz4.Uncompress(data[b:b+n], uncompressed); err != nil {
t.Fatalf("Unable to uncompress LZ4 data (%s), %d bytes: %v\n", apiStr, n, err)
}
} else {
uncompressed = data[b : b+n]
}
vol.testBlock(t, x, y, z, uncompressed)
b += n
bx++
}
}
func (vol labelVol) getLabelVolume(t *testing.T, uuid dvid.UUID, compression, roi string) []byte {
apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/%d_%d_%d", server.WebAPIPath,
uuid, vol.name, vol.nx, vol.ny, vol.nz, vol.offset[0], vol.offset[1], vol.offset[2])
query := true
switch compression {
case "lz4":
apiStr += "?compression=lz4"
case "gzip":
apiStr += "?compression=gzip"
default:
query = false
}
if roi != "" {
if query {
apiStr += "&roi=" + roi
} else {
apiStr += "?roi=" + roi
}
}
data := server.TestHTTP(t, "GET", apiStr, nil)
switch compression {
case "lz4":
uncompressed := make([]byte, vol.numBytes())
if err := lz4.Uncompress(data, uncompressed); err != nil {
t.Fatalf("Unable to uncompress LZ4 data (%s), %d bytes: %v\n", apiStr, len(data), err)
}
data = uncompressed
case "gzip":
buf := bytes.NewBuffer(data)
gr, err := gzip.NewReader(buf)
if err != nil {
t.Fatalf("Error on gzip new reader: %v\n", err)
}
uncompressed, err := ioutil.ReadAll(gr)
if err != nil {
t.Fatalf("Error on reading gzip: %v\n", err)
}
if err = gr.Close(); err != nil {
t.Fatalf("Error on closing gzip: %v\n", err)
}
data = uncompressed
default:
}
if len(data) != int(vol.numBytes()) {
t.Errorf("Expected %d uncompressed bytes from 3d labelblk GET. Got %d instead.", vol.numBytes(), len(data))
}
return data
}
func testResponseLabel(t *testing.T, expected interface{}, template string, args ...interface{}) {
var useRels bool
if strings.HasSuffix(template, "?relationships=true") {
useRels = true
}
url := fmt.Sprintf(template, args...)
returnValue := server.TestHTTP(t, "GET", url, nil)
if useRels {
var elems Elements
if expected == nil {
elems = Elements{}
} else {
var ok bool
elems, ok = expected.(Elements)
if !ok {
t.Fatalf("testResponseLabel with template %q didn't get passed Elements for expected: %v\n", template, expected)
}
}
got := Elements{}
if err := json.Unmarshal(returnValue, &got); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(elems.Normalize(), got.Normalize()) {
t.Fatalf("Expected for %s:\n%v\nGot:\n%v\n", url, elems.Normalize(), got.Normalize())
}
} else {
var elems ElementsNR
if expected == nil {
elems = ElementsNR{}
} else {
var ok bool
elems, ok = expected.(ElementsNR)
if !ok {
t.Fatalf("testResponseLabel with template %q didn't get passed ElementsNR for expected: %v\n", template, expected)
}
}
got := ElementsNR{}
if err := json.Unmarshal(returnValue, &got); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(elems.Normalize(), got.Normalize()) {
t.Fatalf("Expected for %s:\n%v\nGot:\n%v\n", url, elems.Normalize(), got.Normalize())
}
}
}
func TestTileKey(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := initTestRepo()
server.CreateTestInstance(t, uuid, "imagetile", "tiles", dvid.Config{})
keyURL := fmt.Sprintf("%snode/%s/tiles/tilekey/xy/0/1_2_3", server.WebAPIPath, uuid)
respStr := server.TestHTTP(t, "GET", keyURL, nil)
keyResp := struct {
Key string `json:"key"`
}{}
if err := json.Unmarshal(respStr, &keyResp); err != nil {
t.Fatalf("Couldn't parse JSON response to tilekey request (%v):\n%s\n", err, keyResp)
}
kb := make([]byte, hex.DecodedLen(len(keyResp.Key)))
_, err := hex.Decode(kb, []byte(keyResp.Key))
if err != nil {
t.Fatalf("Couldn't parse return hex key: %s", keyResp.Key)
}
// Decipher TKey portion to make sure it's correct.
key := storage.Key(kb)
tk, err := storage.TKeyFromKey(key)
if err != nil {
t.Fatalf("Couldn't get TKey from returned key (%v): %x", err, kb)
}
tile, plane, scale, err := DecodeTKey(tk)
if err != nil {
t.Fatalf("Bad decode of TKey (%v): %x", err, tk)
}
expectTile := dvid.ChunkPoint3d{1, 2, 3}
if tile != expectTile {
t.Errorf("Expected tile %v, got %v\n", expectTile, tile)
}
if !plane.Equals(dvid.XY) {
t.Errorf("Expected plane to be XY, got %v\n", plane)
}
if scale != 0 {
t.Errorf("Expected scale to be 0, got %d\n", scale)
}
}
// Create a new label volume and post it to the test datastore.
// Each voxel in volume has sequential labels in X, Y, then Z order.
func (vol *labelVol) postLabelVolume(t *testing.T, uuid dvid.UUID, compression, roi string, startLabel uint64) {
vol.makeLabelVolume(t, uuid, startLabel)
apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/%d_%d_%d", server.WebAPIPath,
uuid, vol.name, vol.nx, vol.ny, vol.nz, vol.offset[0], vol.offset[1], vol.offset[2])
query := true
var data []byte
var err error
switch compression {
case "lz4":
apiStr += "?compression=lz4"
compressed := make([]byte, lz4.CompressBound(vol.data))
var outSize int
if outSize, err = lz4.Compress(vol.data, compressed); err != nil {
t.Fatal(err)
}
data = compressed[:outSize]
case "gzip":
apiStr += "?compression=gzip"
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
if _, err = gw.Write(vol.data); err != nil {
t.Fatal(err)
}
data = buf.Bytes()
if err = gw.Close(); err != nil {
t.Fatal(err)
}
default:
data = vol.data
query = false
}
if roi != "" {
if query {
apiStr += "&roi=" + roi
} else {
apiStr += "?roi=" + roi
}
}
server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(data))
}
func testResponse(t *testing.T, expected Elements, template string, args ...interface{}) {
url := fmt.Sprintf(template, args...)
returnValue := server.TestHTTP(t, "GET", url, nil)
got := Elements{}
if err := json.Unmarshal(returnValue, &got); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(expected.Normalize(), got.Normalize()) {
var expectedStr, gotStr string
if jsonBytes, err := json.Marshal(expected.Normalize()); err != nil {
t.Fatalf("error converting expected to JSON: %v\n", err)
} else {
expectedStr = string(jsonBytes)
}
if jsonBytes, err := json.Marshal(got.Normalize()); err != nil {
t.Fatalf("error converting got to JSON: %v\n", err)
} else {
gotStr = string(jsonBytes)
}
t.Fatalf("Expected for %s:\n%s\nGot:\n%s\n", url, expectedStr, gotStr)
}
}
func (vol labelVol) testSlices(t *testing.T, uuid dvid.UUID) {
// Verify XY slice.
sliceOffset := vol.offset
sliceOffset[0] += 51
sliceOffset[1] += 11
sliceOffset[2] += 23
slice := sliceTester{"xy", 67, 83, sliceOffset}
apiStr := slice.apiStr(uuid, vol.name)
xy := server.TestHTTP(t, "GET", apiStr, nil)
img, format, err := dvid.ImageFromBytes(xy, EncodeFormat(), false)
if err != nil {
t.Fatalf("Error on XY labels GET: %v\n", err)
}
if format != "png" {
t.Errorf("Expected XY labels GET to return %q image, got %q instead.\n", "png", format)
}
if img.NumBytes() != 67*83*8 {
t.Errorf("Expected %d bytes from XY labelblk GET. Got %d instead.", 160*160*8, img.NumBytes())
}
slice.testLabel(t, vol, img)
// Verify XZ slice returns what we expect.
sliceOffset = vol.offset
sliceOffset[0] += 11
sliceOffset[1] += 4
sliceOffset[2] += 3
slice = sliceTester{"xz", 67, 83, sliceOffset}
apiStr = slice.apiStr(uuid, vol.name)
xz := server.TestHTTP(t, "GET", apiStr, nil)
img, format, err = dvid.ImageFromBytes(xz, EncodeFormat(), false)
if err != nil {
t.Fatalf("Error on XZ labels GET: %v\n", err)
}
if format != "png" {
t.Errorf("Expected XZ labels GET to return %q image, got %q instead.\n", "png", format)
}
if img.NumBytes() != 67*83*8 {
t.Errorf("Expected %d bytes from XZ labelblk GET. Got %d instead.", 67*83*8, img.NumBytes())
}
slice.testLabel(t, vol, img)
// Verify YZ slice returns what we expect.
sliceOffset = vol.offset
sliceOffset[0] += 7
sliceOffset[1] += 33
sliceOffset[2] += 33
slice = sliceTester{"yz", 67, 83, sliceOffset}
apiStr = slice.apiStr(uuid, vol.name)
yz := server.TestHTTP(t, "GET", apiStr, nil)
img, format, err = dvid.ImageFromBytes(yz, EncodeFormat(), false)
if err != nil {
t.Fatalf("Error on YZ labels GET: %v\n", err)
}
if format != "png" {
t.Errorf("Expected YZ labels GET to return %q image, got %q instead.\n", "png", format)
}
if img.NumBytes() != 67*83*8 {
t.Errorf("Expected %d bytes from YZ labelblk GET. Got %d instead.", 67*83*8, img.NumBytes())
}
slice.testLabel(t, vol, img)
}
// Put label data into given data instance.
func (v *testVolume) put(t *testing.T, uuid dvid.UUID, name string) {
apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/%d_%d_%d?mutate=true", server.WebAPIPath,
uuid, name, v.size[0], v.size[1], v.size[2], v.offset[0], v.offset[1], v.offset[2])
server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(v.data))
}
func TestTileCheck(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
// Make source
uuid, _ := initTestRepo()
makeGrayscale(uuid, t, "grayscale")
// Make imagetile and set various properties
config := dvid.NewConfig()
config.Set("Placeholder", "true")
config.Set("Format", "jpg")
config.Set("Source", "grayscale")
tileservice, err := datastore.NewData(uuid, mstype, "myimagetile", config)
if err != nil {
t.Errorf("Unable to create imagetile instance: %v\n", err)
}
msdata, ok := tileservice.(*Data)
if !ok {
t.Fatalf("Can't cast imagetile data service into imagetile.Data\n")
}
// Store Metadata
url := fmt.Sprintf("%snode/%s/myimagetile/metadata", server.WebAPIPath, uuid)
server.TestHTTP(t, "POST", url, bytes.NewBufferString(testMetadata2))
// Create the ROI
_, err = datastore.NewData(uuid, roitype, "myroi", dvid.NewConfig())
if err != nil {
t.Errorf("Error creating new roi instance: %v\n", err)
}
// PUT an ROI
roiRequest := fmt.Sprintf("%snode/%s/myroi/roi", server.WebAPIPath, uuid)
server.TestHTTP(t, "POST", roiRequest, getSpansJSON(testSpans))
// Create fake filter
spec := fmt.Sprintf("roi:myroi,%s", uuid)
f, err := msdata.NewFilter(storage.FilterSpec(spec))
if err != nil {
t.Errorf("Couldn't make filter: %v\n", err)
}
if f == nil {
t.Fatalf("Couldn't detect myroi data instance\n")
}
// Check various key values for proper spatial checks.
var tx, ty int32
tx = (205 * 32) / 512
ty = (101 * 32) / 512
tile := dvid.ChunkPoint3d{tx, ty, 101 * 32}
scale := Scaling(0)
tkv := &storage.TKeyValue{K: NewTKey(tile, dvid.XY, scale)}
skip, err := f.Check(tkv)
if err != nil {
t.Errorf("Error on Check of key %q: %v\n", tkv.K, err)
}
if skip {
t.Errorf("Expected false skip, got %v for tile %s\n", skip, tile)
}
tile = dvid.ChunkPoint3d{tx, ty, 106 * 32}
tkv = &storage.TKeyValue{K: NewTKey(tile, dvid.XY, scale)}
skip, err = f.Check(tkv)
if err != nil {
t.Errorf("Error on Check of key %q: %v\n", tkv.K, err)
}
if !skip {
t.Errorf("Expected true skip, got %v for tile %s\n", skip, tile)
}
tx = (205 * 32) / 512
ty = (121 * 32) / 512
tile = dvid.ChunkPoint3d{tx, ty, 101 * 32}
tkv = &storage.TKeyValue{K: NewTKey(tile, dvid.XY, scale)}
skip, err = f.Check(tkv)
if err != nil {
t.Errorf("Error on Check of key %q: %v\n", tkv.K, err)
}
if !skip {
t.Errorf("Expected true skip, got %v for tile %s\n", skip, tile)
}
tx = (225 * 32) / 512
ty = (101 * 32) / 512
tile = dvid.ChunkPoint3d{tx, ty, 101 * 32}
tkv = &storage.TKeyValue{K: NewTKey(tile, dvid.XY, scale)}
skip, err = f.Check(tkv)
if err != nil {
t.Errorf("Error on Check of key %q: %v\n", tkv.K, err)
}
if !skip {
t.Errorf("Expected true skip, got %v for tile %s\n", skip, tile)
}
}
func TestFilter(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
// Create testbed volume and data instances
uuid, _ := initTestRepo()
var config dvid.Config
server.CreateTestInstance(t, uuid, "labelblk", "labels", config)
d, err := datastore.NewData(uuid, labelvolT, "bodies", config)
if err != nil {
t.Fatalf("Unable to create labelvol instance: %s\n", err)
}
server.CreateTestSync(t, uuid, "labels", "bodies")
server.CreateTestSync(t, uuid, "bodies", "labels")
// Populate the labels, which should automatically populate the labelvol
_ = createLabelTestVolume(t, uuid, "labels")
if err := datastore.BlockOnUpdating(uuid, "bodies"); err != nil {
t.Fatalf("Error blocking on sync of labels -> bodies: %v\n", err)
}
// Create a ROI that will be used for filter test.
server.CreateTestInstance(t, uuid, "roi", "myroi", config)
roiRequest := fmt.Sprintf("%snode/%s/myroi/roi", server.WebAPIPath, uuid)
server.TestHTTP(t, "POST", roiRequest, getROIReader())
// Create the filter spec
fs := storage.FilterSpec(fmt.Sprintf("roi:myroi,%s", uuid))
var filter storage.Filter
filterer, ok := d.(storage.Filterer)
if !ok {
t.Fatalf("labelvol instance does not implement storage.Filterer\n")
}
filter, err = filterer.NewFilter(fs)
if err != nil {
t.Fatalf("Can't create filter from spec %q: %v\n", fs, err)
}
if filter == nil {
t.Fatalf("No filter could be created from spec %q\n", fs)
}
// Test the filter.
tkv := storage.TKeyValue{K: NewTKey(23, dvid.ChunkPoint3d{0, 0, 0}.ToIZYXString())}
skip, err := filter.Check(&tkv)
if !skip {
t.Errorf("Expected filter check 1 to skip, instead filter.Check() returned not skip")
}
tkv = storage.TKeyValue{K: NewTKey(23, dvid.ChunkPoint3d{1, 1, 1}.ToIZYXString())}
skip, err = filter.Check(&tkv)
if skip {
t.Errorf("Expected filter check 2 to not skip!")
}
tkv = storage.TKeyValue{K: NewTKey(23, dvid.ChunkPoint3d{2, 1, 2}.ToIZYXString())}
skip, err = filter.Check(&tkv)
if skip {
t.Errorf("Expected filter check 2 to not skip!")
}
tkv = storage.TKeyValue{K: NewTKey(23, dvid.ChunkPoint3d{3, 1, 1}.ToIZYXString())}
skip, err = filter.Check(&tkv)
if !skip {
t.Errorf("Expected filter check 3 to skip!")
}
}
func (mjson mergeJSON) send(t *testing.T, uuid dvid.UUID, name string) {
apiStr := fmt.Sprintf("%snode/%s/%s/merge", server.WebAPIPath, uuid, name)
server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer([]byte(mjson)))
}
func TestLabels(t *testing.T) {
tests.UseStore()
defer tests.CloseStore()
uuid := dvid.UUID(server.NewTestRepo(t))
if len(uuid) < 5 {
t.Fatalf("Bad root UUID for new repo: %s\n", uuid)
}
// Create a labelblk instance
server.CreateTestInstance(t, uuid, "labelblk", "labels", dvid.Config{})
vol := labelVol{
size: dvid.Point3d{5, 5, 5}, // in blocks
blockSize: dvid.Point3d{32, 32, 32},
offset: dvid.Point3d{32, 64, 96},
name: "labels",
}
vol.postLabelVolume(t, uuid, "", "", 0)
// Repost the label volume 3 more times with increasing starting values.
vol.postLabelVolume(t, uuid, "", "", 2100)
vol.postLabelVolume(t, uuid, "", "", 8176)
vol.postLabelVolume(t, uuid, "", "", 16623)
vol.testSlices(t, uuid)
// Try to post last volume concurrently 3x and then check result.
wg := new(sync.WaitGroup)
wg.Add(3)
go func() {
vol.postLabelVolume(t, uuid, "", "", 16623)
wg.Done()
}()
go func() {
vol.postLabelVolume(t, uuid, "", "", 16623)
wg.Done()
}()
go func() {
vol.postLabelVolume(t, uuid, "", "", 16623)
wg.Done()
}()
wg.Wait()
vol.testGetLabelVolume(t, uuid, "", "")
// Try concurrent write of disjoint subvolumes.
vol2 := labelVol{
size: dvid.Point3d{5, 5, 5}, // in blocks
blockSize: dvid.Point3d{32, 32, 32},
offset: dvid.Point3d{192, 64, 96},
name: "labels",
}
vol3 := labelVol{
size: dvid.Point3d{5, 5, 5}, // in blocks
blockSize: dvid.Point3d{32, 32, 32},
offset: dvid.Point3d{192, 224, 96},
name: "labels",
}
vol4 := labelVol{
size: dvid.Point3d{5, 5, 5}, // in blocks
blockSize: dvid.Point3d{32, 32, 32},
offset: dvid.Point3d{32, 224, 96},
name: "labels",
}
wg.Add(3)
go func() {
vol2.postLabelVolume(t, uuid, "lz4", "", 4000)
wg.Done()
}()
go func() {
vol3.postLabelVolume(t, uuid, "lz4", "", 8000)
wg.Done()
}()
go func() {
vol4.postLabelVolume(t, uuid, "lz4", "", 1200)
wg.Done()
}()
wg.Wait()
vol.testGetLabelVolume(t, uuid, "", "")
vol2.testGetLabelVolume(t, uuid, "", "")
vol3.testGetLabelVolume(t, uuid, "", "")
vol4.testGetLabelVolume(t, uuid, "", "")
// Verify various GET 3d volume with compressions and no ROI.
vol.testGetLabelVolume(t, uuid, "", "")
vol.testGetLabelVolume(t, uuid, "lz4", "")
vol.testGetLabelVolume(t, uuid, "gzip", "")
// Create a new ROI instance.
roiName := "myroi"
server.CreateTestInstance(t, uuid, "roi", roiName, dvid.Config{})
// Add ROI data
apiStr := fmt.Sprintf("%snode/%s/%s/roi", server.WebAPIPath, uuid, roiName)
server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(labelsJSON()))
// Post updated labels without ROI and make sure it returns those values.
var labelNoROI uint64 = 20000
vol.postLabelVolume(t, uuid, "", "", labelNoROI)
returned := vol.testGetLabelVolume(t, uuid, "", "")
startLabel := binary.LittleEndian.Uint64(returned[0:8])
if startLabel != labelNoROI+1 {
t.Errorf("Expected first voxel to be label %d and got %d instead\n", labelNoROI+1, startLabel)
}
// TODO - Use the ROI to retrieve a 2d xy image.
// TODO - Make sure we aren't getting labels back in non-ROI points.
// Post again but now with ROI
var labelWithROI uint64 = 40000
vol.postLabelVolume(t, uuid, "", roiName, labelWithROI)
// Verify ROI masking of POST where anything outside ROI is old labels.
returned = vol.getLabelVolume(t, uuid, "", "")
var newlabel uint64 = labelWithROI
var oldlabel uint64 = labelNoROI
nx := vol.size[0] * vol.blockSize[0]
ny := vol.size[1] * vol.blockSize[1]
nz := vol.size[2] * vol.blockSize[2]
var x, y, z, v int32
for z = 0; z < nz; z++ {
voxz := z + vol.offset[2]
blockz := voxz / DefaultBlockSize
for y = 0; y < ny; y++ {
voxy := y + vol.offset[1]
blocky := voxy / DefaultBlockSize
for x = 0; x < nx; x++ {
voxx := x + vol.offset[0]
blockx := voxx / DefaultBlockSize
oldlabel++
newlabel++
got := binary.LittleEndian.Uint64(returned[v : v+8])
if inroi(blockx, blocky, blockz) {
if got != newlabel {
t.Fatalf("Expected %d in ROI, got %d\n", newlabel, got)
}
} else {
if got != oldlabel {
t.Fatalf("Expected %d outside ROI, got %d\n", oldlabel, got)
}
}
v += 8
}
}
}
// Verify that a ROI-enabled GET has zeros everywhere outside ROI.
returned = vol.getLabelVolume(t, uuid, "", roiName)
newlabel = labelWithROI
x, y, z, v = 0, 0, 0, 0
for z = 0; z < nz; z++ {
voxz := z + vol.offset[2]
blockz := voxz / DefaultBlockSize
for y = 0; y < ny; y++ {
voxy := y + vol.offset[1]
blocky := voxy / DefaultBlockSize
for x = 0; x < nx; x++ {
voxx := x + vol.offset[0]
blockx := voxx / DefaultBlockSize
oldlabel++
newlabel++
got := binary.LittleEndian.Uint64(returned[v : v+8])
if inroi(blockx, blocky, blockz) {
if got != newlabel {
t.Fatalf("Expected %d in ROI, got %d\n", newlabel, got)
}
} else {
if got != 0 {
t.Fatalf("Expected zero outside ROI, got %d\n", got)
}
}
v += 8
}
}
}
}
func TestForegroundROI(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := initTestRepo()
grayscale := makeGrayscale(uuid, t, "grayscale")
// Create a fake 128^3 volume with inner 64^3 foreground and
// outer background split between 0 and 255 at z = 63,64
data := make([]uint8, 128*128*128, 128*128*128)
for z := 64; z < 128; z++ {
for y := 0; y < 128; y++ {
for x := 0; x < 128; x++ {
data[z*128*128+y*128+x] = 255
}
}
}
for z := 32; z < 96; z++ {
for y := 32; y < 96; y++ {
for x := 32; x < 96; x++ {
data[z*128*128+y*128+x] = 128
}
}
}
// Put the volume so it's block-aligned
buf := bytes.NewBuffer(data)
putRequest := fmt.Sprintf("%snode/%s/grayscale/raw/0_1_2/128_128_128/160_64_128", server.WebAPIPath, uuid)
server.TestHTTP(t, "POST", putRequest, buf)
// Request a foreground ROI
var reply datastore.Response
cmd := dvid.Command{"node", string(uuid), "grayscale", "roi", "foreground", "0,255"}
if err := grayscale.DoRPC(datastore.Request{Command: cmd}, &reply); err != nil {
t.Fatalf("Error running foreground ROI command: %v\n", err)
}
// Check results, making sure it's valid (200).
var roiJSON string
for {
roiRequest := fmt.Sprintf("%snode/%s/foreground/roi", server.WebAPIPath, uuid)
req, err := http.NewRequest("GET", roiRequest, nil)
if err != nil {
t.Fatalf("Unsuccessful GET on foreground ROI: %v", err)
}
w := httptest.NewRecorder()
server.ServeSingleHTTP(w, req)
roiJSON = string(w.Body.Bytes())
if w.Code == http.StatusOK {
break
} else if w.Code != http.StatusPartialContent {
t.Fatalf("Unknown reponse for GET foreground ROI (%d): %s\n", w.Code, roiJSON)
}
}
// Check results
// We have block-aligned 2^3 block foreground
// from z = 32 -> 95, block coord 1 -> 2 + offset in z by 4 blocks = 5, 6
// from y = 32 -> 95, offset in y by 2 blocks = 3, 4
// from x = 32 -> 95, offset in x by 5 blocks = 6, 7
expected := "[[5,3,6,7],[5,4,6,7],[6,3,6,7],[6,4,6,7]]"
if roiJSON != expected {
t.Errorf("Expected the following foreground ROI:\n%s\nGot instead:\n%s\n", expected, roiJSON)
}
}
func TestROIRequests(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
// Create the ROI dataservice.
uuid, _ := initTestRepo()
config := dvid.NewConfig()
dataservice, err := datastore.NewData(uuid, roitype, "roi", config)
if err != nil {
t.Errorf("Error creating new roi instance: %v\n", err)
}
data, ok := dataservice.(*Data)
if !ok {
t.Errorf("Returned new data instance is not roi.Data\n")
}
// PUT an ROI
roiRequest := fmt.Sprintf("%snode/%s/%s/roi", server.WebAPIPath, uuid, data.DataName())
server.TestHTTP(t, "POST", roiRequest, getSpansJSON(testSpans))
// Get back the ROI
returnedData := server.TestHTTP(t, "GET", roiRequest, nil)
spans, err := putSpansJSON(returnedData)
if err != nil {
t.Errorf("Error on getting back JSON from roi GET: %v\n", err)
}
// Make sure the two are the same.
if !reflect.DeepEqual(spans, testSpans) {
t.Errorf("Bad PUT/GET ROI roundtrip\nOriginal:\n%s\nReturned:\n%s\n", testSpans, spans)
}
// Test the ptquery
ptqueryRequest := fmt.Sprintf("%snode/%s/%s/ptquery", server.WebAPIPath, uuid, data.DataName())
returnedData = server.TestHTTP(t, "POST", ptqueryRequest, getPointsJSON(testPoints))
inclusions, err := putInclusionJSON(returnedData)
if err != nil {
t.Fatalf("Error on getting back JSON from ptquery: %v\n", err)
}
// Make sure the two are the same.
if !reflect.DeepEqual(inclusions, expectedInclusions) {
t.Errorf("Bad ptquery results\nOriginal:\n%s\nReturned:\n%s\n", expectedInclusions, inclusions)
}
// Test ROI mask out of range -- should be all 0.
maskRequest := fmt.Sprintf("%snode/%s/%s/mask/0_1_2/100_100_100/10_40_70", server.WebAPIPath, uuid, data.DataName())
returnedData = server.TestHTTP(t, "GET", maskRequest, nil)
if len(returnedData) != 100*100*100 {
t.Errorf("Expected mask volume of %d bytes, got %d bytes instead\n", 100*100*100, len(returnedData))
}
for i, value := range returnedData {
if value != 0 {
t.Errorf("Expected all-zero mask, got %d at index %d\n", value, i)
break
}
}
// Test ROI mask within range.
maskRequest = fmt.Sprintf("%snode/%s/%s/mask/0_1_2/100_100_100/6350_3232_3200", server.WebAPIPath, uuid, data.DataName())
returnedData = server.TestHTTP(t, "GET", maskRequest, nil)
if len(returnedData) != 100*100*100 {
t.Errorf("Expected mask volume of %d bytes, got %d bytes instead\n", 100*100*100, len(returnedData))
}
// Check first block plane
for y := 0; y < 100; y++ {
for x := 0; x < 100; x++ {
value := returnedData[y*100+x]
if x < 50 && value != 0 {
t.Errorf("Expected mask to be zero at (%d, %d) before ROI, got %d instead\n", x, y, value)
break
}
if x >= 50 && y < 64 && value != 1 {
t.Errorf("Expected mask to be 1 at (%d, %d) within ROI, got %d instead\n", x, y, value)
break
}
// tuple{100, 103, 201, 212}
if x <= 81 && y >= 64 && y < 96 && value != 0 {
t.Errorf("Expected mask to be zero at (%d, %d) before ROI, got %d instead\n", x, y, value)
break
}
if x > 81 && y >= 64 && y < 96 && value != 1 {
t.Errorf("Expected mask to be 1 at (%d, %d) within ROI, got %d instead\n", x, y, value)
break
}
}
}
// Check second block plane
offset := 32 * 100 * 100 // moves to next block in Z
for y := 0; y < 100; y++ {
for x := 0; x < 100; x++ {
value := returnedData[offset+y*100+x]
if x < 50 && value != 0 {
t.Errorf("Expected mask to be zero at (%d, %d) before ROI, got %d instead\n", x, y, value)
break
}
if x <= 81 && y < 32 && value != 0 {
t.Errorf("Expected mask to be zero at (%d, %d) before ROI, got %d instead\n", x, y, value)
break
}
if x > 81 && y < 32 && value != 1 {
t.Errorf("Expected mask to be 1 at (%d, %d) within ROI, got %d instead\n", x, y, value)
break
}
if y >= 32 && value != 0 {
t.Errorf("Expected mask to be zero at (%d, %d) before ROI, got %d instead\n", x, y, value)
break
}
}
}
// Check last block plane
offset = 96 * 100 * 100 // moves to last ROI layer in Z
for y := 0; y < 100; y++ {
for x := 0; x < 100; x++ {
value := returnedData[offset+y*100+x]
if x < 50 && value != 0 {
t.Errorf("Expected mask to be zero at (%d, %d) before ROI, got %d instead\n", x, y, value)
break
}
if x >= 50 && y < 32 && value != 1 {
t.Errorf("Expected mask to be 1 at (%d, %d) within ROI, got %d instead\n", x, y, value)
break
}
if y >= 32 && y < 64 && value != 0 {
t.Errorf("Expected mask to be zero at (%d, %d) before ROI, got %d instead\n", x, y, value)
break
}
if x >= 50 && y >= 64 && y < 96 && value != 1 {
t.Errorf("Expected mask to be 1 at (%d, %d) within ROI, got %d instead\n", x, y, value)
break
}
if y >= 96 && value != 0 {
t.Errorf("Expected mask to be zero at (%d, %d) before ROI, got %d instead\n", x, y, value)
break
}
}
}
}
func TestTagRequests(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := initTestRepo()
config := dvid.NewConfig()
dataservice, err := datastore.NewData(uuid, syntype, "mysynapses", config)
if err != nil {
t.Fatalf("Error creating new data instance: %v\n", err)
}
data, ok := dataservice.(*Data)
if !ok {
t.Fatalf("Returned new data instance is not synapse.Data\n")
}
// PUT first batch of synapses
testJSON, err := json.Marshal(testTagData)
if err != nil {
t.Fatal(err)
}
url1 := fmt.Sprintf("%snode/%s/%s/elements", server.WebAPIPath, uuid, data.DataName())
server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
// Test Tags
expected := Elements{
{
ElementNR{
Pos: dvid.Point3d{15, 27, 35}, // Label 1
Kind: PreSyn,
Tags: []Tag{"Synapse1", "Zlt90"},
Prop: map[string]string{
"Im a T-Bar": "yes",
"I'm not a PSD": "sure",
"i'm really special": "",
},
},
[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
},
{
ElementNR{
Pos: dvid.Point3d{21, 33, 40}, // Label 2
Kind: PostSyn,
Tags: []Tag{"Synapse1"},
},
[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
},
}
tag := Tag("Synapse1")
testResponse(t, expected, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag)
expected = Elements{
{
ElementNR{
Pos: dvid.Point3d{20, 30, 40}, // Label 2
Kind: PostSyn,
Tags: []Tag{"Synapse10"},
},
[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
},
}
tag = Tag("Synapse10")
testResponse(t, expected, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag)
expected = Elements{
{
ElementNR{
Pos: dvid.Point3d{14, 25, 37}, // Label 3
Kind: PostSyn,
Tags: []Tag{"Synapse11", "Zlt90"},
},
[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
},
}
tag = Tag("Synapse11")
testResponse(t, expected, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag)
expected = Elements{
{
ElementNR{
Pos: dvid.Point3d{33, 30, 31},
Kind: PostSyn,
Tags: []Tag{"Synapse111", "Zlt90"},
},
[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
},
}
tag = Tag("Synapse111")
testResponse(t, expected, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag)
// delete an annotation and check if its deleted in tag
delurl := fmt.Sprintf("%snode/%s/%s/element/15_27_35", server.WebAPIPath, uuid, data.DataName())
server.TestHTTP(t, "DELETE", delurl, nil)
expected = Elements{
{
ElementNR{
Pos: dvid.Point3d{21, 33, 40}, // Label 2
Kind: PostSyn,
Tags: []Tag{"Synapse1"},
},
[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
},
}
testResponse(t, expected, "%snode/%s/%s/tag/Synapse1?relationships=true", server.WebAPIPath, uuid, data.DataName())
}
func TestKeyvalueVersioning(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := initTestRepo()
config := dvid.NewConfig()
dataservice, err := datastore.NewData(uuid, kvtype, "versiontest", config)
if err != nil {
t.Fatalf("Error creating new keyvalue instance: %v\n", err)
}
data, ok := dataservice.(*Data)
if !ok {
t.Fatalf("Returned new data instance is not roi.Data\n")
}
// PUT a value
key1 := "mykey"
value1 := "some stuff"
key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1)
server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
// Add 2nd k/v
key2 := "my2ndkey"
value2 := "more good stuff"
key2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key2)
server.TestHTTP(t, "POST", key2req, strings.NewReader(value2))
// Create a new version in repo
if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil {
t.Errorf("Unable to lock root node %s: %v\n", uuid, err)
}
uuid2, err := datastore.NewVersion(uuid, "some child", nil)
if err != nil {
t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err)
}
_, err = datastore.VersionFromUUID(uuid2)
if err != nil {
t.Fatalf("Unable to get version ID from new uuid %s: %v\n", uuid2, err)
}
// Change the 2nd k/v
uuid2val := "this is completely different"
uuid2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid2, data.DataName(), key2)
server.TestHTTP(t, "POST", uuid2req, strings.NewReader(uuid2val))
// Get the first version value
returnValue := server.TestHTTP(t, "GET", key2req, nil)
if string(returnValue) != value2 {
t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, value2, string(returnValue))
}
// Get the second version value
returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
if string(returnValue) != uuid2val {
t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
}
// Check return of first two keys in range.
rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(),
"my", "zebra")
returnValue = server.TestHTTP(t, "GET", rangereq, nil)
var retrievedKeys []string
if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
t.Errorf("Bad key range request unmarshal: %v\n", err)
}
if len(retrievedKeys) != 2 || retrievedKeys[1] != "mykey" && retrievedKeys[0] != "my2ndKey" {
t.Errorf("Bad key range request return. Expected: [%q,%q]. Got: %s\n",
key1, key2, string(returnValue))
}
// Commit the repo
if err = datastore.Commit(uuid2, "my 2nd commit msg", []string{"changed 2nd k/v"}); err != nil {
t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
}
// Make grandchild of root
uuid3, err := datastore.NewVersion(uuid2, "some child", nil)
if err != nil {
t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err)
}
// Delete the 2nd k/v
uuid3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid3, data.DataName(), key2)
server.TestHTTP(t, "DELETE", uuid3req, nil)
server.TestBadHTTP(t, "GET", uuid3req, nil)
// Make sure the 2nd k/v is correct for each of previous versions.
returnValue = server.TestHTTP(t, "GET", key2req, nil)
if string(returnValue) != value2 {
t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, value2, string(returnValue))
}
returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
if string(returnValue) != uuid2val {
t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
}
// Make a child
if err = datastore.Commit(uuid3, "my 3rd commit msg", []string{"deleted 2nd k/v"}); err != nil {
t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
}
uuid4, err := datastore.NewVersion(uuid3, "some child", nil)
if err != nil {
t.Fatalf("Unable to create new version off node %s: %v\n", uuid3, err)
}
// Change the 2nd k/v
uuid4val := "we are reintroducing this k/v"
uuid4req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid4, data.DataName(), key2)
server.TestHTTP(t, "POST", uuid4req, strings.NewReader(uuid4val))
if err = datastore.Commit(uuid4, "commit node 4", []string{"we modified stuff"}); err != nil {
t.Errorf("Unable to commit node %s: %v\n", uuid4, err)
}
// Make sure the 2nd k/v is correct for each of previous versions.
returnValue = server.TestHTTP(t, "GET", key2req, nil)
if string(returnValue) != value2 {
t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, value2, string(returnValue))
}
returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
if string(returnValue) != uuid2val {
t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
}
server.TestBadHTTP(t, "GET", uuid3req, nil)
returnValue = server.TestHTTP(t, "GET", uuid4req, nil)
if string(returnValue) != uuid4val {
t.Errorf("Error on fourth version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue))
}
// Let's try a merge!
// Make a child off the 2nd version from root.
uuid5, err := datastore.NewVersion(uuid2, "some child", nil)
if err != nil {
t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err)
}
// Store new stuff in 2nd k/v
uuid5val := "this is forked value"
uuid5req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid5, data.DataName(), key2)
server.TestHTTP(t, "POST", uuid5req, strings.NewReader(uuid5val))
returnValue = server.TestHTTP(t, "GET", uuid5req, nil)
if string(returnValue) != uuid5val {
t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid5val, string(returnValue))
}
// Commit node
if err = datastore.Commit(uuid5, "forked node", []string{"we modified stuff"}); err != nil {
t.Errorf("Unable to commit node %s: %v\n", uuid5, err)
}
// Should be able to merge using conflict-free (disjoint at key level) merge even though
// its conflicted. Will get lazy error on request.
badChild, err := datastore.Merge([]dvid.UUID{uuid4, uuid5}, "some child", datastore.MergeConflictFree)
if err != nil {
t.Errorf("Error doing merge: %v\n", err)
}
childreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, badChild, data.DataName(), key2)
server.TestBadHTTP(t, "GET", childreq, nil)
// Manually fix conflict: Branch, and then delete 2nd k/v and commit.
uuid6, err := datastore.NewVersion(uuid5, "some child", nil)
if err != nil {
t.Fatalf("Unable to create new version off node %s: %v\n", uuid5, err)
}
uuid6req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid6, data.DataName(), key2)
server.TestHTTP(t, "DELETE", uuid6req, nil)
server.TestBadHTTP(t, "GET", uuid6req, nil)
if err = datastore.Commit(uuid6, "deleted forked node 2nd k/v", []string{"we modified stuff"}); err != nil {
t.Errorf("Unable to commit node %s: %s\n", uuid6, err)
}
// Should now be able to correctly merge the two branches.
goodChild, err := datastore.Merge([]dvid.UUID{uuid4, uuid6}, "merging stuff", datastore.MergeConflictFree)
if err != nil {
t.Errorf("Error doing merge: %v\n", err)
}
// We should be able to see just the original uuid4 value of the 2nd k/v
childreq = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, goodChild, data.DataName(), key2)
returnValue = server.TestHTTP(t, "GET", childreq, nil)
if string(returnValue) != uuid4val {
t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid4val, string(returnValue))
}
// Apply the automatic conflict resolution using ordering.
payload := fmt.Sprintf(`{"data":["versiontest"],"parents":[%q,%q],"note":"automatic resolved merge"}`, uuid5, uuid4)
resolveReq := fmt.Sprintf("%srepo/%s/resolve", server.WebAPIPath, uuid4)
returnValue = server.TestHTTP(t, "POST", resolveReq, bytes.NewBufferString(payload))
resolveResp := struct {
Child dvid.UUID `json:"child"`
}{}
if err := json.Unmarshal(returnValue, &resolveResp); err != nil {
t.Fatalf("Can't parse return of resolve request: %s\n", string(returnValue))
}
// We should now see the uuid5 version of the 2nd k/v in the returned merged node.
childreq = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, resolveResp.Child, data.DataName(), key2)
returnValue = server.TestHTTP(t, "GET", childreq, nil)
if string(returnValue) != uuid5val {
t.Errorf("Error on auto merged child, key %q: expected %q, got %q\n", key2, uuid5val, string(returnValue))
}
// Introduce a child off root but don't add 2nd k/v to it.
uuid7, err := datastore.NewVersion(uuid, "2nd child off root", nil)
if err != nil {
t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err)
}
if err = datastore.Commit(uuid7, "useless node", []string{"we modified nothing!"}); err != nil {
t.Errorf("Unable to commit node %s: %v\n", uuid7, err)
}
// Now merge the previously merged node with the newly created "blank" child off root.
if err = datastore.Commit(goodChild, "this was a good merge", []string{}); err != nil {
t.Errorf("Unable to commit node %s: %v\n", goodChild, err)
}
merge2, err := datastore.Merge([]dvid.UUID{goodChild, uuid7}, "merging a useless path", datastore.MergeConflictFree)
if err != nil {
t.Errorf("Error doing merge: %v\n", err)
}
merge3, err := datastore.Merge([]dvid.UUID{uuid7, goodChild}, "merging a useless path in reverse order", datastore.MergeConflictFree)
if err != nil {
t.Errorf("Error doing merge: %v\n", err)
}
// We should still be conflict free since 2nd key in left parent path will take precedent over shared 2nd key
// in root. This tests our invalidation of ancestors.
toughreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, merge2, data.DataName(), key2)
returnValue = server.TestHTTP(t, "GET", toughreq, nil)
if string(returnValue) != uuid4val {
t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid4val, string(returnValue))
}
toughreq = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, merge3, data.DataName(), key2)
returnValue = server.TestHTTP(t, "GET", toughreq, nil)
if string(returnValue) != uuid4val {
t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid4val, string(returnValue))
}
}
func testRequest(t *testing.T, uuid dvid.UUID, versionID dvid.VersionID, name dvid.InstanceName) {
config := dvid.NewConfig()
dataservice, err := datastore.NewData(uuid, kvtype, name, config)
if err != nil {
t.Fatalf("Error creating new keyvalue instance: %v\n", err)
}
data, ok := dataservice.(*Data)
if !ok {
t.Fatalf("Returned new data instance is not roi.Data\n")
}
// PUT a value
key1 := "mykey"
value1 := "some stuff"
key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1)
server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
// Get back k/v
returnValue := server.TestHTTP(t, "GET", key1req, nil)
if string(returnValue) != value1 {
t.Errorf("Error on key %q: expected %s, got %s\n", key1, value1, string(returnValue))
}
// Expect error if no key used.
badrequest := fmt.Sprintf("%snode/%s/%s/key/", server.WebAPIPath, uuid, data.DataName())
server.TestBadHTTP(t, "GET", badrequest, nil)
// Add 2nd k/v
key2 := "my2ndkey"
value2 := "more good stuff"
key2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key2)
server.TestHTTP(t, "POST", key2req, strings.NewReader(value2))
// Add 3rd k/v
key3 := "heresanotherkey"
value3 := "my 3rd value"
key3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key3)
server.TestHTTP(t, "POST", key3req, strings.NewReader(value3))
// Check return of first two keys in range.
rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(),
"my", "zebra")
returnValue = server.TestHTTP(t, "GET", rangereq, nil)
var retrievedKeys []string
if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
t.Errorf("Bad key range request unmarshal: %v\n", err)
}
if len(retrievedKeys) != 2 || retrievedKeys[1] != "mykey" && retrievedKeys[0] != "my2ndKey" {
t.Errorf("Bad key range request return. Expected: [%q,%q]. Got: %s\n",
key2, key1, string(returnValue))
}
// Check return of all keys
allkeyreq := fmt.Sprintf("%snode/%s/%s/keys", server.WebAPIPath, uuid, data.DataName())
returnValue = server.TestHTTP(t, "GET", allkeyreq, nil)
if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
t.Errorf("Bad key range request unmarshal: %v\n", err)
}
if len(retrievedKeys) != 3 || retrievedKeys[0] != "heresanotherkey" && retrievedKeys[1] != "my2ndKey" && retrievedKeys[2] != "mykey" {
t.Errorf("Bad all key request return. Expected: [%q,%q,%q]. Got: %s\n",
key3, key2, key1, string(returnValue))
}
}
func TestKeyvalueUnversioned(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
uuid, _ := initTestRepo()
config := dvid.NewConfig()
config.Set("versioned", "false")
dataservice, err := datastore.NewData(uuid, kvtype, "unversiontest", config)
if err != nil {
t.Fatalf("Error creating new keyvalue instance: %v\n", err)
}
data, ok := dataservice.(*Data)
if !ok {
t.Fatalf("Returned new data instance is not roi.Data\n")
}
// PUT a value
key1 := "mykey"
value1 := "some stuff"
key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1)
server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
// Add 2nd k/v
key2 := "my2ndkey"
value2 := "more good stuff"
key2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key2)
server.TestHTTP(t, "POST", key2req, strings.NewReader(value2))
// Create a new version in repo
if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil {
t.Errorf("Unable to lock root node %s: %v\n", uuid, err)
}
uuid2, err := datastore.NewVersion(uuid, "some child", nil)
if err != nil {
t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err)
}
_, err = datastore.VersionFromUUID(uuid2)
if err != nil {
t.Fatalf("Unable to get version ID from new uuid %s: %v\n", uuid2, err)
}
// Change the 2nd k/v
uuid2val := "this is completely different"
uuid2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid2, data.DataName(), key2)
server.TestHTTP(t, "POST", uuid2req, strings.NewReader(uuid2val))
// Now the first version value should equal the new value
returnValue := server.TestHTTP(t, "GET", key2req, nil)
if string(returnValue) != uuid2val {
t.Errorf("Error on unversioned key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
}
// Get the second version value
returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
if string(returnValue) != uuid2val {
t.Errorf("Error on unversioned key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
}
// Check return of first two keys in range.
rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(),
"my", "zebra")
returnValue = server.TestHTTP(t, "GET", rangereq, nil)
var retrievedKeys []string
if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
t.Errorf("Bad key range request unmarshal: %v\n", err)
}
if len(retrievedKeys) != 2 || retrievedKeys[1] != "mykey" && retrievedKeys[0] != "my2ndKey" {
t.Errorf("Bad key range request return. Expected: [%q,%q]. Got: %s\n",
key1, key2, string(returnValue))
}
// Commit the repo
if err = datastore.Commit(uuid2, "my 2nd commit msg", []string{"changed 2nd k/v"}); err != nil {
t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
}
// Make grandchild of root
uuid3, err := datastore.NewVersion(uuid2, "some child", nil)
if err != nil {
t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err)
}
// Delete the 2nd k/v
uuid3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid3, data.DataName(), key2)
server.TestHTTP(t, "DELETE", uuid3req, nil)
server.TestBadHTTP(t, "GET", uuid3req, nil)
// Make sure the 2nd k/v is now missing for previous versions.
server.TestBadHTTP(t, "GET", key2req, nil)
server.TestBadHTTP(t, "GET", uuid2req, nil)
// Make a child
if err = datastore.Commit(uuid3, "my 3rd commit msg", []string{"deleted 2nd k/v"}); err != nil {
t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
}
uuid4, err := datastore.NewVersion(uuid3, "some child", nil)
if err != nil {
t.Fatalf("Unable to create new version off node %s: %v\n", uuid3, err)
}
// Change the 2nd k/v
uuid4val := "we are reintroducing this k/v"
uuid4req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid4, data.DataName(), key2)
server.TestHTTP(t, "POST", uuid4req, strings.NewReader(uuid4val))
if err = datastore.Commit(uuid4, "commit node 4", []string{"we modified stuff"}); err != nil {
t.Errorf("Unable to commit node %s: %v\n", uuid4, err)
}
// Make sure the 2nd k/v is correct for each of previous versions.
returnValue = server.TestHTTP(t, "GET", key2req, nil)
if string(returnValue) != uuid4val {
t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue))
}
returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
if string(returnValue) != uuid4val {
t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue))
}
returnValue = server.TestHTTP(t, "GET", uuid3req, nil)
if string(returnValue) != uuid4val {
t.Errorf("Error on third version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue))
}
returnValue = server.TestHTTP(t, "GET", uuid4req, nil)
if string(returnValue) != uuid4val {
t.Errorf("Error on fourth version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue))
}
}
func TestLabels(t *testing.T) {
datastore.OpenTest()
defer datastore.CloseTest()
// Create testbed volume and data instances
uuid, _ := initTestRepo()
var config dvid.Config
server.CreateTestInstance(t, uuid, "labelblk", "labels", config)
server.CreateTestInstance(t, uuid, "labelvol", "bodies", config)
// Establish syncs
server.CreateTestSync(t, uuid, "labels", "bodies")
server.CreateTestSync(t, uuid, "bodies", "labels")
// Populate the labels, which should automatically populate the labelvol
_ = createLabelTestVolume(t, uuid, "labels")
if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
t.Fatalf("Error blocking on sync of labels: %v\n", err)
}
// Add annotations syncing with "labels" instance checking for deduplication.
server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config)
server.CreateTestSync(t, uuid, "mysynapses", "labels,bodies,labels,bodies,labels,bodies")
dataservice, err := datastore.GetDataByUUIDName(uuid, "mysynapses")
if err != nil {
t.Fatal(err)
}
data, ok := dataservice.(*Data)
if !ok {
t.Fatalf("Can't convert dataservice %v into datastore.Data\n", dataservice)
}
if len(data.SyncedData()) != 2 {
t.Fatalf("Expected 2 syncs (uuids for labels and bodies], got %v\n", data.SyncedData())
}
// PUT first batch of synapses
testJSON, err := json.Marshal(testData)
if err != nil {
t.Fatal(err)
}
url1 := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid)
server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
// Test if labels were properly denormalized. For the POST we have synchronized label denormalization.
// If this were to become asynchronous, we'd want to block on updating like the labelblk<->labelvol sync.
testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel2, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel3, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel3NoRel, "%snode/%s/mysynapses/label/3", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
// Make change to labelblk and make sure our label synapses have been adjusted (case A)
_ = modifyLabelTestVolume(t, uuid, "labels")
if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
t.Fatalf("Error blocking on sync of labels->annotations: %v\n", err)
}
testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel2a, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel3a, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel4NoRel, "%snode/%s/mysynapses/label/4", server.WebAPIPath, uuid)
// Make change to labelvol and make sure our label synapses have been adjusted (case B).
// Merge 3a into 2a.
testMerge := mergeJSON(`[2, 3]`)
testMerge.send(t, uuid, "bodies")
if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
t.Fatalf("Error blocking on sync of labels: %v\n", err)
}
if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
t.Fatalf("Error blocking on sync of synapses: %v\n", err)
}
testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel2b, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, nil, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
// Now split label 2b off and check if annotations also split
// Create the sparsevol encoding for split area
numspans := len(bodysplit.voxelSpans)
rles := make(dvid.RLEs, numspans, numspans)
for i, span := range bodysplit.voxelSpans {
start := dvid.Point3d{span[2], span[1], span[0]}
length := span[3] - span[2] + 1
rles[i] = dvid.NewRLE(start, length)
}
buf := getBytesRLE(t, rles)
// Submit the split sparsevol
reqStr := fmt.Sprintf("%snode/%s/%s/split/%d?splitlabel=7", server.WebAPIPath, uuid, "bodies", 2)
r := server.TestHTTP(t, "POST", reqStr, buf)
jsonVal := make(map[string]uint64)
if err := json.Unmarshal(r, &jsonVal); err != nil {
t.Errorf("Unable to get new label from split. Instead got: %v\n", jsonVal)
}
// Verify that the annotations are correct.
if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
t.Fatalf("Error blocking on sync of split->annotations: %v\n", err)
}
testResponseLabel(t, expectedLabel2c, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
url2 := fmt.Sprintf("%snode/%s/mysynapses/label/7?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, expectedLabel7, url2)
// Change the name of the annotations.
if err = datastore.RenameData(uuid, "mysynapses", "bodies", "foobar"); err == nil {
t.Fatalf("Should have been prevented from renaming data 'mysynapses' to existing data 'bodies'!\n")
}
if err = datastore.RenameData(uuid, "mysynapses", "renamedData", "foobar"); err != nil {
t.Fatalf("Error renaming annotations: %v\n", err)
}
// Make sure the old name is no longer there and the new one is.
server.TestBadHTTP(t, "GET", url2, nil)
testResponseLabel(t, expectedLabel2c, "%snode/%s/renamedData/label/2?relationships=true", server.WebAPIPath, uuid)
// Try a coarse split.
// Create the encoding for split area in block coordinates.
rles = dvid.RLEs{
dvid.NewRLE(dvid.Point3d{3, 1, 3}, 1),
}
buf = getBytesRLE(t, rles)
// Submit the coarse split
reqStr = fmt.Sprintf("%snode/%s/%s/split-coarse/2?splitlabel=8", server.WebAPIPath, uuid, "bodies")
r = server.TestHTTP(t, "POST", reqStr, buf)
jsonVal = make(map[string]uint64)
if err := json.Unmarshal(r, &jsonVal); err != nil {
t.Errorf("Unable to get new label from split. Instead got: %v\n", jsonVal)
}
// Verify that the annotations are correct.
if err := datastore.BlockOnUpdating(uuid, "renamedData"); err != nil {
t.Fatalf("Error blocking on sync of split->annotations: %v\n", err)
}
testResponseLabel(t, expectedLabel2c, "%snode/%s/renamedData/label/8?relationships=true", server.WebAPIPath, uuid)
testResponseLabel(t, nil, "%snode/%s/renamedData/label/2?relationships=true", server.WebAPIPath, uuid)
// Delete a labeled annotation and make sure it's not in label
delurl := fmt.Sprintf("%snode/%s/%s/element/20_30_40", server.WebAPIPath, uuid, "renamedData")
server.TestHTTP(t, "DELETE", delurl, nil)
testResponseLabel(t, afterDeleteOn7, "%snode/%s/%s/label/7?relationships=true", server.WebAPIPath, uuid, "renamedData")
}