func getBytesRLE(t *testing.T, rles dvid.RLEs) *bytes.Buffer { n := len(rles) buf := new(bytes.Buffer) buf.WriteByte(dvid.EncodingBinary) binary.Write(buf, binary.LittleEndian, uint8(3)) // # of dimensions binary.Write(buf, binary.LittleEndian, byte(0)) // dimension of run (X = 0) buf.WriteByte(byte(0)) // reserved for later binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # voxels binary.Write(buf, binary.LittleEndian, uint32(n)) // Placeholder for # spans rleBytes, err := rles.MarshalBinary() if err != nil { t.Errorf("Unable to serialize RLEs: %v\n", err) } buf.Write(rleBytes) return buf }
// PutSparseVol stores an encoded sparse volume that stays within a given forward label. // Note that this encoded sparse volume is added to any existing sparse volume for // a body rather than replace it. To carve out a part of an existing sparse volume, // use the SplitLabels(). // // This function handles modification/deletion of all denormalized data touched by this // sparse label volume. // // EVENTS // // func (d *Data) PutSparseVol(v dvid.VersionID, label uint64, r io.Reader) error { store, err := storage.SmallDataStore() if err != nil { return fmt.Errorf("Data type labelvol had error initializing store: %v\n", err) } batcher, ok := store.(storage.KeyValueBatcher) if !ok { return fmt.Errorf("Data type labelvol requires batch-enabled store, which %q is not\n", store) } // Mark the label as dirty until done. iv := dvid.InstanceVersion{d.DataName(), v} dirtyLabels.Incr(iv, label) defer dirtyLabels.Decr(iv, label) // TODO -- Signal that we are starting a label modification. // Read the sparse volume from reader. header := make([]byte, 8) if _, err = io.ReadFull(r, header); err != nil { return err } if header[0] != dvid.EncodingBinary { return fmt.Errorf("sparse vol for split has unknown encoding format: %v", header[0]) } var numSpans uint32 if err = binary.Read(r, binary.LittleEndian, &numSpans); err != nil { return err } var mods dvid.RLEs if err = mods.UnmarshalBinaryReader(r, numSpans); err != nil { return err } // Partition the mods spans into blocks. var modmap dvid.BlockRLEs modmap, err = mods.Partition(d.BlockSize) if err != nil { return err } // Publish sparsevol mod event evt := datastore.SyncEvent{d.DataName(), labels.SparsevolModEvent} msg := datastore.SyncMessage{v, labels.DeltaSparsevol{label, modmap}} if err := datastore.NotifySubscribers(evt, msg); err != nil { return err } // Get a sorted list of blocks that cover mods. modblks := modmap.SortedKeys() ctx := datastore.NewVersionedCtx(d, v) batch := batcher.NewBatch(ctx) var voxelsAdded int64 for _, modblk := range modblks { // Get original block tk := NewTKey(label, modblk) val, err := store.Get(ctx, tk) if err != nil { return err } // If there's no original block, just write the mod block. if val == nil || len(val) == 0 { numVoxels, _ := modmap[modblk].Stats() voxelsAdded += int64(numVoxels) rleBytes, err := modmap[modblk].MarshalBinary() if err != nil { return fmt.Errorf("can't serialize modified RLEs for %d: %v\n", label, err) } batch.Put(tk, rleBytes) continue } // if there's an original, integrate and write merged RLE. var rles dvid.RLEs if err := rles.UnmarshalBinary(val); err != nil { return fmt.Errorf("Unable to unmarshal RLE for original labels in block %s", modblk.Print()) } voxelsAdded += rles.Add(modmap[modblk]) rleBytes, err := rles.MarshalBinary() if err != nil { return fmt.Errorf("can't serialize modified RLEs of %d: %v\n", label, err) } batch.Put(tk, rleBytes) } if err := batch.Commit(); err != nil { return fmt.Errorf("Batch commit during mod of %s label %d: %v\n", d.DataName(), label, err) } // Publish change in label sizes. delta := labels.DeltaModSize{ Label: label, SizeChange: voxelsAdded, } evt = datastore.SyncEvent{d.DataName(), labels.ChangeSizeEvent} msg = datastore.SyncMessage{v, delta} if err := datastore.NotifySubscribers(evt, msg); err != nil { return err } // TODO -- Publish label mod end return nil }