// Returns function that stores a tile as an optionally compressed PNG image.
func (d *Data) putTileFunc(versionID dvid.VersionID) (outFunc, error) {
db, err := d.GetKeyValueDB()
if err != nil {
return nil, fmt.Errorf("Cannot open imagetile store: %v\n", err)
}
ctx := datastore.NewVersionedCtx(d, versionID)
return func(req TileReq, tile *dvid.Image) error {
var err error
var data []byte
switch d.Encoding {
case LZ4:
compression, err := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
if err != nil {
return err
}
data, err = tile.Serialize(compression, d.Checksum())
case PNG:
data, err = tile.GetPNG()
case JPG:
data, err = tile.GetJPEG(d.Quality)
}
if err != nil {
return err
}
return db.Put(ctx, NewTKeyByTileReq(req), data)
}, nil
}
// Returns function that stores a tile as an optionally compressed PNG image.
func (d *Data) putTileFunc(versionID dvid.VersionID) (outFunc, error) {
bigdata, err := storage.BigDataStore()
if err != nil {
return nil, fmt.Errorf("Cannot open big data store: %v\n", err)
}
ctx := datastore.NewVersionedCtx(d, versionID)
return func(index *IndexTile, tile *dvid.Image) error {
var err error
var data []byte
switch d.Encoding {
case LZ4:
compression, err := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
if err != nil {
return err
}
data, err = tile.Serialize(compression, d.Checksum())
case PNG:
data, err = tile.GetPNG()
case JPG:
data, err = tile.GetJPEG(d.Quality)
}
if err != nil {
return err
}
return bigdata.Put(ctx, index.Bytes(), data)
}, nil
}
func TestDataGobEncoding(t *testing.T) {
compression, _ := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
data := &TestData{&Data{
typename: "testtype",
typeurl: "foo.bar.baz/testtype",
typeversion: "1.0",
id: dvid.InstanceID(13),
name: "my fabulous data",
rootUUID: dvid.UUID("42"),
dataUUID: dvid.NewUUID(),
compression: compression,
checksum: dvid.DefaultChecksum,
syncData: dvid.UUIDSet{"moo": struct{}{}, "bar": struct{}{}, "baz": struct{}{}},
}}
encoding, err := data.GobEncode()
if err != nil {
t.Fatalf("Couldn't Gob encode test data: %v\n", err)
}
data2 := &TestData{new(Data)}
if err = data2.GobDecode(encoding); err != nil {
t.Fatalf("Couldn't Gob decode test data: %v\n", err)
}
if !reflect.DeepEqual(data, data2) {
t.Errorf("Bad Gob roundtrip:\nOriginal: %v\nReturned: %v\n", data, data2)
}
}
// serializeVertex serializes a dvid.GraphVertex (compression turned off for now)
func (db *GraphKeyValueDB) serializeVertex(vert dvid.GraphVertex) []byte {
// encode: vertex id, vertex weight, num vertices, vertex array,
// num properties, property array
total_size := 24 + 8*len(vert.Vertices) + 8
// find size for property strings (account for null character)
for propname, _ := range vert.Properties {
total_size += len(propname) + 1
}
// create byte buffer
buf := make([]byte, total_size, total_size)
// encode vertex id
start := 0
binary.LittleEndian.PutUint64(buf[start:], uint64(vert.Id))
start += 8
// encode vertex weight
floatbits := math.Float64bits(vert.Weight)
binary.LittleEndian.PutUint64(buf[start:], floatbits)
start += 8
// encode number of vertices
binary.LittleEndian.PutUint64(buf[start:], uint64(len(vert.Vertices)))
start += 8
// encode vertex partners
for _, vertpartner := range vert.Vertices {
binary.LittleEndian.PutUint64(buf[start:], uint64(vertpartner))
start += 8
}
// encode number of properties
binary.LittleEndian.PutUint64(buf[start:], uint64(len(vert.Properties)))
start += 8
// encode property strings
for propname, _ := range vert.Properties {
for _, propchar := range propname {
buf[start] = byte(propchar)
start += 1
}
buf[start] = byte(0)
start += 1
}
// encode DVID related info (currenlty compression is disabled)
compression, _ := dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression)
finalbuf, _ := dvid.SerializeData(buf, compression, dvid.NoChecksum)
return finalbuf
}
func (r *repoT) save() error {
compression, err := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
if err != nil {
return err
}
serialization, err := dvid.Serialize(r, compression, dvid.CRC32)
if err != nil {
return err
}
var ctx storage.MetadataContext
return manager.store.Put(ctx, storage.NewTKey(repoKey, r.id.Bytes()), serialization)
}
// serializeEdge serializes a dvid.GraphEdge (compression turned off for now)
func (db *GraphKeyValueDB) serializeEdge(edge dvid.GraphEdge) []byte {
// encode: vertex1 id, vertex2 id, weight,
// num properties, property array
total_size := 32
// find size for property strings (account for null character)
for propname, _ := range edge.Properties {
total_size += len(propname) + 1
}
// create byte buffer
buf := make([]byte, total_size, total_size)
// encode vertex 1
start := 0
binary.LittleEndian.PutUint64(buf[start:], uint64(edge.Vertexpair.Vertex1))
start += 8
// encode vertex 2
binary.LittleEndian.PutUint64(buf[start:], uint64(edge.Vertexpair.Vertex2))
start += 8
// encode weight
floatbits := math.Float64bits(edge.Weight)
binary.LittleEndian.PutUint64(buf[start:], floatbits)
start += 8
// encode number of properties
binary.LittleEndian.PutUint64(buf[start:], uint64(len(edge.Properties)))
start += 8
// encode property strings
for propname, _ := range edge.Properties {
for _, propchar := range propname {
buf[start] = byte(propchar)
start += 1
}
buf[start] = byte(0)
start += 1
}
// encode DVID related info (currenlty compression is disabled)
compression, _ := dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression)
finalbuf, _ := dvid.SerializeData(buf, compression, dvid.NoChecksum)
return finalbuf
}
// NewDataService returns a new Data instance that fulfills the DataService interface.
// The UUID passed in corresponds to the root UUID of the repo that should hold the data.
// This returned Data struct is usually embedded by datatype-specific data instances.
// By default, LZ4 and the default checksum is used.
func NewDataService(t TypeService, uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) {
if _, reserved := reservedNames[string(name)]; reserved {
return nil, fmt.Errorf("cannot use reserved name %q", name)
}
compression, _ := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
data := &Data{
typename: t.GetTypeName(),
typeurl: t.GetTypeURL(),
typeversion: t.GetTypeVersion(),
name: name,
id: id,
uuid: uuid,
compression: compression,
checksum: dvid.DefaultChecksum,
syncs: []dvid.InstanceName{},
}
err := data.ModifyConfig(c)
return data, err
}
// NewDataService returns a new Data instance that fulfills the DataService interface.
// The UUID passed in corresponds to the root UUID of the DAG subgraph that should hold the data.
// This returned Data struct is usually embedded by datatype-specific data instances.
// By default, LZ4 and the default checksum is used.
func NewDataService(t TypeService, rootUUID dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) {
if _, reserved := reservedNames[string(name)]; reserved {
return nil, fmt.Errorf("cannot use reserved name %q", name)
}
// // Don't allow identical names to be used in the same repo.
// d, err := GetDataByUUIDName(rootUUID, name)
// if err == nil && d != nil {
// return nil, fmt.Errorf("cannot create data instance %q when one already exists in repo with UUID %s", name, rootUUID)
// }
// See if a store was defined for a particular data instance.
store, err := storage.GetAssignedStore(name, rootUUID, t.GetTypeName())
if err != nil {
return nil, err
}
// Make sure we generate a valid UUID for the data instance.
dataUUID := dvid.NewUUID()
if dataUUID == dvid.NilUUID {
return nil, fmt.Errorf("Unable to generate new UUID for data %q creation", name)
}
// Setup the basic data instance structure.
compression, _ := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
data := &Data{
typename: t.GetTypeName(),
typeurl: t.GetTypeURL(),
typeversion: t.GetTypeVersion(),
dataUUID: dataUUID,
id: id,
name: name,
rootUUID: rootUUID,
compression: compression,
checksum: dvid.DefaultChecksum,
syncNames: []dvid.InstanceName{},
syncData: dvid.UUIDSet{},
unversioned: false,
store: store,
}
return data, data.ModifyConfig(c)
}
func (d *Data) ModifyConfig(config dvid.Config) error {
// Set compression for this instance
s, found, err := config.GetString("Compression")
if err != nil {
return err
}
if found {
format := strings.ToLower(s)
switch format {
case "none":
d.compression, _ = dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression)
case "snappy":
d.compression, _ = dvid.NewCompression(dvid.Snappy, dvid.DefaultCompression)
case "lz4":
d.compression, _ = dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
case "gzip":
d.compression, _ = dvid.NewCompression(dvid.Gzip, dvid.DefaultCompression)
default:
// Check for gzip + compression level
parts := strings.Split(format, ":")
if len(parts) == 2 && parts[0] == "gzip" {
level, err := strconv.Atoi(parts[1])
if err != nil {
return fmt.Errorf("Unable to parse gzip compression level ('%d'). Should be 'gzip:<level>'.", parts[1])
}
d.compression, _ = dvid.NewCompression(dvid.Gzip, dvid.CompressionLevel(level))
} else {
return fmt.Errorf("Illegal compression specified: %s", s)
}
}
}
// Set checksum for this instance
s, found, err = config.GetString("Checksum")
if err != nil {
return err
}
if found {
checksum := strings.ToLower(s)
switch checksum {
case "none":
d.checksum = dvid.NoChecksum
case "crc32":
d.checksum = dvid.CRC32
default:
return fmt.Errorf("Illegal checksum specified: %s", s)
}
}
// Set data instances for syncing.
s, found, err = config.GetString("sync")
if err != nil {
return err
}
if found {
names := strings.Split(s, ",")
if len(names) > 0 {
for _, name := range names {
d.syncs = append(d.syncs, dvid.InstanceName(name))
}
}
}
// Set versioning
s, found, err = config.GetString("Versioned")
if err != nil {
return err
}
if found {
versioned := strings.ToLower(s)
switch versioned {
case "false", "0":
d.unversioned = true
case "true", "1":
d.unversioned = false
default:
return fmt.Errorf("Illegal setting for 'versioned' (needs to be 'false', '0', 'true', or '1'): %s", s)
}
}
return nil
}
