// Returns function that stores a tile as an optionally compressed PNG image. func (d *Data) putTileFunc(versionID dvid.VersionID) (outFunc, error) { db, err := d.GetKeyValueDB() if err != nil { return nil, fmt.Errorf("Cannot open imagetile store: %v\n", err) } ctx := datastore.NewVersionedCtx(d, versionID) return func(req TileReq, tile *dvid.Image) error { var err error var data []byte switch d.Encoding { case LZ4: compression, err := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression) if err != nil { return err } data, err = tile.Serialize(compression, d.Checksum()) case PNG: data, err = tile.GetPNG() case JPG: data, err = tile.GetJPEG(d.Quality) } if err != nil { return err } return db.Put(ctx, NewTKeyByTileReq(req), data) }, nil }
// Returns function that stores a tile as an optionally compressed PNG image. func (d *Data) putTileFunc(versionID dvid.VersionID) (outFunc, error) { bigdata, err := storage.BigDataStore() if err != nil { return nil, fmt.Errorf("Cannot open big data store: %v\n", err) } ctx := datastore.NewVersionedCtx(d, versionID) return func(index *IndexTile, tile *dvid.Image) error { var err error var data []byte switch d.Encoding { case LZ4: compression, err := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression) if err != nil { return err } data, err = tile.Serialize(compression, d.Checksum()) case PNG: data, err = tile.GetPNG() case JPG: data, err = tile.GetJPEG(d.Quality) } if err != nil { return err } return bigdata.Put(ctx, index.Bytes(), data) }, nil }
func TestDataGobEncoding(t *testing.T) { compression, _ := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression) data := &TestData{&Data{ typename: "testtype", typeurl: "foo.bar.baz/testtype", typeversion: "1.0", id: dvid.InstanceID(13), name: "my fabulous data", rootUUID: dvid.UUID("42"), dataUUID: dvid.NewUUID(), compression: compression, checksum: dvid.DefaultChecksum, syncData: dvid.UUIDSet{"moo": struct{}{}, "bar": struct{}{}, "baz": struct{}{}}, }} encoding, err := data.GobEncode() if err != nil { t.Fatalf("Couldn't Gob encode test data: %v\n", err) } data2 := &TestData{new(Data)} if err = data2.GobDecode(encoding); err != nil { t.Fatalf("Couldn't Gob decode test data: %v\n", err) } if !reflect.DeepEqual(data, data2) { t.Errorf("Bad Gob roundtrip:\nOriginal: %v\nReturned: %v\n", data, data2) } }
// serializeVertex serializes a dvid.GraphVertex (compression turned off for now) func (db *GraphKeyValueDB) serializeVertex(vert dvid.GraphVertex) []byte { // encode: vertex id, vertex weight, num vertices, vertex array, // num properties, property array total_size := 24 + 8*len(vert.Vertices) + 8 // find size for property strings (account for null character) for propname, _ := range vert.Properties { total_size += len(propname) + 1 } // create byte buffer buf := make([]byte, total_size, total_size) // encode vertex id start := 0 binary.LittleEndian.PutUint64(buf[start:], uint64(vert.Id)) start += 8 // encode vertex weight floatbits := math.Float64bits(vert.Weight) binary.LittleEndian.PutUint64(buf[start:], floatbits) start += 8 // encode number of vertices binary.LittleEndian.PutUint64(buf[start:], uint64(len(vert.Vertices))) start += 8 // encode vertex partners for _, vertpartner := range vert.Vertices { binary.LittleEndian.PutUint64(buf[start:], uint64(vertpartner)) start += 8 } // encode number of properties binary.LittleEndian.PutUint64(buf[start:], uint64(len(vert.Properties))) start += 8 // encode property strings for propname, _ := range vert.Properties { for _, propchar := range propname { buf[start] = byte(propchar) start += 1 } buf[start] = byte(0) start += 1 } // encode DVID related info (currenlty compression is disabled) compression, _ := dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression) finalbuf, _ := dvid.SerializeData(buf, compression, dvid.NoChecksum) return finalbuf }
func (r *repoT) save() error { compression, err := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression) if err != nil { return err } serialization, err := dvid.Serialize(r, compression, dvid.CRC32) if err != nil { return err } var ctx storage.MetadataContext return manager.store.Put(ctx, storage.NewTKey(repoKey, r.id.Bytes()), serialization) }
// serializeEdge serializes a dvid.GraphEdge (compression turned off for now) func (db *GraphKeyValueDB) serializeEdge(edge dvid.GraphEdge) []byte { // encode: vertex1 id, vertex2 id, weight, // num properties, property array total_size := 32 // find size for property strings (account for null character) for propname, _ := range edge.Properties { total_size += len(propname) + 1 } // create byte buffer buf := make([]byte, total_size, total_size) // encode vertex 1 start := 0 binary.LittleEndian.PutUint64(buf[start:], uint64(edge.Vertexpair.Vertex1)) start += 8 // encode vertex 2 binary.LittleEndian.PutUint64(buf[start:], uint64(edge.Vertexpair.Vertex2)) start += 8 // encode weight floatbits := math.Float64bits(edge.Weight) binary.LittleEndian.PutUint64(buf[start:], floatbits) start += 8 // encode number of properties binary.LittleEndian.PutUint64(buf[start:], uint64(len(edge.Properties))) start += 8 // encode property strings for propname, _ := range edge.Properties { for _, propchar := range propname { buf[start] = byte(propchar) start += 1 } buf[start] = byte(0) start += 1 } // encode DVID related info (currenlty compression is disabled) compression, _ := dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression) finalbuf, _ := dvid.SerializeData(buf, compression, dvid.NoChecksum) return finalbuf }
// NewDataService returns a new Data instance that fulfills the DataService interface. // The UUID passed in corresponds to the root UUID of the repo that should hold the data. // This returned Data struct is usually embedded by datatype-specific data instances. // By default, LZ4 and the default checksum is used. func NewDataService(t TypeService, uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) { if _, reserved := reservedNames[string(name)]; reserved { return nil, fmt.Errorf("cannot use reserved name %q", name) } compression, _ := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression) data := &Data{ typename: t.GetTypeName(), typeurl: t.GetTypeURL(), typeversion: t.GetTypeVersion(), name: name, id: id, uuid: uuid, compression: compression, checksum: dvid.DefaultChecksum, syncs: []dvid.InstanceName{}, } err := data.ModifyConfig(c) return data, err }
// NewDataService returns a new Data instance that fulfills the DataService interface. // The UUID passed in corresponds to the root UUID of the DAG subgraph that should hold the data. // This returned Data struct is usually embedded by datatype-specific data instances. // By default, LZ4 and the default checksum is used. func NewDataService(t TypeService, rootUUID dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) { if _, reserved := reservedNames[string(name)]; reserved { return nil, fmt.Errorf("cannot use reserved name %q", name) } // // Don't allow identical names to be used in the same repo. // d, err := GetDataByUUIDName(rootUUID, name) // if err == nil && d != nil { // return nil, fmt.Errorf("cannot create data instance %q when one already exists in repo with UUID %s", name, rootUUID) // } // See if a store was defined for a particular data instance. store, err := storage.GetAssignedStore(name, rootUUID, t.GetTypeName()) if err != nil { return nil, err } // Make sure we generate a valid UUID for the data instance. dataUUID := dvid.NewUUID() if dataUUID == dvid.NilUUID { return nil, fmt.Errorf("Unable to generate new UUID for data %q creation", name) } // Setup the basic data instance structure. compression, _ := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression) data := &Data{ typename: t.GetTypeName(), typeurl: t.GetTypeURL(), typeversion: t.GetTypeVersion(), dataUUID: dataUUID, id: id, name: name, rootUUID: rootUUID, compression: compression, checksum: dvid.DefaultChecksum, syncNames: []dvid.InstanceName{}, syncData: dvid.UUIDSet{}, unversioned: false, store: store, } return data, data.ModifyConfig(c) }
func (d *Data) ModifyConfig(config dvid.Config) error { // Set compression for this instance s, found, err := config.GetString("Compression") if err != nil { return err } if found { format := strings.ToLower(s) switch format { case "none": d.compression, _ = dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression) case "snappy": d.compression, _ = dvid.NewCompression(dvid.Snappy, dvid.DefaultCompression) case "lz4": d.compression, _ = dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression) case "gzip": d.compression, _ = dvid.NewCompression(dvid.Gzip, dvid.DefaultCompression) default: // Check for gzip + compression level parts := strings.Split(format, ":") if len(parts) == 2 && parts[0] == "gzip" { level, err := strconv.Atoi(parts[1]) if err != nil { return fmt.Errorf("Unable to parse gzip compression level ('%d'). Should be 'gzip:<level>'.", parts[1]) } d.compression, _ = dvid.NewCompression(dvid.Gzip, dvid.CompressionLevel(level)) } else { return fmt.Errorf("Illegal compression specified: %s", s) } } } // Set checksum for this instance s, found, err = config.GetString("Checksum") if err != nil { return err } if found { checksum := strings.ToLower(s) switch checksum { case "none": d.checksum = dvid.NoChecksum case "crc32": d.checksum = dvid.CRC32 default: return fmt.Errorf("Illegal checksum specified: %s", s) } } // Set data instances for syncing. s, found, err = config.GetString("sync") if err != nil { return err } if found { names := strings.Split(s, ",") if len(names) > 0 { for _, name := range names { d.syncs = append(d.syncs, dvid.InstanceName(name)) } } } // Set versioning s, found, err = config.GetString("Versioned") if err != nil { return err } if found { versioned := strings.ToLower(s) switch versioned { case "false", "0": d.unversioned = true case "true", "1": d.unversioned = false default: return fmt.Errorf("Illegal setting for 'versioned' (needs to be 'false', '0', 'true', or '1'): %s", s) } } return nil }