Exemplo n.º 1
0
// Returns function that stores a tile as an optionally compressed PNG image.
func (d *Data) putTileFunc(versionID dvid.VersionID) (outFunc, error) {
	db, err := d.GetKeyValueDB()
	if err != nil {
		return nil, fmt.Errorf("Cannot open imagetile store: %v\n", err)
	}
	ctx := datastore.NewVersionedCtx(d, versionID)

	return func(req TileReq, tile *dvid.Image) error {
		var err error
		var data []byte

		switch d.Encoding {
		case LZ4:
			compression, err := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
			if err != nil {
				return err
			}
			data, err = tile.Serialize(compression, d.Checksum())
		case PNG:
			data, err = tile.GetPNG()
		case JPG:
			data, err = tile.GetJPEG(d.Quality)
		}
		if err != nil {
			return err
		}
		return db.Put(ctx, NewTKeyByTileReq(req), data)
	}, nil
}
Exemplo n.º 2
0
// Returns function that stores a tile as an optionally compressed PNG image.
func (d *Data) putTileFunc(versionID dvid.VersionID) (outFunc, error) {
	bigdata, err := storage.BigDataStore()
	if err != nil {
		return nil, fmt.Errorf("Cannot open big data store: %v\n", err)
	}
	ctx := datastore.NewVersionedCtx(d, versionID)

	return func(index *IndexTile, tile *dvid.Image) error {
		var err error
		var data []byte

		switch d.Encoding {
		case LZ4:
			compression, err := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
			if err != nil {
				return err
			}
			data, err = tile.Serialize(compression, d.Checksum())
		case PNG:
			data, err = tile.GetPNG()
		case JPG:
			data, err = tile.GetJPEG(d.Quality)
		}
		if err != nil {
			return err
		}
		return bigdata.Put(ctx, index.Bytes(), data)
	}, nil
}
Exemplo n.º 3
0
func TestDataGobEncoding(t *testing.T) {
	compression, _ := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
	data := &TestData{&Data{
		typename:    "testtype",
		typeurl:     "foo.bar.baz/testtype",
		typeversion: "1.0",
		id:          dvid.InstanceID(13),
		name:        "my fabulous data",
		rootUUID:    dvid.UUID("42"),
		dataUUID:    dvid.NewUUID(),
		compression: compression,
		checksum:    dvid.DefaultChecksum,
		syncData:    dvid.UUIDSet{"moo": struct{}{}, "bar": struct{}{}, "baz": struct{}{}},
	}}

	encoding, err := data.GobEncode()
	if err != nil {
		t.Fatalf("Couldn't Gob encode test data: %v\n", err)
	}
	data2 := &TestData{new(Data)}
	if err = data2.GobDecode(encoding); err != nil {
		t.Fatalf("Couldn't Gob decode test data: %v\n", err)
	}
	if !reflect.DeepEqual(data, data2) {
		t.Errorf("Bad Gob roundtrip:\nOriginal: %v\nReturned: %v\n", data, data2)
	}
}
Exemplo n.º 4
0
// serializeVertex serializes a dvid.GraphVertex (compression turned off for now)
func (db *GraphKeyValueDB) serializeVertex(vert dvid.GraphVertex) []byte {
	// encode: vertex id, vertex weight, num vertices, vertex array,
	// num properties, property array
	total_size := 24 + 8*len(vert.Vertices) + 8

	// find size for property strings (account for null character)
	for propname, _ := range vert.Properties {
		total_size += len(propname) + 1
	}

	// create byte buffer
	buf := make([]byte, total_size, total_size)

	// encode vertex id
	start := 0
	binary.LittleEndian.PutUint64(buf[start:], uint64(vert.Id))
	start += 8

	// encode vertex weight
	floatbits := math.Float64bits(vert.Weight)
	binary.LittleEndian.PutUint64(buf[start:], floatbits)
	start += 8

	// encode number of vertices
	binary.LittleEndian.PutUint64(buf[start:], uint64(len(vert.Vertices)))
	start += 8

	// encode vertex partners
	for _, vertpartner := range vert.Vertices {
		binary.LittleEndian.PutUint64(buf[start:], uint64(vertpartner))
		start += 8
	}

	// encode number of properties
	binary.LittleEndian.PutUint64(buf[start:], uint64(len(vert.Properties)))
	start += 8

	// encode property strings
	for propname, _ := range vert.Properties {
		for _, propchar := range propname {
			buf[start] = byte(propchar)
			start += 1
		}
		buf[start] = byte(0)
		start += 1
	}

	// encode DVID related info (currenlty compression is disabled)
	compression, _ := dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression)
	finalbuf, _ := dvid.SerializeData(buf, compression, dvid.NoChecksum)

	return finalbuf
}
Exemplo n.º 5
0
func (r *repoT) save() error {
	compression, err := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
	if err != nil {
		return err
	}
	serialization, err := dvid.Serialize(r, compression, dvid.CRC32)
	if err != nil {
		return err
	}

	var ctx storage.MetadataContext
	return manager.store.Put(ctx, storage.NewTKey(repoKey, r.id.Bytes()), serialization)
}
Exemplo n.º 6
0
// serializeEdge serializes a dvid.GraphEdge (compression turned off for now)
func (db *GraphKeyValueDB) serializeEdge(edge dvid.GraphEdge) []byte {
	// encode: vertex1 id, vertex2 id, weight,
	// num properties, property array
	total_size := 32

	// find size for property strings (account for null character)
	for propname, _ := range edge.Properties {
		total_size += len(propname) + 1
	}

	// create byte buffer
	buf := make([]byte, total_size, total_size)

	// encode vertex 1
	start := 0
	binary.LittleEndian.PutUint64(buf[start:], uint64(edge.Vertexpair.Vertex1))
	start += 8

	// encode vertex 2
	binary.LittleEndian.PutUint64(buf[start:], uint64(edge.Vertexpair.Vertex2))
	start += 8

	// encode weight
	floatbits := math.Float64bits(edge.Weight)
	binary.LittleEndian.PutUint64(buf[start:], floatbits)
	start += 8

	// encode number of properties
	binary.LittleEndian.PutUint64(buf[start:], uint64(len(edge.Properties)))
	start += 8

	// encode property strings
	for propname, _ := range edge.Properties {
		for _, propchar := range propname {
			buf[start] = byte(propchar)
			start += 1
		}
		buf[start] = byte(0)
		start += 1
	}

	// encode DVID related info (currenlty compression is disabled)
	compression, _ := dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression)
	finalbuf, _ := dvid.SerializeData(buf, compression, dvid.NoChecksum)

	return finalbuf
}
Exemplo n.º 7
0
// NewDataService returns a new Data instance that fulfills the DataService interface.
// The UUID passed in corresponds to the root UUID of the repo that should hold the data.
// This returned Data struct is usually embedded by datatype-specific data instances.
// By default, LZ4 and the default checksum is used.
func NewDataService(t TypeService, uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) {
	if _, reserved := reservedNames[string(name)]; reserved {
		return nil, fmt.Errorf("cannot use reserved name %q", name)
	}
	compression, _ := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
	data := &Data{
		typename:    t.GetTypeName(),
		typeurl:     t.GetTypeURL(),
		typeversion: t.GetTypeVersion(),
		name:        name,
		id:          id,
		uuid:        uuid,
		compression: compression,
		checksum:    dvid.DefaultChecksum,
		syncs:       []dvid.InstanceName{},
	}
	err := data.ModifyConfig(c)
	return data, err
}
Exemplo n.º 8
0
// NewDataService returns a new Data instance that fulfills the DataService interface.
// The UUID passed in corresponds to the root UUID of the DAG subgraph that should hold the data.
// This returned Data struct is usually embedded by datatype-specific data instances.
// By default, LZ4 and the default checksum is used.
func NewDataService(t TypeService, rootUUID dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) {
	if _, reserved := reservedNames[string(name)]; reserved {
		return nil, fmt.Errorf("cannot use reserved name %q", name)
	}

	// // Don't allow identical names to be used in the same repo.
	// d, err := GetDataByUUIDName(rootUUID, name)
	// if err == nil && d != nil {
	// 	return nil, fmt.Errorf("cannot create data instance %q when one already exists in repo with UUID %s", name, rootUUID)
	// }

	// See if a store was defined for a particular data instance.
	store, err := storage.GetAssignedStore(name, rootUUID, t.GetTypeName())
	if err != nil {
		return nil, err
	}

	// Make sure we generate a valid UUID for the data instance.
	dataUUID := dvid.NewUUID()
	if dataUUID == dvid.NilUUID {
		return nil, fmt.Errorf("Unable to generate new UUID for data %q creation", name)
	}

	// Setup the basic data instance structure.
	compression, _ := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
	data := &Data{
		typename:    t.GetTypeName(),
		typeurl:     t.GetTypeURL(),
		typeversion: t.GetTypeVersion(),
		dataUUID:    dataUUID,
		id:          id,
		name:        name,
		rootUUID:    rootUUID,
		compression: compression,
		checksum:    dvid.DefaultChecksum,
		syncNames:   []dvid.InstanceName{},
		syncData:    dvid.UUIDSet{},
		unversioned: false,
		store:       store,
	}
	return data, data.ModifyConfig(c)
}
Exemplo n.º 9
0
func (d *Data) ModifyConfig(config dvid.Config) error {
	// Set compression for this instance
	s, found, err := config.GetString("Compression")
	if err != nil {
		return err
	}
	if found {
		format := strings.ToLower(s)
		switch format {
		case "none":
			d.compression, _ = dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression)
		case "snappy":
			d.compression, _ = dvid.NewCompression(dvid.Snappy, dvid.DefaultCompression)
		case "lz4":
			d.compression, _ = dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
		case "gzip":
			d.compression, _ = dvid.NewCompression(dvid.Gzip, dvid.DefaultCompression)
		default:
			// Check for gzip + compression level
			parts := strings.Split(format, ":")
			if len(parts) == 2 && parts[0] == "gzip" {
				level, err := strconv.Atoi(parts[1])
				if err != nil {
					return fmt.Errorf("Unable to parse gzip compression level ('%d').  Should be 'gzip:<level>'.", parts[1])
				}
				d.compression, _ = dvid.NewCompression(dvid.Gzip, dvid.CompressionLevel(level))
			} else {
				return fmt.Errorf("Illegal compression specified: %s", s)
			}
		}
	}

	// Set checksum for this instance
	s, found, err = config.GetString("Checksum")
	if err != nil {
		return err
	}
	if found {
		checksum := strings.ToLower(s)
		switch checksum {
		case "none":
			d.checksum = dvid.NoChecksum
		case "crc32":
			d.checksum = dvid.CRC32
		default:
			return fmt.Errorf("Illegal checksum specified: %s", s)
		}
	}

	// Set data instances for syncing.
	s, found, err = config.GetString("sync")
	if err != nil {
		return err
	}
	if found {
		names := strings.Split(s, ",")
		if len(names) > 0 {
			for _, name := range names {
				d.syncs = append(d.syncs, dvid.InstanceName(name))
			}
		}
	}

	// Set versioning
	s, found, err = config.GetString("Versioned")
	if err != nil {
		return err
	}
	if found {
		versioned := strings.ToLower(s)
		switch versioned {
		case "false", "0":
			d.unversioned = true
		case "true", "1":
			d.unversioned = false
		default:
			return fmt.Errorf("Illegal setting for 'versioned' (needs to be 'false', '0', 'true', or '1'): %s", s)
		}
	}
	return nil
}