// deserializeVertex deserializes a dvid.GraphVertex (compression turned off for now)
func (db *GraphKeyValueDB) deserializeVertex(vertexdata []byte) (dvid.GraphVertex, error) {
// create vertex to be returned
vert := dvid.GraphVertex{GraphElement: &dvid.GraphElement{}}
// if vertexdata is empty return an error
if vertexdata == nil || len(vertexdata) == 0 {
return vert, fmt.Errorf("Vertex data empty")
}
// boilerplate deserialization from DVID
data, _, err := dvid.DeserializeData(vertexdata, true)
if err != nil {
return vert, err
}
// load data from vertex (vertex id, vertex weight, num vertices,
// vertex array, num properties, property array
// load vertex id
start := 0
vert.Id = dvid.VertexID(binary.LittleEndian.Uint64(data[start:]))
start += 8
// load vertex weight
floatbits := binary.LittleEndian.Uint64(data[start:])
vert.Weight = math.Float64frombits(floatbits)
start += 8
// number of vertices
count := binary.LittleEndian.Uint64(data[start:])
start += 8
vert.Vertices = make([]dvid.VertexID, count, count)
// load vertices
for i := uint64(0); i < count; i++ {
vert.Vertices[i] = dvid.VertexID(binary.LittleEndian.Uint64(data[start:]))
start += 8
}
// number of properties
vert.Properties = make(dvid.ElementProperties)
count = binary.LittleEndian.Uint64(data[start:])
start += 8
// create property strings
for i := uint64(0); i < count; i++ {
propertyname := string("")
// null separated strings
for data[start] != byte(0) {
propertyname += string(data[start])
start += 1
}
vert.Properties[propertyname] = struct{}{}
// increment beyond null
start += 1
}
return vert, nil
}
// deserializeEdge deserializes a dvid.GraphEdge (compression turned off for now)
func (db *GraphKeyValueDB) deserializeEdge(edgedata []byte) (dvid.GraphEdge, error) {
// create edge to be returned
edge := dvid.GraphEdge{GraphElement: &dvid.GraphElement{}}
// if edgedata is empty return an error
if edgedata == nil || len(edgedata) == 0 {
return edge, fmt.Errorf("Edge data empty")
}
// boilerplate deserialization from DVID
data, _, err := dvid.DeserializeData(edgedata, true)
if err != nil {
return edge, err
}
// load data from edge (vertex1 id, vertex2 id, edge weight,
// num properties, property array
// load vertex1 id
start := 0
edge.Vertexpair.Vertex1 = dvid.VertexID(binary.LittleEndian.Uint64(data[start:]))
start += 8
// load vertex2 id
edge.Vertexpair.Vertex2 = dvid.VertexID(binary.LittleEndian.Uint64(data[start:]))
start += 8
// load edge weight
floatbits := binary.LittleEndian.Uint64(data[start:])
edge.Weight = math.Float64frombits(floatbits)
start += 8
// number of properties
count := binary.LittleEndian.Uint64(data[start:])
start += 8
// create property strings
edge.Properties = make(dvid.ElementProperties)
for i := uint64(0); i < count; i++ {
propertyname := string("")
// null separated strings
for data[start] != 0 {
propertyname += string(data[start])
start += 1
}
edge.Properties[propertyname] = struct{}{}
// increment beyond null
start += 1
}
return edge, nil
}
// BytesToIndex returns an Index from a byte representation
// TODO -- Add error checking on slice format.
func (i *graphIndex) IndexFromBytes(b []byte) error {
keyType := graphType(b[0])
vertex1 := dvid.VertexID(binary.BigEndian.Uint64(b[1 : 1+vertexIDSize]))
vertex2 := dvid.VertexID(0)
start := 1 + vertexIDSize
if keyType == keyEdge || keyType == keyEdgeProperty {
vertex2 = dvid.VertexID(binary.BigEndian.Uint64(b[start : start+vertexIDSize]))
start += vertexIDSize
}
property := ""
if keyType == keyVertexProperty || keyType == keyEdgeProperty {
property = string(b[start:])
}
*i = graphIndex{keyType, vertex1, vertex2, property}
return nil
}
// createTransactionGroupBinary reads through binary representation of vertex / transaction id
// pairs and calls creatTransactionGroup
func (t *transactionLog) createTransactionGroupBinary(data []byte, readonly bool) (*transactionGroup, int, error) {
start := 0
numtrans := binary.LittleEndian.Uint64(data[start:])
start += 8
var vertices []transactionItem
for i := uint64(0); i < numtrans; i++ {
vertex := binary.LittleEndian.Uint64(data[start:])
start += 8
trans := binary.LittleEndian.Uint64(data[start:])
start += 8
vertices = append(vertices, transactionItem{dvid.VertexID(vertex), trans})
}
transaction_group, err := t.createTransactionGroup(vertices, readonly)
return transaction_group, start, err
}
// GetVertices uses a range query to get all vertices (#reads = #vertices)
func (db *GraphKeyValueDB) GetVertices(ctx Context) ([]dvid.GraphVertex, error) {
minid := dvid.VertexID(0)
maxid := ^minid
keylb := &graphIndex{keyVertex, minid, 0, ""}
keyub := &graphIndex{keyVertex, maxid, 0, ""}
keyvalues, err := db.GetRange(ctx, keylb.Bytes(), keyub.Bytes())
var vertexlist []dvid.GraphVertex
if err != nil {
return vertexlist, err
}
for _, keyvalue := range keyvalues {
vertex, err := db.deserializeVertex(keyvalue.V)
if err != nil {
return vertexlist, err
}
vertexlist = append(vertexlist, vertex)
}
return vertexlist, nil
}
// GetEdges uses a range query to get all edges (#reads = #edges)
func (db *GraphKeyValueDB) GetEdges(ctx Context) ([]dvid.GraphEdge, error) {
minid := dvid.VertexID(0)
maxid := ^minid
keylb := &graphIndex{keyEdge, minid, minid, ""}
keyub := &graphIndex{keyEdge, maxid, maxid, ""}
keyvalues, err := db.GetRange(ctx, keylb.Bytes(), keyub.Bytes())
var edgelist []dvid.GraphEdge
if err != nil {
return edgelist, err
}
for _, keyvalue := range keyvalues {
edge, err := db.deserializeEdge(keyvalue.V)
if err != nil {
return edgelist, err
}
edgelist = append(edgelist, edge)
}
return edgelist, nil
}
// handleNeighbors returns the vertices and edges connected to the provided vertex
// (Should I protect this transaction like the update weight function? It is probably
// unnecessary because any update based on retrieved information will be written to a
// property which has transactional protection)
func (d *Data) handleNeighbors(ctx *datastore.VersionedCtx, db storage.GraphDB, w http.ResponseWriter, path []string) error {
labelgraph := new(LabelGraph)
temp, err := strconv.Atoi(path[0])
if err != nil {
return fmt.Errorf("Vertex number not provided")
}
id := dvid.VertexID(temp)
storedvert, err := db.GetVertex(ctx, id)
if err != nil {
return fmt.Errorf("Failed to retrieve vertix %d: %v\n", id, err)
}
labelgraph.Vertices = append(labelgraph.Vertices, labelVertex{storedvert.Id, storedvert.Weight})
for _, vert2 := range storedvert.Vertices {
vertex, err := db.GetVertex(ctx, vert2)
if err != nil {
return fmt.Errorf("Failed to retrieve vertex %d: %v\n", vertex.Id, err)
}
labelgraph.Vertices = append(labelgraph.Vertices, labelVertex{vertex.Id, vertex.Weight})
edge, err := db.GetEdge(ctx, id, vert2)
if err != nil {
return fmt.Errorf("Failed to retrieve edge %d-%d: %v\n", vertex.Id, vert2, err)
}
labelgraph.Edges = append(labelgraph.Edges, labelEdge{edge.Vertexpair.Vertex1, edge.Vertexpair.Vertex2, edge.Weight})
}
m, err := json.Marshal(labelgraph)
if err != nil {
return fmt.Errorf("Could not serialize graph")
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(m))
return nil
}
// handleProperty retrieves or deletes properties that can be added to a vertex or edge -- data posted
// or retrieved uses default compression
func (d *Data) handleProperty(ctx *datastore.VersionedCtx, db storage.GraphDB, w http.ResponseWriter, r *http.Request, path []string, method string) error {
edgemode := false
var propertyname string
if len(path) == 3 {
edgemode = true
propertyname = path[2]
} else if len(path) != 2 {
return fmt.Errorf("Incorrect number of parameters specified for handling properties")
} else {
propertyname = path[1]
}
temp, err := strconv.Atoi(path[0])
if err != nil {
return fmt.Errorf("Vertex number not provided")
}
id1 := dvid.VertexID(temp)
id2 := dvid.VertexID(0)
if edgemode {
temp, err := strconv.Atoi(path[1])
if err != nil {
return fmt.Errorf("Vertex number not provided")
}
id2 = dvid.VertexID(temp)
}
// remove a property from a vertex or edge
if method == "delete" {
if edgemode {
db.RemoveEdgeProperty(ctx, id1, id2, propertyname)
if err != nil {
return fmt.Errorf("Failed to remove edge property %d-%d %s: %v\n", id1, id2, propertyname, err)
}
} else {
db.RemoveVertexProperty(ctx, id1, propertyname)
if err != nil {
return fmt.Errorf("Failed to remove vertex property %d %s: %v\n", id1, propertyname, err)
}
}
} else if method == "get" {
var data []byte
if edgemode {
data, err = db.GetEdgeProperty(ctx, id1, id2, propertyname)
} else {
data, err = db.GetVertexProperty(ctx, id1, propertyname)
}
if err != nil {
return fmt.Errorf("Failed to get property %s: %v\n", propertyname, err)
}
uncompress := true
value, _, e := dvid.DeserializeData(data, uncompress)
if e != nil {
err = fmt.Errorf("Unable to deserialize data for property '%s': %v\n", propertyname, e.Error())
return err
}
w.Header().Set("Content-Type", "application/octet-stream")
_, err = w.Write(value)
if err != nil {
return err
}
} else if method == "post" {
// read as binary and load into propertyname
data, err := ioutil.ReadAll(r.Body)
if err != nil {
return err
}
serialization, err := dvid.SerializeData(data, d.Compression(), d.Checksum())
if err != nil {
return fmt.Errorf("Unable to serialize data: %v\n", err)
}
if edgemode {
err = db.SetEdgeProperty(ctx, id1, id2, propertyname, serialization)
} else {
err = db.SetVertexProperty(ctx, id1, propertyname, serialization)
}
if err != nil {
return fmt.Errorf("Failed to add property %s: %v\n", propertyname, err)
}
}
return err
}
// handelPropertyTransaction allows gets/posts (really puts) of edge or vertex properties.
func (d *Data) handlePropertyTransaction(ctx *datastore.VersionedCtx, db storage.GraphDB, w http.ResponseWriter, r *http.Request, path []string, method string) error {
if len(path) < 2 {
return fmt.Errorf("Must specify edges or vertices in URI and property name")
}
if method == "delete" {
return fmt.Errorf("Transactional delete not supported")
}
edgemode := false
if path[0] == "edges" {
edgemode = true
} else if path[0] != "vertices" {
return fmt.Errorf("Must specify edges or vertices in URI")
}
propertyname := path[1]
readonly := false
if method == "get" {
readonly = true
}
data, err := ioutil.ReadAll(r.Body)
// only allow 1000 vertices to be locked
transactions, start, err := d.transaction_log.createTransactionGroupBinary(data, readonly)
defer transactions.closeTransaction()
if err != nil {
return fmt.Errorf("Failed to create property transaction: %v", err)
}
returned_data := transactions.exportTransactionsBinary()
if method == "post" {
// deserialize transaction (vertex or edge) -- use URI?
num_properties := binary.LittleEndian.Uint64(data[start:])
start += 8
for i := uint64(0); i < num_properties; i++ {
temp := binary.LittleEndian.Uint64(data[start:])
id := dvid.VertexID(temp)
var id2 dvid.VertexID
start += 8
if edgemode {
temp = binary.LittleEndian.Uint64(data[start:])
id2 = dvid.VertexID(temp)
start += 8
}
data_size := binary.LittleEndian.Uint64(data[start:])
start += 8
data_begin := start
start += int(data_size)
data_end := start
if data_begin == data_end {
continue
}
// check if post is possible
if _, ok := transactions.locked_ids[id]; !ok {
continue
}
if edgemode {
if _, ok := transactions.locked_ids[id2]; !ok {
continue
}
}
// execute post
serialization, err := dvid.SerializeData(data[data_begin:data_end], d.Compression(), d.Checksum())
if err != nil {
return fmt.Errorf("Unable to serialize data: %v\n", err)
}
if edgemode {
err = db.SetEdgeProperty(ctx, id, id2, propertyname, serialization)
} else {
err = db.SetVertexProperty(ctx, id, propertyname, serialization)
}
if err != nil {
return fmt.Errorf("Failed to add property %s: %v\n", propertyname, err)
}
}
} else {
num_properties := binary.LittleEndian.Uint64(data[start:])
start += 8
num_properties_loc := len(returned_data)
longbuf := make([]byte, 8, 8)
binary.LittleEndian.PutUint64(longbuf, 0)
returned_data = append(returned_data, longbuf...)
num_executed_transactions := uint64(0)
// read the vertex or edge properties desired
for i := uint64(0); i < num_properties; i++ {
temp := binary.LittleEndian.Uint64(data[start:])
id := dvid.VertexID(temp)
var id2 dvid.VertexID
start += 8
if edgemode {
temp := binary.LittleEndian.Uint64(data[start:])
id2 = dvid.VertexID(temp)
start += 8
}
// check if post is possible
if _, ok := transactions.locked_ids[id]; !ok {
continue
}
if edgemode {
if _, ok := transactions.locked_ids[id2]; !ok {
continue
}
}
// execute get command
var dataout []byte
if edgemode {
dataout, err = db.GetEdgeProperty(ctx, id, id2, propertyname)
} else {
dataout, err = db.GetVertexProperty(ctx, id, propertyname)
}
// serialize return data only if there is return data and no error;
// otherwise return just return the id and size of 0
var data_serialized []byte
if (err == nil) && len(dataout) > 0 {
uncompress := true
data_serialized, _, err = dvid.DeserializeData(dataout, uncompress)
if err != nil {
return fmt.Errorf("Unable to deserialize data for property '%s': %v\n", propertyname, err)
}
}
// save transaction
num_executed_transactions += 1
binary.LittleEndian.PutUint64(longbuf, uint64(id))
returned_data = append(returned_data, longbuf...)
if edgemode {
binary.LittleEndian.PutUint64(longbuf, uint64(id2))
returned_data = append(returned_data, longbuf...)
}
binary.LittleEndian.PutUint64(longbuf, uint64(len(data_serialized)))
returned_data = append(returned_data, longbuf...)
returned_data = append(returned_data, data_serialized...)
}
// update the number of transactions
binary.LittleEndian.PutUint64(returned_data[num_properties_loc:], num_executed_transactions)
}
w.Header().Set("Content-Type", "application/octet-stream")
_, err = w.Write(returned_data)
return err
}
