Example #1
0
// deserializeVertex deserializes a dvid.GraphVertex (compression turned off for now)
func (db *GraphKeyValueDB) deserializeVertex(vertexdata []byte) (dvid.GraphVertex, error) {
	// create vertex to be returned
	vert := dvid.GraphVertex{GraphElement: &dvid.GraphElement{}}

	// if vertexdata is empty return an error
	if vertexdata == nil || len(vertexdata) == 0 {
		return vert, fmt.Errorf("Vertex data empty")
	}

	// boilerplate deserialization from DVID
	data, _, err := dvid.DeserializeData(vertexdata, true)
	if err != nil {
		return vert, err
	}

	// load data from vertex (vertex id, vertex weight, num vertices,
	// vertex array, num properties, property array

	// load vertex id
	start := 0
	vert.Id = dvid.VertexID(binary.LittleEndian.Uint64(data[start:]))
	start += 8

	// load vertex weight
	floatbits := binary.LittleEndian.Uint64(data[start:])
	vert.Weight = math.Float64frombits(floatbits)
	start += 8

	// number of vertices
	count := binary.LittleEndian.Uint64(data[start:])
	start += 8

	vert.Vertices = make([]dvid.VertexID, count, count)
	// load vertices
	for i := uint64(0); i < count; i++ {
		vert.Vertices[i] = dvid.VertexID(binary.LittleEndian.Uint64(data[start:]))
		start += 8
	}

	// number of properties
	vert.Properties = make(dvid.ElementProperties)
	count = binary.LittleEndian.Uint64(data[start:])
	start += 8

	// create property strings
	for i := uint64(0); i < count; i++ {
		propertyname := string("")
		// null separated strings
		for data[start] != byte(0) {
			propertyname += string(data[start])
			start += 1
		}
		vert.Properties[propertyname] = struct{}{}

		// increment beyond null
		start += 1
	}

	return vert, nil
}
Example #2
0
// deserializeEdge deserializes a dvid.GraphEdge (compression turned off for now)
func (db *GraphKeyValueDB) deserializeEdge(edgedata []byte) (dvid.GraphEdge, error) {
	// create edge to be returned
	edge := dvid.GraphEdge{GraphElement: &dvid.GraphElement{}}

	// if edgedata is empty return an error
	if edgedata == nil || len(edgedata) == 0 {
		return edge, fmt.Errorf("Edge data empty")
	}

	// boilerplate deserialization from DVID
	data, _, err := dvid.DeserializeData(edgedata, true)
	if err != nil {
		return edge, err
	}

	// load data from edge (vertex1 id, vertex2 id, edge weight,
	// num properties, property array

	// load vertex1 id
	start := 0
	edge.Vertexpair.Vertex1 = dvid.VertexID(binary.LittleEndian.Uint64(data[start:]))
	start += 8

	// load vertex2 id
	edge.Vertexpair.Vertex2 = dvid.VertexID(binary.LittleEndian.Uint64(data[start:]))
	start += 8

	// load edge weight
	floatbits := binary.LittleEndian.Uint64(data[start:])
	edge.Weight = math.Float64frombits(floatbits)
	start += 8

	// number of properties
	count := binary.LittleEndian.Uint64(data[start:])
	start += 8

	// create property strings
	edge.Properties = make(dvid.ElementProperties)
	for i := uint64(0); i < count; i++ {
		propertyname := string("")
		// null separated strings
		for data[start] != 0 {
			propertyname += string(data[start])
			start += 1
		}
		edge.Properties[propertyname] = struct{}{}

		// increment beyond null
		start += 1
	}

	return edge, nil
}
Example #3
0
// BytesToIndex returns an Index from a byte representation
// TODO -- Add error checking on slice format.
func (i *graphIndex) IndexFromBytes(b []byte) error {
	keyType := graphType(b[0])
	vertex1 := dvid.VertexID(binary.BigEndian.Uint64(b[1 : 1+vertexIDSize]))
	vertex2 := dvid.VertexID(0)
	start := 1 + vertexIDSize
	if keyType == keyEdge || keyType == keyEdgeProperty {
		vertex2 = dvid.VertexID(binary.BigEndian.Uint64(b[start : start+vertexIDSize]))
		start += vertexIDSize
	}
	property := ""
	if keyType == keyVertexProperty || keyType == keyEdgeProperty {
		property = string(b[start:])
	}

	*i = graphIndex{keyType, vertex1, vertex2, property}
	return nil
}
Example #4
0
// createTransactionGroupBinary reads through binary representation of vertex / transaction id
// pairs and calls creatTransactionGroup
func (t *transactionLog) createTransactionGroupBinary(data []byte, readonly bool) (*transactionGroup, int, error) {
	start := 0
	numtrans := binary.LittleEndian.Uint64(data[start:])
	start += 8

	var vertices []transactionItem
	for i := uint64(0); i < numtrans; i++ {
		vertex := binary.LittleEndian.Uint64(data[start:])
		start += 8
		trans := binary.LittleEndian.Uint64(data[start:])
		start += 8

		vertices = append(vertices, transactionItem{dvid.VertexID(vertex), trans})
	}

	transaction_group, err := t.createTransactionGroup(vertices, readonly)
	return transaction_group, start, err
}
Example #5
0
// GetVertices uses a range query to get all vertices (#reads = #vertices)
func (db *GraphKeyValueDB) GetVertices(ctx Context) ([]dvid.GraphVertex, error) {
	minid := dvid.VertexID(0)
	maxid := ^minid

	keylb := &graphIndex{keyVertex, minid, 0, ""}
	keyub := &graphIndex{keyVertex, maxid, 0, ""}
	keyvalues, err := db.GetRange(ctx, keylb.Bytes(), keyub.Bytes())
	var vertexlist []dvid.GraphVertex
	if err != nil {
		return vertexlist, err
	}

	for _, keyvalue := range keyvalues {
		vertex, err := db.deserializeVertex(keyvalue.V)
		if err != nil {
			return vertexlist, err
		}
		vertexlist = append(vertexlist, vertex)
	}

	return vertexlist, nil
}
Example #6
0
// GetEdges uses a range query to get all edges (#reads = #edges)
func (db *GraphKeyValueDB) GetEdges(ctx Context) ([]dvid.GraphEdge, error) {
	minid := dvid.VertexID(0)
	maxid := ^minid

	keylb := &graphIndex{keyEdge, minid, minid, ""}
	keyub := &graphIndex{keyEdge, maxid, maxid, ""}

	keyvalues, err := db.GetRange(ctx, keylb.Bytes(), keyub.Bytes())
	var edgelist []dvid.GraphEdge
	if err != nil {
		return edgelist, err
	}

	for _, keyvalue := range keyvalues {
		edge, err := db.deserializeEdge(keyvalue.V)
		if err != nil {
			return edgelist, err
		}
		edgelist = append(edgelist, edge)
	}

	return edgelist, nil
}
Example #7
0
// handleNeighbors returns the vertices and edges connected to the provided vertex
// (Should I protect this transaction like the update weight function?  It is probably
// unnecessary because any update based on retrieved information will be written to a
// property which has transactional protection)
func (d *Data) handleNeighbors(ctx *datastore.VersionedCtx, db storage.GraphDB, w http.ResponseWriter, path []string) error {
	labelgraph := new(LabelGraph)

	temp, err := strconv.Atoi(path[0])
	if err != nil {
		return fmt.Errorf("Vertex number not provided")
	}
	id := dvid.VertexID(temp)

	storedvert, err := db.GetVertex(ctx, id)
	if err != nil {
		return fmt.Errorf("Failed to retrieve vertix %d: %v\n", id, err)
	}

	labelgraph.Vertices = append(labelgraph.Vertices, labelVertex{storedvert.Id, storedvert.Weight})

	for _, vert2 := range storedvert.Vertices {
		vertex, err := db.GetVertex(ctx, vert2)
		if err != nil {
			return fmt.Errorf("Failed to retrieve vertex %d: %v\n", vertex.Id, err)
		}
		labelgraph.Vertices = append(labelgraph.Vertices, labelVertex{vertex.Id, vertex.Weight})
		edge, err := db.GetEdge(ctx, id, vert2)
		if err != nil {
			return fmt.Errorf("Failed to retrieve edge %d-%d: %v\n", vertex.Id, vert2, err)
		}
		labelgraph.Edges = append(labelgraph.Edges, labelEdge{edge.Vertexpair.Vertex1, edge.Vertexpair.Vertex2, edge.Weight})
	}
	m, err := json.Marshal(labelgraph)
	if err != nil {
		return fmt.Errorf("Could not serialize graph")
	}
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, string(m))
	return nil
}
Example #8
0
// handleProperty retrieves or deletes properties that can be added to a vertex or edge -- data posted
// or retrieved uses default compression
func (d *Data) handleProperty(ctx *datastore.VersionedCtx, db storage.GraphDB, w http.ResponseWriter, r *http.Request, path []string, method string) error {

	edgemode := false
	var propertyname string

	if len(path) == 3 {
		edgemode = true
		propertyname = path[2]
	} else if len(path) != 2 {
		return fmt.Errorf("Incorrect number of parameters specified for handling properties")
	} else {
		propertyname = path[1]
	}
	temp, err := strconv.Atoi(path[0])
	if err != nil {
		return fmt.Errorf("Vertex number not provided")
	}
	id1 := dvid.VertexID(temp)

	id2 := dvid.VertexID(0)
	if edgemode {
		temp, err := strconv.Atoi(path[1])
		if err != nil {
			return fmt.Errorf("Vertex number not provided")
		}
		id2 = dvid.VertexID(temp)
	}

	// remove a property from a vertex or edge
	if method == "delete" {
		if edgemode {
			db.RemoveEdgeProperty(ctx, id1, id2, propertyname)
			if err != nil {
				return fmt.Errorf("Failed to remove edge property %d-%d %s: %v\n", id1, id2, propertyname, err)
			}
		} else {
			db.RemoveVertexProperty(ctx, id1, propertyname)
			if err != nil {
				return fmt.Errorf("Failed to remove vertex property %d %s: %v\n", id1, propertyname, err)
			}
		}
	} else if method == "get" {
		var data []byte
		if edgemode {
			data, err = db.GetEdgeProperty(ctx, id1, id2, propertyname)
		} else {
			data, err = db.GetVertexProperty(ctx, id1, propertyname)
		}
		if err != nil {
			return fmt.Errorf("Failed to get property %s: %v\n", propertyname, err)
		}
		uncompress := true
		value, _, e := dvid.DeserializeData(data, uncompress)
		if e != nil {
			err = fmt.Errorf("Unable to deserialize data for property '%s': %v\n", propertyname, e.Error())
			return err
		}

		w.Header().Set("Content-Type", "application/octet-stream")
		_, err = w.Write(value)
		if err != nil {
			return err
		}
	} else if method == "post" {
		// read as binary and load into propertyname
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			return err
		}
		serialization, err := dvid.SerializeData(data, d.Compression(), d.Checksum())
		if err != nil {
			return fmt.Errorf("Unable to serialize data: %v\n", err)
		}
		if edgemode {
			err = db.SetEdgeProperty(ctx, id1, id2, propertyname, serialization)
		} else {
			err = db.SetVertexProperty(ctx, id1, propertyname, serialization)
		}
		if err != nil {
			return fmt.Errorf("Failed to add property %s: %v\n", propertyname, err)
		}
	}

	return err
}
Example #9
0
// handelPropertyTransaction allows gets/posts (really puts) of edge or vertex properties.
func (d *Data) handlePropertyTransaction(ctx *datastore.VersionedCtx, db storage.GraphDB, w http.ResponseWriter, r *http.Request, path []string, method string) error {
	if len(path) < 2 {
		return fmt.Errorf("Must specify edges or vertices in URI and property name")
	}
	if method == "delete" {
		return fmt.Errorf("Transactional delete not supported")
	}

	edgemode := false
	if path[0] == "edges" {
		edgemode = true
	} else if path[0] != "vertices" {
		return fmt.Errorf("Must specify edges or vertices in URI")
	}
	propertyname := path[1]

	readonly := false
	if method == "get" {
		readonly = true
	}
	data, err := ioutil.ReadAll(r.Body)

	// only allow 1000 vertices to be locked
	transactions, start, err := d.transaction_log.createTransactionGroupBinary(data, readonly)
	defer transactions.closeTransaction()
	if err != nil {
		return fmt.Errorf("Failed to create property transaction: %v", err)
	}

	returned_data := transactions.exportTransactionsBinary()

	if method == "post" {
		// deserialize transaction (vertex or edge) -- use URI?
		num_properties := binary.LittleEndian.Uint64(data[start:])
		start += 8

		for i := uint64(0); i < num_properties; i++ {
			temp := binary.LittleEndian.Uint64(data[start:])
			id := dvid.VertexID(temp)
			var id2 dvid.VertexID
			start += 8
			if edgemode {
				temp = binary.LittleEndian.Uint64(data[start:])
				id2 = dvid.VertexID(temp)
				start += 8
			}
			data_size := binary.LittleEndian.Uint64(data[start:])
			start += 8
			data_begin := start

			start += int(data_size)
			data_end := start

			if data_begin == data_end {
				continue
			}

			// check if post is possible
			if _, ok := transactions.locked_ids[id]; !ok {
				continue
			}
			if edgemode {
				if _, ok := transactions.locked_ids[id2]; !ok {
					continue
				}
			}

			// execute post
			serialization, err := dvid.SerializeData(data[data_begin:data_end], d.Compression(), d.Checksum())
			if err != nil {
				return fmt.Errorf("Unable to serialize data: %v\n", err)
			}
			if edgemode {
				err = db.SetEdgeProperty(ctx, id, id2, propertyname, serialization)
			} else {
				err = db.SetVertexProperty(ctx, id, propertyname, serialization)
			}
			if err != nil {
				return fmt.Errorf("Failed to add property %s: %v\n", propertyname, err)
			}
		}
	} else {
		num_properties := binary.LittleEndian.Uint64(data[start:])
		start += 8
		num_properties_loc := len(returned_data)
		longbuf := make([]byte, 8, 8)
		binary.LittleEndian.PutUint64(longbuf, 0)
		returned_data = append(returned_data, longbuf...)
		num_executed_transactions := uint64(0)

		// read the vertex or edge properties desired
		for i := uint64(0); i < num_properties; i++ {
			temp := binary.LittleEndian.Uint64(data[start:])
			id := dvid.VertexID(temp)
			var id2 dvid.VertexID
			start += 8
			if edgemode {
				temp := binary.LittleEndian.Uint64(data[start:])
				id2 = dvid.VertexID(temp)
				start += 8
			}

			// check if post is possible
			if _, ok := transactions.locked_ids[id]; !ok {
				continue
			}
			if edgemode {
				if _, ok := transactions.locked_ids[id2]; !ok {
					continue
				}
			}

			// execute get command
			var dataout []byte
			if edgemode {
				dataout, err = db.GetEdgeProperty(ctx, id, id2, propertyname)
			} else {
				dataout, err = db.GetVertexProperty(ctx, id, propertyname)
			}

			// serialize return data only if there is return data and no error;
			// otherwise return just return the id and size of 0
			var data_serialized []byte
			if (err == nil) && len(dataout) > 0 {
				uncompress := true
				data_serialized, _, err = dvid.DeserializeData(dataout, uncompress)
				if err != nil {
					return fmt.Errorf("Unable to deserialize data for property '%s': %v\n", propertyname, err)
				}
			}

			// save transaction
			num_executed_transactions += 1
			binary.LittleEndian.PutUint64(longbuf, uint64(id))
			returned_data = append(returned_data, longbuf...)
			if edgemode {
				binary.LittleEndian.PutUint64(longbuf, uint64(id2))
				returned_data = append(returned_data, longbuf...)
			}
			binary.LittleEndian.PutUint64(longbuf, uint64(len(data_serialized)))
			returned_data = append(returned_data, longbuf...)
			returned_data = append(returned_data, data_serialized...)
		}

		// update the number of transactions
		binary.LittleEndian.PutUint64(returned_data[num_properties_loc:], num_executed_transactions)
	}

	w.Header().Set("Content-Type", "application/octet-stream")
	_, err = w.Write(returned_data)
	return err
}