Example #1
0
// BatchWrites returns a channel of WriteRequests for the given entries.
// Consecutive entries with the same Source will be collected in the same
// WriteRequest, with each request containing up to maxSize updates.
func BatchWrites(entries <-chan *spb.Entry, maxSize int) <-chan *spb.WriteRequest {
	ch := make(chan *spb.WriteRequest)
	go func() {
		defer close(ch)
		var req *spb.WriteRequest
		for entry := range entries {
			update := &spb.WriteRequest_Update{
				EdgeKind:  entry.EdgeKind,
				Target:    entry.Target,
				FactName:  entry.FactName,
				FactValue: entry.FactValue,
			}

			if req != nil && (!compare.VNamesEqual(req.Source, entry.Source) || len(req.Update) >= maxSize) {
				ch <- req
				req = nil
			}

			if req == nil {
				req = &spb.WriteRequest{
					Source: entry.Source,
					Update: []*spb.WriteRequest_Update{update},
				}
			} else {
				req.Update = append(req.Update, update)
			}
		}
		if req != nil {
			ch <- req
		}
	}()
	return ch
}
Example #2
0
// Write implements part of the graphstore.Service interface.
func (db *DB) Write(ctx context.Context, req *spb.WriteRequest) error {
	if req.GetSource() == nil {
		return fmt.Errorf("missing Source in WriteRequest: {%v}", req)
	}
	tx, err := db.Begin()
	if err != nil {
		return fmt.Errorf("sql transaction begin error: %v", err)
	}
	writeStmt := tx.Stmt(db.writeStmt)
	defer func() {
		if err := writeStmt.Close(); err != nil {
			log.Printf("error closing SQL Stmt: %v", err)
		}
	}()
	for _, update := range req.Update {
		if update.Target == nil {
			update.Target = emptyVName
		}
		_, err := writeStmt.Exec(
			req.Source.Signature, req.Source.Corpus, req.Source.Root, req.Source.Path, req.Source.Language,
			update.EdgeKind,
			update.FactName,
			update.Target.Signature, update.Target.Corpus, update.Target.Root, update.Target.Path, update.Target.Language,
			update.FactValue)
		if err != nil {
			tx.Rollback()
			return fmt.Errorf("sql insertion error: %v", err)
		}
	}
	if err := tx.Commit(); err != nil {
		return fmt.Errorf("sql commit error: %v", err)
	}
	return nil
}
Example #3
0
func writeEdges(ctx context.Context, t table.Proto, edges <-chan *spb.Entry) error {
	tempDir, err := ioutil.TempDir("", "reverse.edges")
	if err != nil {
		return fmt.Errorf("failed to create temporary directory: %v", err)
	}
	defer func() {
		drainEntries(edges) // ensure channel is drained on errors
		log.Println("Removing temporary edges table", tempDir)
		if err := os.RemoveAll(tempDir); err != nil {
			log.Printf("Failed to remove temporary directory %q: %v", tempDir, err)
		}
	}()
	gs, err := leveldb.OpenGraphStore(tempDir, nil)
	if err != nil {
		return fmt.Errorf("failed to create temporary GraphStore: %v", err)
	}
	defer gs.Close(ctx)

	log.Println("Writing temporary reverse edges table")
	var writeReq *spb.WriteRequest
	for e := range edges {
		if writeReq != nil && !compare.VNamesEqual(e.Source, writeReq.Source) {
			if err := writeWithReverses(ctx, gs, writeReq); err != nil {
				return err
			}
			writeReq = nil
		}
		if writeReq == nil {
			writeReq = &spb.WriteRequest{Source: e.Source}
		}
		writeReq.Update = append(writeReq.Update, &spb.WriteRequest_Update{
			Target:    e.Target,
			EdgeKind:  e.EdgeKind,
			FactName:  e.FactName,
			FactValue: e.FactValue,
		})
	}
	if writeReq != nil {
		if err := writeWithReverses(ctx, gs, writeReq); err != nil {
			return err
		}
	}

	return writeEdgePages(ctx, t, gs)
}