예제 #1
0
파일: pipeline.go 프로젝트: kidaa/kythe
func createFragmentsTable(ctx context.Context, entries <-chan *spb.Entry) (t *table.KVProto, err error) {
	log.Println("Writing decoration fragments")

	defer drainEntries(entries) // ensure channel is drained on errors

	temp, err := tempTable("decor.fragments")
	if err != nil {
		return nil, fmt.Errorf("failed to create temporary table: %v", err)
	}
	fragments := &table.KVProto{temp}
	defer func() {
		if err != nil {
			if err := fragments.Close(ctx); err != nil {
				log.Println("Error closing fragments table: %v", err)
			}
		}
	}()

	for src := range assemble.Sources(entries) {
		for _, fragment := range assemble.DecorationFragments(src) {
			fileTicket := fragment.FileTicket
			var anchorTicket string
			if len(fragment.Decoration) != 0 {
				// don't keep decor.FileTicket for anchors; we aren't sure we have a file yet
				fragment.FileTicket = ""
				anchorTicket = fragment.Decoration[0].Anchor.Ticket
			}
			if err := fragments.Put(ctx, []byte(fileTicket+tempTableKeySep+anchorTicket), fragment); err != nil {
				return nil, err
			}
		}
	}
	return fragments, nil
}
예제 #2
0
파일: pipeline.go 프로젝트: benjyw/kythe
func combineNodesAndEdges(ctx context.Context, opts *Options, out *servingOutput, rdIn stream.EntryReader) (disksort.Interface, error) {
	log.Println("Writing partial edges")

	tree := filetree.NewMap()
	rd := func(f func(*spb.Entry) error) error {
		return rdIn(func(e *spb.Entry) error {
			if e.FactName == facts.NodeKind && string(e.FactValue) == nodes.File {
				tree.AddFile(e.Source)
				// TODO(schroederc): evict finished directories (based on GraphStore order)
			}
			return f(e)
		})
	}

	partialSorter, err := opts.diskSorter(edgeLesser{}, edgeMarshaler{})
	if err != nil {
		return nil, err
	}

	bIdx := out.idx.Buffered()
	if err := assemble.Sources(rd, func(src *ipb.Source) error {
		return writePartialEdges(ctx, partialSorter, bIdx, src)
	}); err != nil {
		return nil, err
	}
	if err := bIdx.Flush(ctx); err != nil {
		return nil, err
	}

	if err := writeFileTree(ctx, tree, out.xs); err != nil {
		return nil, fmt.Errorf("error writing file tree: %v", err)
	}
	tree = nil

	log.Println("Writing complete edges")

	cSorter, err := opts.diskSorter(edgeLesser{}, edgeMarshaler{})
	if err != nil {
		return nil, err
	}

	var n *srvpb.Node
	if err := partialSorter.Read(func(i interface{}) error {
		e := i.(*srvpb.Edge)
		if n == nil || n.Ticket != e.Source.Ticket {
			n = e.Source
			if e.Target != nil {
				if opts.Verbose {
					log.Printf("WARNING: missing node facts for: %q", e.Source.Ticket)
				}
				// This is needed to satisfy later parts of the pipeline that look for targetless edges
				// to signify new nodes.
				if err := cSorter.Add(&srvpb.Edge{Source: &srvpb.Node{Ticket: e.Source.Ticket}}); err != nil {
					return fmt.Errorf("error writing complete edge: %v", err)
				}
			}
		}
		if e.Target == nil {
			// pass-through self-edges
			return cSorter.Add(e)
		}
		e.Source = n
		if err := writeCompletedEdges(ctx, cSorter, e); err != nil {
			return fmt.Errorf("error writing complete edge: %v", err)
		}
		return nil
	}); err != nil {
		return nil, fmt.Errorf("error reading/writing edges: %v", err)
	}

	return cSorter, nil
}