Example #1
0
// Open opens a kindex file at the given path (using vfs.Open) and reads
// its contents into memory.
func Open(ctx context.Context, path string) (*Compilation, error) {
	f, err := vfs.Open(ctx, path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return New(f)
}
Example #2
0
func main() {
	flag.Parse()
	if gs == nil && *entriesFile == "" {
		flagutil.UsageError("missing --graphstore or --entries")
	} else if gs != nil && *entriesFile != "" {
		flagutil.UsageError("--graphstore and --entries are mutually exclusive")
	} else if *tablePath == "" {
		flagutil.UsageError("missing required --out flag")
	}

	db, err := leveldb.Open(*tablePath, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	ctx := context.Background()

	if err := profile.Start(ctx); err != nil {
		log.Fatal(err)
	}
	defer profile.Stop()

	var rd stream.EntryReader
	if gs != nil {
		rd = func(f func(e *spb.Entry) error) error {
			defer gs.Close(ctx)
			return gs.Scan(ctx, &spb.ScanRequest{}, f)
		}
	} else {
		f, err := vfs.Open(ctx, *entriesFile)
		if err != nil {
			log.Fatalf("Error opening %q: %v", *entriesFile, err)
		}
		defer f.Close()
		rd = stream.NewReader(f)
	}

	if err := pipeline.Run(ctx, rd, db, &pipeline.Options{
		Verbose:        *verbose,
		MaxPageSize:    *maxPageSize,
		CompressShards: *compressShards,
		MaxShardSize:   *maxShardSize,
		IOBufferSize:   int(shardIOBufferSize.Bytes()),
	}); err != nil {
		log.Fatal("FATAL ERROR: ", err)
	}
}
Example #3
0
func readDirtyBuffer(ctx context.Context) []byte {
	if *dirtyBuffer == "" {
		return nil
	}

	f, err := vfs.Open(ctx, *dirtyBuffer)
	if err != nil {
		log.Fatalf("ERROR: could not open dirty buffer at %q: %v", *dirtyBuffer, err)
	}
	defer f.Close()
	data, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatalf("ERROR: could read dirty buffer at %q: %v", *dirtyBuffer, err)
	}
	return data
}
Example #4
0
func main() {
	flag.Parse()

	f, err := vfs.Open(context.Background(), flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	data, err := ioutil.ReadAll(f)
	f.Close()
	if err != nil {
		log.Fatal(err)
	}

	var xa xapb.ExtraActionInfo
	if err := proto.Unmarshal(data, &xa); err != nil {
		log.Fatal(err)
	}

	obj := make(map[string]interface{})
	obj["extra_action_info"] = &xa

	xs, err := proto.GetExtensions(&xa, knownExtensions)
	if err != nil {
		log.Fatal(err)
	}

	var extensions []interface{}
	for _, e := range xs {
		if e != nil {
			extensions = append(extensions, e)
		}
	}

	if len(extensions) > 0 {
		obj["extensions"] = extensions
	}

	if err := json.NewEncoder(os.Stdout).Encode(obj); err != nil {
		log.Fatal(err)
	}

}
Example #5
0
// Client returns an oauth2-based http.Client based on the receiver Config.
func (c *Config) Client(ctx context.Context) (*http.Client, error) {
	switch {
	case c.GCEAccount != "":
		return oauth2.NewClient(ctx, google.ComputeTokenSource(c.GCEAccount)), nil
	case c.ConfigFile != "":
		f, err := vfs.Open(ctx, c.ConfigFile)
		if err != nil {
			return nil, fmt.Errorf("error opening ConfigFile: %v", err)
		}
		data, err := ioutil.ReadAll(f)
		f.Close()
		if err != nil {
			return nil, fmt.Errorf("error reading ConfigFile: %v", err)
		}
		config, err := google.JWTConfigFromJSON(data, c.Scopes...)
		if err != nil {
			return nil, fmt.Errorf("JWT config error: %v", err)
		}
		return config.Client(ctx), nil
	default:
		return nil, errors.New("oauth2 misconfigured")
	}
}
Example #6
0
func main() {
	flag.Parse()

	f, err := vfs.Open(context.Background(), flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	data, err := ioutil.ReadAll(f)
	f.Close()
	if err != nil {
		log.Fatal(err)
	}

	var tr tspb.TestResultData
	if err := proto.Unmarshal(data, &tr); err != nil {
		log.Fatal(err)
	}

	if err := json.NewEncoder(os.Stdout).Encode(&tr); err != nil {
		log.Fatal(err)
	}

}
Example #7
0
func main() {
	flag.Parse()

	if len(flag.Args()) > 2 || (gs != nil && len(flag.Args()) > 1) {
		fmt.Fprintf(os.Stderr, "ERROR: too many arguments %v\n", flag.Args())
		flag.Usage()
		os.Exit(1)
	}

	if gs != nil {
		defer gsutil.LogClose(context.Background(), gs)
	}

	var in io.ReadCloser = os.Stdin
	if gs == nil && len(flag.Args()) > 0 {
		file, err := vfs.Open(context.Background(), flag.Arg(0))
		if err != nil {
			log.Fatalf("Failed to open input file %q: %v", flag.Arg(0), err)
		}
		defer file.Close()
		in = file
	}

	outIdx := 1
	if gs != nil {
		outIdx = 0
	}

	var out io.WriteCloser = os.Stdout
	if len(flag.Args()) > outIdx {
		file, err := vfs.Create(context.Background(), flag.Arg(outIdx))
		if err != nil {
			log.Fatalf("Failed to create output file %q: %v", flag.Arg(outIdx), err)
		}
		defer file.Close()
		out = file
	}

	var (
		entries      <-chan *spb.Entry
		reverseEdges int
		triples      int
	)

	if gs == nil {
		entries = stream.ReadEntries(in)
	} else {
		ch := make(chan *spb.Entry)
		entries = ch
		go func() {
			defer close(ch)
			if err := gs.Scan(context.Background(), &spb.ScanRequest{}, func(e *spb.Entry) error {
				ch <- e
				return nil
			}); err != nil {
				log.Fatalf("Error scanning graphstore: %v", err)
			}
		}()
	}

	for entry := range entries {
		if schema.EdgeDirection(entry.EdgeKind) == schema.Reverse && !*keepReverseEdges {
			reverseEdges++
			continue
		}

		t, err := toTriple(entry)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Fprintln(out, t)
		triples++
	}

	if !*quiet {
		if !*keepReverseEdges {
			log.Printf("Skipped %d reverse edges", reverseEdges)
		}
		log.Printf("Wrote %d triples", triples)
	}
}
Example #8
0
			flag.StringVar(&decorSpan, "span", "", spanHelp)
		},
		func(flag *flag.FlagSet) error {
			req := &xpb.DecorationsRequest{
				Location: &xpb.Location{
					Ticket: flag.Arg(0),
				},
				References: true,
				Filter: []string{
					schema.NodeKindFact,
					schema.SubkindFact,
					schema.AnchorLocFilter, // TODO(schroederc): remove this backwards-compatibility fix
				},
			}
			if dirtyFile != "" {
				f, err := vfs.Open(ctx, dirtyFile)
				if err != nil {
					return fmt.Errorf("error opening dirty buffer file at %q: %v", dirtyFile, err)
				}
				buf, err := ioutil.ReadAll(f)
				if err != nil {
					f.Close()
					return fmt.Errorf("error reading dirty buffer file: %v", err)
				} else if err := f.Close(); err != nil {
					return fmt.Errorf("error closing dirty buffer file: %v", err)
				}
				req.DirtyBuffer = buf
			}
			if decorSpan != "" {
				start, end, err := parseSpan(decorSpan)
				if err != nil {