Beispiel #1
0
func prepare() (dir1, dir2 string, ps *store.Store, clog *commit.Logger, rerr error) {
	var err error
	dir1, err = ioutil.TempDir("", "storetest_")
	if err != nil {
		return "", "", nil, nil, err
	}
	ps = new(store.Store)
	ps.Init(dir1)

	dir2, err = ioutil.TempDir("", "storemuts_")
	if err != nil {
		return dir1, "", nil, nil, err
	}
	clog = commit.NewLogger(dir2, "mutations", 50<<20)
	clog.Init()

	posting.Init(clog)
	worker.Init(ps)
	uid.Init(ps)
	loader.Init(ps, ps)

	f, err := os.Open("testdata.nq")
	if err != nil {
		return dir1, dir2, nil, clog, err
	}
	defer f.Close()
	_, err = loader.HandleRdfReader(f, 0, 1)
	if err != nil {
		return dir1, dir2, nil, clog, err
	}
	return dir1, dir2, ps, clog, nil
}
Beispiel #2
0
func prepare() (dir1, dir2 string, ps *store.Store, rerr error) {
	var err error
	dir1, err = ioutil.TempDir("", "storetest_")
	if err != nil {
		return "", "", nil, err
	}
	ps, err = store.NewStore(dir1)
	if err != nil {
		return "", "", nil, err
	}

	dir2, err = ioutil.TempDir("", "wal_")
	if err != nil {
		return dir1, "", nil, err
	}

	posting.Init(ps)
	loader.Init(ps)
	group.ParseGroupConfig("groups.conf")
	worker.StartRaftNodes(dir2)

	{
		// Then load data.
		f, err := os.Open("testdata.nq")
		if err != nil {
			return dir1, dir2, nil, err
		}
		gm := map[uint32]bool{
			0: true,
		}
		_, err = loader.LoadEdges(f, gm)
		f.Close()
		if err != nil {
			return dir1, dir2, nil, err
		}
	}

	return dir1, dir2, ps, nil
}
Beispiel #3
0
func main() {
	flag.Parse()
	if !flag.Parsed() {
		glog.Fatal("Unable to parse flags")
	}
	if len(*cpuprofile) > 0 {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	logrus.SetLevel(logrus.InfoLevel)
	numCpus := *numcpu
	prevProcs := runtime.GOMAXPROCS(numCpus)
	glog.WithField("num_cpus", numCpus).
		WithField("prev_maxprocs", prevProcs).
		Info("Set max procs to num  cpus")

	glog.WithField("instanceIdx", *instanceIdx).
		WithField("numInstances", *numInstances).
		Info("Only XIDs with FP(xid)%numInstance == instanceIdx will be given UID")

	if len(*rdfGzips) == 0 {
		glog.Fatal("No RDF GZIP files specified")
	}

	ps := new(store.Store)
	ps.Init(*uidDir)
	defer ps.Close()

	posting.Init(nil)
	uid.Init(ps)
	loader.Init(nil, ps)

	files := strings.Split(*rdfGzips, ",")
	for _, path := range files {
		if len(path) == 0 {
			continue
		}
		glog.WithField("path", path).Info("Handling...")
		f, err := os.Open(path)
		if err != nil {
			glog.WithError(err).Fatal("Unable to open rdf file.")
		}

		r, err := gzip.NewReader(f)
		if err != nil {
			glog.WithError(err).Fatal("Unable to create gzip reader.")
		}

		count, err := loader.HandleRdfReaderWhileAssign(r, *instanceIdx,
			*numInstances)
		if err != nil {
			glog.WithError(err).Fatal("While handling rdf reader.")
		}
		glog.WithField("count", count).Info("RDFs parsed")
		r.Close()
		f.Close()
	}
	glog.Info("Calling merge lists")
	posting.MergeLists(100 * numCpus) // 100 per core.
}