Example #1
0
func main() {
	flag.Parse()
	logrus.SetLevel(logrus.DebugLevel)
	var srcl, dstl []R
	f, bufReader := getReader(*src)
	var err error

	srcCount := 0
	var strBuf bytes.Buffer
	for {
		err = x.ReadLine(bufReader, &strBuf)
		if err != nil {
			break
		}
		srcCount++
		rnq, err := rdf.Parse(strBuf.String())
		x.Checkf(err, "Unable to parse line: [%v]", strBuf.String())
		srcl = append(srcl, convert(rnq))
	}
	if err != nil && err != io.EOF {
		err := x.Errorf("Error while reading file: %v", err)
		log.Fatalf("%+v", err)
	}
	x.Check(f.Close())
	fmt.Println("Source done")

	f, bufReader = getReader(*dst)
	dstCount := 0
	for {
		err = x.ReadLine(bufReader, &strBuf)
		if err != nil {
			break
		}
		dstCount++
		rnq, err := rdf.Parse(strBuf.String())
		x.Checkf(err, "Unable to parse line: [%v]", strBuf.String())
		dstl = append(dstl, convert(rnq))
	}
	if err != nil && err != io.EOF {
		err := x.Errorf("Error while reading file: %v", err)
		log.Fatalf("%+v", err)
	}
	x.Check(f.Close())

	fmt.Printf("Src: [%d] Dst: [%d]\n", srcCount, dstCount)
	sort.Sort(ByR(srcl))
	sort.Sort(ByR(dstl))
	fmt.Println("Comparing now")
	//for i := 0; i < 100; i++ {
	//fmt.Printf("[S,D] %v %v\n", srcl[i], dstl[i])
	//}
	compare(srcl, dstl)
}
Example #2
0
func convertToNQuad(ctx context.Context, mutation string) ([]rdf.NQuad, error) {
	var nquads []rdf.NQuad
	r := strings.NewReader(mutation)
	reader := bufio.NewReader(r)
	x.Trace(ctx, "Converting to NQuad")

	var strBuf bytes.Buffer
	var err error
	for {
		err = x.ReadLine(reader, &strBuf)
		if err != nil {
			break
		}
		ln := strings.Trim(strBuf.String(), " \t")
		if len(ln) == 0 {
			continue
		}
		nq, err := rdf.Parse(ln)
		if err != nil {
			x.TraceError(ctx, x.Wrapf(err, "Error while parsing RDF"))
			return nquads, err
		}
		nquads = append(nquads, nq)
	}
	if err != io.EOF {
		return nquads, err
	}
	return nquads, nil
}
Example #3
0
// readLines reads the file and pushes the nquads onto a channel.
// Run this in a single goroutine. This function closes s.input channel.
func (s *state) readLines(r io.Reader) {
	var buf []string
	var err error
	var strBuf bytes.Buffer
	bufReader := bufio.NewReader(r)
	// Randomize lines to avoid contention on same subject.
	for i := 0; i < 1000; i++ {
		err = x.ReadLine(bufReader, &strBuf)
		if err != nil {
			break
		}
		buf = append(buf, strBuf.String())
		atomic.AddUint64(&s.ctr.read, 1)
	}

	if err != nil && err != io.EOF {
		err := x.Errorf("Error while reading file: %v", err)
		log.Fatalf("%+v", err)
	}

	// If we haven't yet finished reading the file read the rest of the rows.
	for {
		err = x.ReadLine(bufReader, &strBuf)
		if err != nil {
			break
		}
		k := rand.Intn(len(buf))
		s.input <- buf[k]
		buf[k] = strBuf.String()
		atomic.AddUint64(&s.ctr.read, 1)
	}

	if err != io.EOF {
		err := x.Errorf("Error while reading file: %v", err)
		log.Fatalf("%+v", err)
	}
	for i := 0; i < len(buf); i++ {
		s.input <- buf[i]
	}
	close(s.input)
}