func main() {
flag.Parse()
logrus.SetLevel(logrus.DebugLevel)
var srcl, dstl []R
f, bufReader := getReader(*src)
var err error
srcCount := 0
var strBuf bytes.Buffer
for {
err = x.ReadLine(bufReader, &strBuf)
if err != nil {
break
}
srcCount++
rnq, err := rdf.Parse(strBuf.String())
x.Checkf(err, "Unable to parse line: [%v]", strBuf.String())
srcl = append(srcl, convert(rnq))
}
if err != nil && err != io.EOF {
err := x.Errorf("Error while reading file: %v", err)
log.Fatalf("%+v", err)
}
x.Check(f.Close())
fmt.Println("Source done")
f, bufReader = getReader(*dst)
dstCount := 0
for {
err = x.ReadLine(bufReader, &strBuf)
if err != nil {
break
}
dstCount++
rnq, err := rdf.Parse(strBuf.String())
x.Checkf(err, "Unable to parse line: [%v]", strBuf.String())
dstl = append(dstl, convert(rnq))
}
if err != nil && err != io.EOF {
err := x.Errorf("Error while reading file: %v", err)
log.Fatalf("%+v", err)
}
x.Check(f.Close())
fmt.Printf("Src: [%d] Dst: [%d]\n", srcCount, dstCount)
sort.Sort(ByR(srcl))
sort.Sort(ByR(dstl))
fmt.Println("Comparing now")
//for i := 0; i < 100; i++ {
//fmt.Printf("[S,D] %v %v\n", srcl[i], dstl[i])
//}
compare(srcl, dstl)
}
func convertToNQuad(ctx context.Context, mutation string) ([]rdf.NQuad, error) {
var nquads []rdf.NQuad
r := strings.NewReader(mutation)
reader := bufio.NewReader(r)
x.Trace(ctx, "Converting to NQuad")
var strBuf bytes.Buffer
var err error
for {
err = x.ReadLine(reader, &strBuf)
if err != nil {
break
}
ln := strings.Trim(strBuf.String(), " \t")
if len(ln) == 0 {
continue
}
nq, err := rdf.Parse(ln)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while parsing RDF"))
return nquads, err
}
nquads = append(nquads, nq)
}
if err != io.EOF {
return nquads, err
}
return nquads, nil
}
func (s *state) parseStream(done chan error) {
for line := range s.input {
line = strings.Trim(line, " \t")
if len(line) == 0 {
glog.Info("Empty line.")
continue
}
glog.Debugf("Got line: %q", line)
nq, err := rdf.Parse(line)
if err != nil {
glog.WithError(err).Errorf("While parsing: %q", line)
done <- err
return
}
s.cnq <- nq
atomic.AddUint64(&s.ctr.parsed, 1)
}
done <- nil
}
// parseStream consumes the lines, converts them to nquad
// and sends them into cnq channel.
func (s *state) parseStream(wg *sync.WaitGroup) {
defer wg.Done()
for line := range s.input {
if s.Error() != nil {
return
}
line = strings.Trim(line, " \t")
if len(line) == 0 {
glog.Info("Empty line.")
continue
}
glog.Debugf("Got line: %q", line)
nq, err := rdf.Parse(line)
if err != nil {
s.SetError(err)
return
}
s.cnq <- nq
atomic.AddUint64(&s.ctr.parsed, 1)
}
}
func TestBackup(t *testing.T) {
// Index the name predicate. We ensure it doesn't show up on backup.
dir, ps := initTestBackup(t, "scalar name:string @index")
defer os.RemoveAll(dir)
defer ps.Close()
// Remove already existing backup folders is any.
bdir, err := ioutil.TempDir("", "backup")
require.NoError(t, err)
defer os.RemoveAll(bdir)
posting.CommitLists(10)
time.Sleep(time.Second)
// We have 4 friend type edges. FP("friends")%10 = 2.
err = backup(group.BelongsTo("friend"), bdir)
require.NoError(t, err)
// We have 2 name type edges(with index). FP("name")%10 =7.
err = backup(group.BelongsTo("name"), bdir)
require.NoError(t, err)
searchDir := bdir
fileList := []string{}
err = filepath.Walk(searchDir, func(path string, f os.FileInfo, err error) error {
if path != bdir {
fileList = append(fileList, path)
}
return nil
})
require.NoError(t, err)
var counts []int
for _, file := range fileList {
f, err := os.Open(file)
require.NoError(t, err)
r, err := gzip.NewReader(f)
require.NoError(t, err)
scanner := bufio.NewScanner(r)
count := 0
for scanner.Scan() {
nq, err := rdf.Parse(scanner.Text())
require.NoError(t, err)
// Subject should have uid 1/2/3/4.
require.Contains(t, []string{"_uid_:0x1", "_uid_:0x2", "_uid_:0x3", "_uid_:0x4"}, nq.Subject)
// The only value we set was "photon".
if !bytes.Equal(nq.ObjectValue, nil) {
require.Equal(t, []byte("pho\\ton"), nq.ObjectValue)
}
// The only objectId we set was uid 5.
if nq.ObjectId != "" {
require.Equal(t, "_uid_:0x5", nq.ObjectId)
}
count++
}
counts = append(counts, count)
require.NoError(t, scanner.Err())
}
// This order will bw presereved due to file naming.
require.Equal(t, []int{4, 2}, counts)
}