Beispiel #1
0
// Import imports triples from an Turtle stream, in batches of given size.
// It will ignore triples with blank nodes and errors.
// It returns the total number of triples imported.
func (db *DB) Import(r io.Reader, batchSize int) (int, error) {
	dec := rdf.NewDecoder(r)
	g := rdf.NewGraph()
	c := 0 // totalt count
	i := 0 // current batch count
	for tr, err := dec.Decode(); err != io.EOF; tr, err = dec.Decode() {
		if err != nil {
			// log.Println(err.Error())
			continue
		}
		g.Insert(tr)
		i++
		if i == batchSize {
			err = db.ImportGraph(g)
			if err != nil {
				return c, err
			}
			c += i
			i = 0
			g = rdf.NewGraph()
		}
	}
	if len(g.Nodes()) > 0 {
		err := db.ImportGraph(g)
		if err != nil {
			return c, err
		}
		c += i
	}
	return c, nil
}
Beispiel #2
0
func TestImportDumpGraph_Quick(t *testing.T) {
	f := func(items testdata) bool {
		db := newTestDB()
		defer db.Close()

		RemoveDuplicates(&items)

		input := items.Graph().Serialize(rdf.Turtle, "http://test.org/")
		n, err := db.Import(bytes.NewBufferString(input), rand.Intn(1000))
		if err != nil {
			t.Logf("DB.Import() failed: %v", err)
			t.FailNow()
		}
		if n != len(items) {
			t.Logf("DB.Import() => %d; want %d", n, len(items))
			t.FailNow()
		}

		want := items.Graph()
		var b bytes.Buffer
		err = db.Dump(&b)
		if err != nil {
			t.Logf("DB.Dump() failed: %v", err)
			t.FailNow()
		}
		dec := rdf.NewDecoder(&b)
		got, err := dec.DecodeGraph()
		if err != nil {
			t.Logf("Decoding dump of DB failed: %v", err)
			t.FailNow()
		}
		if !got.Eq(want) {
			t.Log("Dump of graph not equal inserted graph")
			t.Log("got:")
			t.Log(got.Triples())
			t.Log("want:")
			t.Log(want.Triples())
			t.FailNow()
		}

		print(".")
		return true
	}
	if err := quick.Check(f, qconfig()); err != nil {
		t.Error(err)
	}
}