Beispiel #1
0
func TestWriter(t *testing.T) {
	buf := bytes.NewBuffer(nil)
	for _, c := range testData {
		buf.Reset()
		w := gml.NewWriter(buf)
		n, err := quad.Copy(w, quad.NewReader(c.quads))
		if err != nil {
			t.Fatalf("write failed after %d quads: %v", n, err)
		}
		if err = w.Close(); err != nil {
			t.Fatal("error on close:", err)
		}
		if c.data != buf.String() {
			t.Fatalf("wrong output:\n%s\n\nvs\n\n%s", buf.String(), c.data)
		}
	}
}
Beispiel #2
0
// Dump the content of the database into a file based
// on a few different formats
func Dump(qs graph.QuadStore, outFile, typ string) error {
	var f *os.File
	if outFile == "-" {
		f = os.Stdout
	} else {
		var err error
		f, err = os.Create(outFile)
		if err != nil {
			return fmt.Errorf("could not open file %q: %v", outFile, err)
		}
		defer f.Close()
		fmt.Printf("dumping db to file %q\n", outFile)
	}

	var w io.Writer = f
	if filepath.Ext(outFile) == ".gz" {
		gzip := gzip.NewWriter(f)
		defer gzip.Close()
		w = gzip
	}
	qr := graph.NewQuadReader(qs) //TODO: add possible support for exporting specific queries only

	if typ == "quad" { // compatibility
		typ = "nquads"
	}
	format := quad.FormatByName(typ)
	if format == nil {
		return fmt.Errorf("unknown format %q", typ)
	} else if format.Writer == nil {
		return fmt.Errorf("format %q: encoding is not supported", typ)
	}
	qw := format.Writer(w)
	count, err := quad.Copy(qw, qr)
	if err != nil {
		qw.Close()
		return err
	}
	if err = qw.Close(); err != nil {
		return err
	}
	if outFile != "-" {
		fmt.Printf("%d entries were written\n", count)
	}
	return nil
}
Beispiel #3
0
func (api *API) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) int {
	if api.config.ReadOnly {
		return jsonResponse(w, 400, "Database is read-only.")
	}
	qr := quadReaderFromRequest(r)
	defer qr.Close()

	h, err := api.GetHandleForRequest(r)
	if err != nil {
		return jsonResponse(w, 400, err)
	}
	n, err := quad.Copy(h, qr)
	if err != nil {
		return jsonResponse(w, 400, err)
	}

	fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n)
	return 200
}
Beispiel #4
0
func TestWriteJSON(t *testing.T) {
	buf := bytes.NewBuffer(nil)
	for _, test := range writeTests {
		buf.Reset()
		qw := NewWriter(buf)
		_, err := quad.Copy(qw, quad.NewReader(test.input))
		if err != nil {
			t.Errorf("Failed to %v: %v", test.message, err)
			continue
		}
		qw.Close()
		if fmt.Sprint(err) != fmt.Sprint(test.err) {
			t.Errorf("Failed to %v with unexpected error, got:%v expected %v", test.message, err, test.err)
		}
		if got := buf.String(); got != test.expect {
			t.Errorf("Failed to %v, got:%v expect:%v", test.message, got, test.expect)
		}
	}
}
Beispiel #5
0
func TestWrite(t *testing.T) {
	buf := bytes.NewBuffer(nil)
	for i, c := range testWriteCases {
		buf.Reset()
		w := NewWriter(buf)
		w.SetLdContext(c.ctx)
		_, err := quad.Copy(w, quad.NewReader(c.data))
		if err != nil {
			t.Errorf("case %d failed: %v", i, err)
		} else if err = w.Close(); err != nil {
			t.Errorf("case %d failed: %v", i, err)
		}
		data := make([]byte, buf.Len())
		copy(data, buf.Bytes())
		buf.Reset()
		json.Indent(buf, data, "", "  ")
		if buf.String() != c.expect {
			t.Errorf("case %d failed: wrong data returned:\n%v\n%v", i, buf.String(), c.expect)
		}
	}
}
Beispiel #6
0
func main() {
	flag.Parse()
	args := flag.Args()
	if (!*f_drop && len(args) < 2) || (*f_drop && len(args) < 1) {
		fmt.Println("usage: qconv <src> <dst>")
		os.Exit(1)
	}
	start := time.Now()
	var cnt int
	defer func() {
		log.Printf("written %d quads in %v", cnt, time.Since(start))
	}()
	var qw quad.WriteCloser
	if *f_drop {
		qw = devNull{}
	} else {
		name := args[len(args)-1]
		args = args[:len(args)-1]
		file, err := os.Create(name)
		if err != nil {
			log.Fatal(err)
		}
		defer file.Close()
		var w io.Writer = file
		ext := filepath.Ext(name)
		if strings.HasSuffix(ext, ".gz") {
			ext = filepath.Ext(strings.TrimSuffix(name, ".gz"))
			gz := gzip.NewWriter(w)
			defer gz.Close()
			w = gz
		}
		f := quad.FormatByExt(ext)
		if f == nil || f.Writer == nil {
			log.Fatal("unknown extension:", ext)
		}
		qw = f.Writer(w)
	}
	defer qw.Close()

	var qh []func(q quad.Quad)
	if *f_types {
		m := make(map[quad.IRI]struct{})
		check := func(v quad.Value) {
			if t, ok := v.(quad.TypedString); ok {
				if _, ok = m[t.Type]; !ok {
					log.Println("new type found:", t.Type)
					m[t.Type] = struct{}{}
				}
			}
		}
		defer func() {
			log.Printf("dataset uses %d basic types:\n%v", len(m), m)
		}()
		qh = append(qh, func(q quad.Quad) {
			check(q.Subject)
			check(q.Predicate)
			check(q.Object)
		})
	}
	if *f_lang {
		m := make(map[string]struct{})
		check := func(v quad.Value) {
			if t, ok := v.(quad.LangString); ok {
				if _, ok = m[t.Lang]; !ok {
					log.Println("new language found:", t.Lang)
					m[t.Lang] = struct{}{}
				}
			}
		}
		defer func() {
			log.Printf("dataset uses %d languages:\n%v", len(m), m)
		}()
		qh = append(qh, func(q quad.Quad) {
			check(q.Subject)
			check(q.Predicate)
			check(q.Object)
		})
	}
	if qh != nil {
		qw = quadHook{w: qw, f: qh}
	}

	var errored bool
	for _, name := range args {
		if name == "" || name == "-" {
			continue
		}
		err := func() error {
			var qr quad.ReadCloser
			file, err := os.Open(name)
			if err != nil {
				return err
			}
			defer file.Close()

			var r io.Reader = file
			ext := filepath.Ext(name)
			if strings.HasSuffix(ext, ".gz") {
				ext = filepath.Ext(strings.TrimSuffix(name, ".gz"))
				gz, err := gzip.NewReader(r)
				if err != nil {
					return err
				}
				defer gz.Close()
				r = gz
			}
			f := quad.FormatByExt(ext)
			if f == nil || f.Reader == nil {
				return fmt.Errorf("unknown extension: %v", ext)
			}
			qr = f.Reader(r)
			defer qr.Close()
			_, err = quad.Copy(writeHook{w: qw, f: func() {
				cnt++
				if cnt%(1000*1000) == 0 {
					log.Printf("written %dM quads in %v", cnt/1000/1000, time.Since(start))
				}
			}}, qr)
			return err
		}()
		if err != nil {
			log.Println("error:", err)
			errored = true
			continue
		}
	}
	if errored {
		os.Exit(1)
	}
}