func TestWriter(t *testing.T) {
buf := bytes.NewBuffer(nil)
for _, c := range testData {
buf.Reset()
w := gml.NewWriter(buf)
n, err := quad.Copy(w, quad.NewReader(c.quads))
if err != nil {
t.Fatalf("write failed after %d quads: %v", n, err)
}
if err = w.Close(); err != nil {
t.Fatal("error on close:", err)
}
if c.data != buf.String() {
t.Fatalf("wrong output:\n%s\n\nvs\n\n%s", buf.String(), c.data)
}
}
}
// Dump the content of the database into a file based
// on a few different formats
func Dump(qs graph.QuadStore, outFile, typ string) error {
var f *os.File
if outFile == "-" {
f = os.Stdout
} else {
var err error
f, err = os.Create(outFile)
if err != nil {
return fmt.Errorf("could not open file %q: %v", outFile, err)
}
defer f.Close()
fmt.Printf("dumping db to file %q\n", outFile)
}
var w io.Writer = f
if filepath.Ext(outFile) == ".gz" {
gzip := gzip.NewWriter(f)
defer gzip.Close()
w = gzip
}
qr := graph.NewQuadReader(qs) //TODO: add possible support for exporting specific queries only
if typ == "quad" { // compatibility
typ = "nquads"
}
format := quad.FormatByName(typ)
if format == nil {
return fmt.Errorf("unknown format %q", typ)
} else if format.Writer == nil {
return fmt.Errorf("format %q: encoding is not supported", typ)
}
qw := format.Writer(w)
count, err := quad.Copy(qw, qr)
if err != nil {
qw.Close()
return err
}
if err = qw.Close(); err != nil {
return err
}
if outFile != "-" {
fmt.Printf("%d entries were written\n", count)
}
return nil
}
func (api *API) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) int {
if api.config.ReadOnly {
return jsonResponse(w, 400, "Database is read-only.")
}
qr := quadReaderFromRequest(r)
defer qr.Close()
h, err := api.GetHandleForRequest(r)
if err != nil {
return jsonResponse(w, 400, err)
}
n, err := quad.Copy(h, qr)
if err != nil {
return jsonResponse(w, 400, err)
}
fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n)
return 200
}
func TestWriteJSON(t *testing.T) {
buf := bytes.NewBuffer(nil)
for _, test := range writeTests {
buf.Reset()
qw := NewWriter(buf)
_, err := quad.Copy(qw, quad.NewReader(test.input))
if err != nil {
t.Errorf("Failed to %v: %v", test.message, err)
continue
}
qw.Close()
if fmt.Sprint(err) != fmt.Sprint(test.err) {
t.Errorf("Failed to %v with unexpected error, got:%v expected %v", test.message, err, test.err)
}
if got := buf.String(); got != test.expect {
t.Errorf("Failed to %v, got:%v expect:%v", test.message, got, test.expect)
}
}
}
func TestWrite(t *testing.T) {
buf := bytes.NewBuffer(nil)
for i, c := range testWriteCases {
buf.Reset()
w := NewWriter(buf)
w.SetLdContext(c.ctx)
_, err := quad.Copy(w, quad.NewReader(c.data))
if err != nil {
t.Errorf("case %d failed: %v", i, err)
} else if err = w.Close(); err != nil {
t.Errorf("case %d failed: %v", i, err)
}
data := make([]byte, buf.Len())
copy(data, buf.Bytes())
buf.Reset()
json.Indent(buf, data, "", " ")
if buf.String() != c.expect {
t.Errorf("case %d failed: wrong data returned:\n%v\n%v", i, buf.String(), c.expect)
}
}
}
func main() {
flag.Parse()
args := flag.Args()
if (!*f_drop && len(args) < 2) || (*f_drop && len(args) < 1) {
fmt.Println("usage: qconv <src> <dst>")
os.Exit(1)
}
start := time.Now()
var cnt int
defer func() {
log.Printf("written %d quads in %v", cnt, time.Since(start))
}()
var qw quad.WriteCloser
if *f_drop {
qw = devNull{}
} else {
name := args[len(args)-1]
args = args[:len(args)-1]
file, err := os.Create(name)
if err != nil {
log.Fatal(err)
}
defer file.Close()
var w io.Writer = file
ext := filepath.Ext(name)
if strings.HasSuffix(ext, ".gz") {
ext = filepath.Ext(strings.TrimSuffix(name, ".gz"))
gz := gzip.NewWriter(w)
defer gz.Close()
w = gz
}
f := quad.FormatByExt(ext)
if f == nil || f.Writer == nil {
log.Fatal("unknown extension:", ext)
}
qw = f.Writer(w)
}
defer qw.Close()
var qh []func(q quad.Quad)
if *f_types {
m := make(map[quad.IRI]struct{})
check := func(v quad.Value) {
if t, ok := v.(quad.TypedString); ok {
if _, ok = m[t.Type]; !ok {
log.Println("new type found:", t.Type)
m[t.Type] = struct{}{}
}
}
}
defer func() {
log.Printf("dataset uses %d basic types:\n%v", len(m), m)
}()
qh = append(qh, func(q quad.Quad) {
check(q.Subject)
check(q.Predicate)
check(q.Object)
})
}
if *f_lang {
m := make(map[string]struct{})
check := func(v quad.Value) {
if t, ok := v.(quad.LangString); ok {
if _, ok = m[t.Lang]; !ok {
log.Println("new language found:", t.Lang)
m[t.Lang] = struct{}{}
}
}
}
defer func() {
log.Printf("dataset uses %d languages:\n%v", len(m), m)
}()
qh = append(qh, func(q quad.Quad) {
check(q.Subject)
check(q.Predicate)
check(q.Object)
})
}
if qh != nil {
qw = quadHook{w: qw, f: qh}
}
var errored bool
for _, name := range args {
if name == "" || name == "-" {
continue
}
err := func() error {
var qr quad.ReadCloser
file, err := os.Open(name)
if err != nil {
return err
}
defer file.Close()
var r io.Reader = file
ext := filepath.Ext(name)
if strings.HasSuffix(ext, ".gz") {
ext = filepath.Ext(strings.TrimSuffix(name, ".gz"))
gz, err := gzip.NewReader(r)
if err != nil {
return err
}
defer gz.Close()
r = gz
}
f := quad.FormatByExt(ext)
if f == nil || f.Reader == nil {
return fmt.Errorf("unknown extension: %v", ext)
}
qr = f.Reader(r)
defer qr.Close()
_, err = quad.Copy(writeHook{w: qw, f: func() {
cnt++
if cnt%(1000*1000) == 0 {
log.Printf("written %dM quads in %v", cnt/1000/1000, time.Since(start))
}
}}, qr)
return err
}()
if err != nil {
log.Println("error:", err)
errored = true
continue
}
}
if errored {
os.Exit(1)
}
}