Beispiel #1
0
func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
	if api.config.ReadOnly {
		return jsonResponse(w, 400, "Database is read-only.")
	}

	formFile, _, err := r.FormFile("NQuadFile")
	if err != nil {
		glog.Errorln(err)
		return jsonResponse(w, 500, "Couldn't read file: "+err.Error())
	}
	defer formFile.Close()

	blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64)
	if blockErr != nil {
		blockSize = int64(api.config.LoadSize)
	}

	quadReader, err := internal.Decompressor(formFile)
	// TODO(kortschak) Make this configurable from the web UI.
	dec := cquads.NewDecoder(quadReader)

	h, err := api.GetHandleForRequest(r)
	if err != nil {
		return jsonResponse(w, 400, err)
	}

	n, err := quad.CopyBatch(h, dec, int(blockSize))
	if err != nil {
		return jsonResponse(w, 400, err)
	}
	fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n)

	return 200
}
Beispiel #2
0
// DecompressAndLoad will load or fetch a graph from the given path, decompress
// it, and then call the given load function to process the decompressed graph.
// If no loadFn is provided, db.Load is called.
func DecompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Reader) error) error {
	var r io.Reader

	if path == "" {
		path = cfg.DatabasePath
	}
	if path == "" {
		return nil
	}
	u, err := url.Parse(path)
	if err != nil || u.Scheme == "file" || u.Scheme == "" {
		// Don't alter relative URL path or non-URL path parameter.
		if u.Scheme != "" && err == nil {
			// Recovery heuristic for mistyping "file://path/to/file".
			path = filepath.Join(u.Host, u.Path)
		}
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf("could not open file %q: %v", path, err)
		}
		defer f.Close()
		r = f
	} else {
		res, err := client.Get(path)
		if err != nil {
			return fmt.Errorf("could not get resource <%s>: %v", u, err)
		}
		defer res.Body.Close()
		r = res.Body
	}

	r, err = Decompressor(r)
	if err != nil {
		if err == io.EOF {
			return nil
		}
		return err
	}

	var dec quad.ReadCloser
	switch typ {
	case "cquad", "cquads":
		dec = cquads.NewDecoder(r)
	case "nquad", "nquads":
		dec = nquads.NewDecoder(r)
	default:
		format := quad.FormatByName(typ)
		if format == nil || format.Reader == nil {
			return fmt.Errorf("unknown quad format %q", typ)
		}
		dec = format.Reader(r)
	}
	defer dec.Close()

	if loadFn != nil {
		return loadFn(qw, cfg, dec)
	}

	return db.Load(qw, cfg, dec)
}
Beispiel #3
0
func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
	if api.config.ReadOnly {
		return jsonResponse(w, 400, "Database is read-only.")
	}

	formFile, _, err := r.FormFile("NQuadFile")
	if err != nil {
		glog.Errorln(err)
		return jsonResponse(w, 500, "Couldn't read file: "+err.Error())
	}
	defer formFile.Close()

	blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64)
	if blockErr != nil {
		blockSize = int64(api.config.LoadSize)
	}

	quadReader, err := internal.Decompressor(formFile)
	// TODO(kortschak) Make this configurable from the web UI.
	dec := cquads.NewDecoder(quadReader)

	h, err := api.GetHandleForRequest(r)
	if err != nil {
		return jsonResponse(w, 400, err)
	}

	var (
		n     int
		block = make([]quad.Quad, 0, blockSize)
	)
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			glog.Fatalln("what can do this here?", err) // FIXME(kortschak)
		}
		block = append(block, t)
		n++
		if len(block) == cap(block) {
			h.QuadWriter.AddQuadSet(block)
			block = block[:0]
		}
	}
	h.QuadWriter.AddQuadSet(block)

	fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n)

	return 200
}
Beispiel #4
0
func Load(ts graph.TripleStore, cfg *config.Config, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("could not open file %q: %v", path, err)
	}
	defer f.Close()

	r, err := decompressor(f)
	if err != nil {
		glog.Fatalln(err)
	}

	dec := cquads.NewDecoder(r)

	bulker, canBulk := ts.(graph.BulkLoader)
	if canBulk {
		err = bulker.BulkLoad(dec)
		if err == nil {
			return nil
		}
		if err == graph.ErrCannotBulkLoad {
			err = nil
		}
	}
	if err != nil {
		return err
	}

	block := make([]*quad.Quad, 0, cfg.LoadSize)
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		block = append(block, t)
		if len(block) == cap(block) {
			ts.AddTripleSet(block)
			block = block[:0]
		}
	}
	ts.AddTripleSet(block)

	return nil
}
Beispiel #5
0
func loadGraph(path string, t testing.TB) []quad.Quad {
	var r io.Reader
	var simpleGraph []quad.Quad
	f, err := os.Open(path)
	if err != nil {
		t.Fatalf("Failed to open %q: %v", path, err)
	}
	defer f.Close()
	r = f

	dec := cquads.NewDecoder(r)
	for q1, err := dec.ReadQuad(); err == nil; q1, err = dec.Unmarshal() {
		simpleGraph = append(simpleGraph, q1)
	}
	if err != nil {
		t.Fatalf("Failed to Unmarshal: %v", err)
	}
	return simpleGraph
}