Пример #1
0
// DecompressAndLoad will load or fetch a graph from the given path, decompress
// it, and then call the given load function to process the decompressed graph.
// If no loadFn is provided, db.Load is called.
func DecompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Reader) error) error {
	var r io.Reader

	if path == "" {
		path = cfg.DatabasePath
	}
	if path == "" {
		return nil
	}
	u, err := url.Parse(path)
	if err != nil || u.Scheme == "file" || u.Scheme == "" {
		// Don't alter relative URL path or non-URL path parameter.
		if u.Scheme != "" && err == nil {
			// Recovery heuristic for mistyping "file://path/to/file".
			path = filepath.Join(u.Host, u.Path)
		}
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf("could not open file %q: %v", path, err)
		}
		defer f.Close()
		r = f
	} else {
		res, err := client.Get(path)
		if err != nil {
			return fmt.Errorf("could not get resource <%s>: %v", u, err)
		}
		defer res.Body.Close()
		r = res.Body
	}

	r, err = Decompressor(r)
	if err != nil {
		if err == io.EOF {
			return nil
		}
		return err
	}

	var dec quad.ReadCloser
	switch typ {
	case "cquad", "cquads":
		dec = cquads.NewDecoder(r)
	case "nquad", "nquads":
		dec = nquads.NewDecoder(r)
	default:
		format := quad.FormatByName(typ)
		if format == nil || format.Reader == nil {
			return fmt.Errorf("unknown quad format %q", typ)
		}
		dec = format.Reader(r)
	}
	defer dec.Close()

	if loadFn != nil {
		return loadFn(qw, cfg, dec)
	}

	return db.Load(qw, cfg, dec)
}
Пример #2
0
func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
	if api.config.ReadOnly {
		return FormatJson400(w, "Database is read-only.")
	}

	formFile, _, err := r.FormFile("NQuadFile")
	if err != nil {
		glog.Errorln(err)
		return FormatJsonError(w, 500, "Couldn't read file: "+err.Error())
	}

	defer formFile.Close()

	blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64)
	if blockErr != nil {
		blockSize = int64(api.config.LoadSize)
	}

	dec := nquads.NewDecoder(formFile)

	var (
		n int

		block = make([]*quad.Quad, 0, blockSize)
	)
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			panic("what can do this here?") // FIXME(kortschak)
		}
		block = append(block, t)
		n++
		if len(block) == cap(block) {
			api.ts.AddTripleSet(block)
			block = block[:0]
		}
	}
	api.ts.AddTripleSet(block)

	fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", n)

	return 200
}