示例#1
0
文件: db.go 项目: oliverp19/cayley
func Load(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
	block := make([]quad.Quad, 0, cfg.LoadSize)
	count := 0
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		block = append(block, t)
		if len(block) == cap(block) {
			count += len(block)
			err := qw.AddQuadSet(block)
			if err != nil {
				return fmt.Errorf("db: failed to load data: %v", err)
			}
			block = block[:0]
			if glog.V(2) {
				glog.V(2).Infof("Wrote %d quads.", count)
			}
		}
	}
	count += len(block)
	err := qw.AddQuadSet(block)
	if err != nil {
		return fmt.Errorf("db: failed to load data: %v", err)
	}
	if glog.V(2) {
		glog.V(2).Infof("Wrote %d quads.", count)
	}

	return nil
}
示例#2
0
文件: db.go 项目: uriencedric/cayley
func Load(ts graph.TripleStore, cfg *config.Config, dec quad.Unmarshaler) error {
	bulker, canBulk := ts.(graph.BulkLoader)
	if canBulk {
		switch err := bulker.BulkLoad(dec); err {
		case nil:
			return nil
		case graph.ErrCannotBulkLoad:
			// Try individual loading.
		default:
			return err
		}
	}

	block := make([]quad.Quad, 0, cfg.LoadSize)
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		block = append(block, t)
		if len(block) == cap(block) {
			ts.AddTripleSet(block)
			block = block[:0]
		}
	}
	ts.AddTripleSet(block)

	return nil
}
示例#3
0
func (qs *TripleStore) BulkLoad(dec quad.Unmarshaler) error {
	if qs.Size() != 0 {
		return graph.ErrCannotBulkLoad
	}

	qs.session.SetSafe(nil)
	for {
		q, err := dec.Unmarshal()
		if err != nil {
			if err != io.EOF {
				return err
			}
			break
		}
		qs.writeTriple(q)
	}

	outputTo := bson.M{"replace": "nodes", "sharded": true}
	glog.Infoln("Mapreducing")
	job := mgo.MapReduce{
		Map: `function() {
      var len = this["_id"].length
      var s_key = this["_id"].slice(0, len / 4)
      var p_key = this["_id"].slice(len / 4, 2 * len / 4)
      var o_key = this["_id"].slice(2 * len / 4, 3 * len / 4)
      var c_key = this["_id"].slice(3 * len / 4)
      emit(s_key, {"_id": s_key, "Name" : this.Subject, "Size" : 1})
      emit(p_key, {"_id": p_key, "Name" : this.Predicate, "Size" : 1})
      emit(o_key, {"_id": o_key, "Name" : this.Object, "Size" : 1})
			if (this.Label != "") {
				emit(c_key, {"_id": c_key, "Name" : this.Label, "Size" : 1})
			}
    }
    `,
		Reduce: `
      function(key, value_list) {
        out = {"_id": key, "Name": value_list[0].Name}
        count = 0
        for (var i = 0; i < value_list.length; i++) {
          count = count + value_list[i].Size

        }
        out["Size"] = count
        return out
      }
    `,
		Out: outputTo,
	}
	qs.db.C("triples").Find(nil).MapReduce(&job, nil)
	glog.Infoln("Fixing")
	qs.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) {
    db.nodes.update({"_id": result._id}, result.value)
  }) }`}, {"args", bson.D{}}}, nil)

	qs.session.SetSafe(&mgo.Safe{})

	return nil
}
示例#4
0
文件: cayley.go 项目: kris-lab/cayley
func remove(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		qw.RemoveQuad(t)
	}
	return nil
}
示例#5
0
文件: db.go 项目: kris-lab/cayley
func Load(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
	block := make([]quad.Quad, 0, cfg.LoadSize)
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		block = append(block, t)
		if len(block) == cap(block) {
			qw.AddQuadSet(block)
			block = block[:0]
		}
	}
	qw.AddQuadSet(block)

	return nil
}