Example #1
0
func (db *DBStore) readBytes(name string) (*bytes.Buffer, error) {
	if db.env == nil {
		return nil, ErrDBStoreNotInitialized
	}

	var b *bytes.Buffer

	err := db.env.View(func(txn *lmdb.Txn) error {
		dbi, err := txn.OpenRoot(0)
		if err != nil {
			return err
		}

		data, err := txn.Get(dbi, []byte(name))
		if err != nil {
			return err
		}

		b = bytes.NewBuffer(data)
		return nil
	})

	if err != nil {
		// conform to semantics of store read operations and return
		// os.ErrNotExist if the entry was not found
		if lmdb.IsNotFound(err) {
			return nil, os.ErrNotExist
		}
		return nil, errors.Wrapf(err, "failed to read data for key %s", name)
	}
	return b, nil
}
Example #2
0
// Contains ???
// BUG(bmatsuo):
// Contains is surely broken. It just looks for v in the given database.  It
// seems like it should buffer the output, or run the iteration again.
func (it *AllIterator) Contains(v graph.Value) (ok bool) {
	graph.ContainsLogIn(it, v)
	defer func() { graph.ContainsLogOut(it, v, ok) }()

	tok, ok := v.(*Token)
	if !ok {
		return false
	}
	if tok.db != it.db {
		return false
	}

	err := it.qs.env.View(func(tx *lmdb.Txn) (err error) {
		tx.RawRead = true
		_, err = tx.Get(it.qs.dbis[tok.db], tok.key)
		ok = !lmdb.IsNotFound(err)
		return err
	})
	if err != nil {
		return false
	}
	if ok {
		it.result = tok
	}
	return ok

}
Example #3
0
func (c *cmdRead) run() {
	// create environment and dbi.
	numDB := 10
	c.sizeDB = 1 << 30
	var err error
	c.env, err = createEnv(c.dbfile, numDB, c.sizeDB)
	for lmdb.IsMapFull(err) {
		c.sizeDB *= 2
		c.env, err = createEnv(c.dbfile, numDB, c.sizeDB)
	}
	raiseError(err)
	defer c.env.Close()

	var sizeDB int64 = 1 << 30
	c.featureEnv, err = createNoLockEnv(c.featureDB, numDB, sizeDB)
	for lmdb.IsMapFull(err) {
		sizeDB *= 2
		c.featureEnv, err = createNoLockEnv(c.featureDB, numDB, sizeDB)
	}
	raiseError(err)
	defer c.featureEnv.Close()

	createDBI(c.env, "gene")
	// open pileup file and read SNP.
	snpChan := readPileup(c.pileupFile)
	// group SNPs into genes.
	geneChan := c.groupSNPs(snpChan)
	c.loadGenes(geneChan)

	err = c.env.View(func(tx *lmdb.Txn) error {
		dbi, err := tx.OpenDBI("gene", 0)
		if err != nil {
			return err
		}
		cur, err := tx.OpenCursor(dbi)
		if err != nil {
			return err
		}
		count := 0
		for {
			_, _, err := cur.Get(nil, nil, lmdb.Next)
			if lmdb.IsNotFound(err) {
				return nil
			}
			if err != nil {
				return err
			}
			count++
		}
		log.Printf("Total gene: %d\n", count)
		return nil
	})
	if err != nil {
		log.Panicln(err)
	}
}
Example #4
0
// Get a K/V pair
func (m *MDBStore) Get(key []byte) ([]byte, error) {
	var val []byte
	err := m.env.View(func(txn *lmdb.Txn) (err error) {
		val, err = txn.Get(m.dbConf, key)
		if lmdb.IsNotFound(err) {
			return fmt.Errorf("not found")
		}
		return err
	})
	return val, err
}
Example #5
0
func (m *MDBStore) GetUint64(key []byte) (v64 uint64, err error) {
	err = m.env.View(func(txn *lmdb.Txn) (err error) {
		txn.RawRead = true
		val, err := txn.Get(m.dbConf, key)
		if lmdb.IsNotFound(err) {
			return fmt.Errorf("not found")
		}
		if err == nil {
			v64 = bytesToUint64(val)
		}
		return err
	})
	return v64, err
}
Example #6
0
func (qs *QuadStore) getInt64ForMetaKey(tx *lmdb.Txn, key string, empty int64) (int64, error) {
	var out int64
	data, err := tx.Get(qs.metaDBI, []byte(key))
	if lmdb.IsNotFound(err) {
		return empty, nil
	}
	if err != nil {
		return 0, err
	}
	buf := bytes.NewBuffer(data)
	err = binary.Read(buf, binary.LittleEndian, &out)
	if err != nil {
		return 0, err
	}
	return out, nil
}
Example #7
0
// Gets a log entry at a given index
func (m *MDBStore) GetLog(index uint64, logOut *raft.Log) error {
	key := uint64ToBytes(index)
	return m.env.View(func(txn *lmdb.Txn) error {
		txn.RawRead = true

		val, err := txn.Get(m.dbLogs, key)
		if lmdb.IsNotFound(err) {
			return raft.ErrLogNotFound
		} else if err != nil {
			return err
		}

		// Convert the value to a log
		return decodeMsgPack(val, logOut)
	})

}
Example #8
0
// Iterate all cr results,
// output a channel of CovRes.
func (c *cmdReport) getCr() chan CovRes {
	ch := make(chan CovRes)
	fn := func(txn *lmdb.Txn) error {
		dbi, err := txn.OpenDBI("cr", 0)
		if err != nil {
			return err
		}

		cur, err := txn.OpenCursor(dbi)
		if err != nil {
			return err
		}
		defer cur.Close()

		for {
			k, v, err := cur.Get(nil, nil, lmdb.Next)
			if lmdb.IsNotFound(err) {
				break
			}

			if err != nil {
				return err
			}

			values := []float64{}
			if err := msgpack.Unmarshal(v, &values); err != nil {
				return err
			}

			ch <- CovRes{Key: k, Values: values}
		}

		return nil
	}

	go func() {
		defer close(ch)
		err := c.resultsDB.View(fn)
		if err != nil {
			log.Panicln(err)
		}
	}()
	return ch
}
Example #9
0
func getAllCr(env *lmdb.Env, dbname string) chan KeyValue {
	ch := make(chan KeyValue)
	go func() {
		defer close(ch)
		fn := func(txn *lmdb.Txn) error {
			dbi, err := txn.OpenDBI(dbname, 0)
			if err != nil {
				return err
			}

			cur, err := txn.OpenCursor(dbi)
			if err != nil {
				return err
			}

			for {
				k, v, err := cur.Get(nil, nil, lmdb.Next)
				if lmdb.IsNotFound(err) {
					break
				}

				if err != nil {
					return err
				}

				ch <- KeyValue{Key: k, Value: v}
			}
			return nil
		}

		err := env.View(fn)
		if err != nil {
			if *debug {
				log.Panicln(err)
			} else {
				log.Fatalln(err)
			}
		}
	}()
	return ch
}
Example #10
0
func (m *MDBStore) getIndex(k, v []byte, op uint) (uint64, error) {
	var k64 uint64
	err := m.env.View(func(txn *lmdb.Txn) error {
		txn.RawRead = true

		cursor, err := txn.OpenCursor(m.dbLogs)
		if err != nil {
			return err
		}
		k, _, err = cursor.Get(nil, nil, op)
		cursor.Close()

		if lmdb.IsNotFound(err) {
			return nil
		} else if err == nil {
			k64 = bytesToUint64(k)
		}

		return err
	})
	return k64, err
}
Example #11
0
func getAllSNPs(env *lmdb.Env) chan SNPArr {
	ch := make(chan SNPArr)
	fn := func(tx *lmdb.Txn) error {
		dbi, err := tx.OpenDBI("gene", 0)
		if err != nil {
			return err
		}

		cur, err := tx.OpenCursor(dbi)
		if err != nil {
			return err
		}
		defer cur.Close()

		for {
			k, v, err := cur.Get(nil, nil, lmdb.Next)
			if lmdb.IsNotFound(err) {
				return nil
			} else if err != nil {
				return err
			}

			arr := []pileup.SNP{}
			if err := msgpack.Unmarshal(v, &arr); err != nil {
				return err
			}

			ch <- SNPArr{Key: k, Arr: arr}
		}
	}

	go func() {
		defer close(ch)
		env.View(fn)
	}()

	return ch
}
Example #12
0
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
	env, err := createLMDB(path, options)
	if err != nil {
		glog.Errorln("Error, couldn't open! ", err)
		return nil, err
	}

	var qs QuadStore
	qs.env = env
	err = qs.openDBIs()
	if lmdb.IsNotFound(err) {
		return nil, errors.New("lmdb: quadstore has not been initialised")
	}

	err = qs.getMetadata()
	if err != nil {
		return nil, err
	}
	if qs.version != latestDataVersion {
		return nil, errors.New("lmdb: data version is out of date. Run cayleyupgrade for your config to update the data")
	}

	return &qs, nil
}
Example #13
0
func (c *cmdRead) queryGenome(reference string) *Genome {
	features := []*Feature{}
	fn := func(tx *lmdb.Txn) error {
		// first, we query genome bucket to
		// obtain a list of gene IDs.
		dbi, err := tx.OpenDBI("genome", 0)
		if err != nil {
			return err
		}
		v, err := tx.Get(dbi, []byte(reference))
		if err != nil {
			return err
		}
		// unpack gene IDs.
		geneIDs := []string{}
		err = msgpack.Unmarshal(v, &geneIDs)
		if err != nil {
			log.Panicln(err)
		}

		// now, we try obtain gene feature informations
		// from feature bucket.
		dbi, err = tx.OpenDBI("feature", 0)
		for _, id := range geneIDs {
			if len(id) == 0 {
				continue
			}
			v, err := tx.Get(dbi, []byte(id))
			if err != nil {
				return err
			}
			if len(v) > 0 {
				f := Feature{}
				if err := msgpack.Unmarshal(v, &f); err != nil {
					log.Panicln(err)
				}
				if f.Type == "CDS" || strings.Contains(f.Type, "RNA") {
					features = append(features, &f)
				}
			}
		}

		return nil
	}

	err := c.featureEnv.View(fn)
	if err != nil {
		if lmdb.IsNotFound(err) {
			if *debug {
				log.Printf("key %s is not found\n", reference)
			}
			return nil
		}
		log.Panicln(err)
	}

	// sort features.
	if len(features) > 0 {
		sort.Sort(ByEnd{features})
	}

	return &Genome{Reference: reference, Features: features}
}
Example #14
0
func (d *DiversityFilter) filter(buf []*sam.Record, acc string, genome []byte) (out []*sam.Record, acc1 string, genome1 []byte) {
	fn := func(txn *lmdb.Txn) error {
		dbi, err := txn.OpenDBI("read", 0)
		if err != nil {
			return err
		}

		for _, r := range buf {
			key := []byte(r.Name)
			val, err := txn.Get(dbi, key)
			if err != nil {
				if lmdb.IsNotFound(err) {
					val, err = r.MarshalText()
					if err != nil {
						return err
					}
					err = txn.Put(dbi, key, val, 0)
					if err != nil {
						return err
					}
				} else {
					return err
				}
			} else {
				var mate *sam.Record = &sam.Record{}
				err := mate.UnmarshalText(val)
				raiseError(err)
				if r.Ref.Name() == mate.Ref.Name() {
					if acc != r.Ref.Name() {
						genome, err = d.findGenome(r, d.featureDB, "fna")
						raiseError(err)
						acc = r.Ref.Name()
						if *debug {
							log.Println(acc)
						}
					}

					diff1, len1 := d.Diff(r, genome)
					diff2, len2 := d.Diff(mate, genome)
					if len1 > 0 && len2 > 0 && float64(diff1+diff2)/float64(len1+len2) <= d.Cutoff {
						out = append(out, r)
						out = append(out, mate)
					} else {
						if *debug {
							log.Printf("%d, %d, %d, %d\n", diff1, diff2, len1, len2)
						}
					}
				}

				txn.Del(dbi, key, val)
			}

		}
		return nil
	}
retry:
	err := d.db.Update(fn)
	if lmdb.IsMapFull(err) {
		d.sizeDB *= 2
		err = d.db.SetMapSize(d.sizeDB)
		raiseError(err)
		goto retry
	}
	raiseError(err)

	genome1 = genome
	acc1 = acc
	return
}